lawspec 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +297 -2
- package/README.md +3 -3
- package/bin/lawspec.mjs +260 -32
- package/build.json +292 -100
- package/core.wasm +0 -0
- package/core_jsffi.js +2 -2
- package/examples/specs/abilities.lawspec +103 -0
- package/examples/specs/ability_imports.lawspec +107 -0
- package/examples/specs/builtins.lawspec +121 -0
- package/examples/specs/crypto.lawspec +51 -0
- package/examples/specs/distribution.lawspec +29 -0
- package/examples/specs/failures.lawspec +96 -0
- package/examples/specs/handler_bindings.lawspec +65 -0
- package/examples/specs/harness.lawspec +75 -0
- package/examples/specs/harness_testing.lawspec +45 -0
- package/examples/specs/matchers.lawspec +112 -0
- package/examples/specs/resources.lawspec +116 -0
- package/examples/specs/scheduling_order.lawspec +18 -0
- package/examples/specs/scheduling_overlap.lawspec +14 -0
- package/examples/specs/tables.lawspec +64 -0
- package/index.d.ts +2 -0
- package/package.json +1 -1
- package/templates.mjs +14 -14
- package/test-command.mjs +165 -30
package/bin/lawspec.mjs
CHANGED
|
@@ -8,8 +8,9 @@ import { targets, templates, commands, setup } from "../templates.mjs";
|
|
|
8
8
|
import { generateExamples } from "../examples-command.mjs";
|
|
9
9
|
import { showScalar } from "../scalars.mjs";
|
|
10
10
|
import { doctor } from "../doctor.mjs";
|
|
11
|
-
import { spawn } from "node:child_process";
|
|
12
|
-
import { environmentDigest, executedTests, invocations, lawKeys, projectDigest
|
|
11
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
12
|
+
import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
13
|
+
harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
|
|
13
14
|
import {
|
|
14
15
|
readOptional,
|
|
15
16
|
planWrites,
|
|
@@ -17,18 +18,23 @@ import {
|
|
|
17
18
|
atomicWrite,
|
|
18
19
|
safePath,
|
|
19
20
|
} from "../files.mjs";
|
|
20
|
-
const VERSION = "0.
|
|
21
|
+
const VERSION = "0.21.0";
|
|
21
22
|
const args = process.argv.slice(2);
|
|
22
23
|
const verb = args.shift();
|
|
23
24
|
const options = {};
|
|
24
25
|
const positional = [];
|
|
25
26
|
for (let i = 0; i < args.length; i++) {
|
|
26
27
|
const arg = args[i];
|
|
27
|
-
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed"].includes(arg)) {
|
|
28
|
+
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
|
|
28
29
|
if (!args[i + 1] || args[i + 1].startsWith("--"))
|
|
29
30
|
throw new Error(`Missing value for ${arg}`);
|
|
30
31
|
options[arg.slice(2)] = args[++i];
|
|
31
|
-
} else if (["--
|
|
32
|
+
} else if (["--tag", "--exclude-tag"].includes(arg)) {
|
|
33
|
+
// Repeatable, and each may list several: --tag a,b --tag c.
|
|
34
|
+
if (!args[i + 1] || args[i + 1].startsWith("--"))
|
|
35
|
+
throw new Error(`Missing value for ${arg}`);
|
|
36
|
+
options[arg.slice(2)] = [...(options[arg.slice(2)] ?? []), ...args[++i].split(",").filter(Boolean)];
|
|
37
|
+
} else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh", "--coverage", "--update-recorded", "--benchmarks"].includes(arg))
|
|
32
38
|
options[arg.slice(2)] = true;
|
|
33
39
|
else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
|
|
34
40
|
else positional.push(arg);
|
|
@@ -236,6 +242,24 @@ function showAssertion(assertion) {
|
|
|
236
242
|
if (assertion.kind === "implies") return `${assertion.guard.text} implies ${showAssertion(assertion.body)}`;
|
|
237
243
|
return assertion.items.map(showAssertion).join(" and ");
|
|
238
244
|
}
|
|
245
|
+
// An ability key, as the spec names it: example.shop::ability::Store(Int32)
|
|
246
|
+
// is Store Int32, lawspec::ability::Fail(example.shop::type::E) is Fail E.
|
|
247
|
+
function abilityName(key) {
|
|
248
|
+
const short = (text) => text.replace(/[A-Za-z0-9_.]+::(ability|type)::/g, "");
|
|
249
|
+
const match = /^(.*?)(\((.*)\))?$/.exec(short(key));
|
|
250
|
+
return match[3] ? `${match[1]} ${match[3].replace(/[()]/g, " ").replace(/;/g, " ").trim()}` : match[1];
|
|
251
|
+
}
|
|
252
|
+
// The handlers a law runs under, and the ability row of each function its
|
|
253
|
+
// expansion calls.
|
|
254
|
+
function explainAbilities(law, expansion, units) {
|
|
255
|
+
const unit = (units || []).find((u) => u.id === law.owner);
|
|
256
|
+
const called = (unit ? unit.declarations : []).filter(
|
|
257
|
+
(d) => d.uses && new RegExp(`\\b${d.name}\\b`).test(expansion),
|
|
258
|
+
);
|
|
259
|
+
const handlers = (law.handlers || []).map((h) => `\n ${abilityName(h.ability)}: ${h.handler}`).join("");
|
|
260
|
+
const rows = called.map((d) => `\n ${d.name} uses ${d.uses.map(abilityName).join(", ")}`).join("");
|
|
261
|
+
return (handlers ? `\nhandlers${handlers}` : "") + (rows ? `\nrows${rows}` : "");
|
|
262
|
+
}
|
|
239
263
|
function explainExamples(law) {
|
|
240
264
|
return law.examples.map(ex =>
|
|
241
265
|
`\nexample ${JSON.stringify(ex.name)}\n` +
|
|
@@ -243,6 +267,50 @@ function explainExamples(law) {
|
|
|
243
267
|
ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
|
|
244
268
|
).join("\n");
|
|
245
269
|
}
|
|
270
|
+
// What lawspec test last recorded for each law, across targets.
|
|
271
|
+
async function lastRuns() {
|
|
272
|
+
const folder = path.join(configRoot, ".lawspec", "results");
|
|
273
|
+
const runs = new Map();
|
|
274
|
+
for (const name of (await readOptionalDirectory(folder)) ?? []) {
|
|
275
|
+
if (!name.endsWith(".json")) continue;
|
|
276
|
+
const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
|
|
277
|
+
for (const [law, run] of Object.entries(recorded.laws ?? {})) {
|
|
278
|
+
const known = runs.get(law) ?? {};
|
|
279
|
+
runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return runs;
|
|
283
|
+
}
|
|
284
|
+
// One law's harness, as lawspec evidence shows it.
|
|
285
|
+
function harnessText(item) {
|
|
286
|
+
const h = item.harness ?? {};
|
|
287
|
+
const parts = [];
|
|
288
|
+
if (h.unit) parts.push(`harness ${h.unit}`);
|
|
289
|
+
if (h.tags?.length) parts.push(`tags ${h.tags.join(", ")}`);
|
|
290
|
+
if (h.strategies?.length) parts.push(h.strategies.map((s) => `${s.input} drawn by ${s.strategy}`).join(", "));
|
|
291
|
+
for (const c of h.cover ?? []) parts.push(`cover ${c.percent}% "${c.label}" when ${c.when.text}`);
|
|
292
|
+
for (const c of h.classify ?? []) parts.push(`classify ${c.when.text} as "${c.label}"`);
|
|
293
|
+
for (const l of h.labels ?? []) parts.push(`label ${l.text}`);
|
|
294
|
+
if (h.target) parts.push(`target maximize ${h.target.text}`);
|
|
295
|
+
if (h.timeoutMilliseconds) parts.push(`timeout ${h.timeoutMilliseconds} ms`);
|
|
296
|
+
if (h.repeat > 1) parts.push(`repeat ${h.repeat}`);
|
|
297
|
+
if (h.retries) parts.push(`retry flaky ${h.retries}`);
|
|
298
|
+
if (h.skip) parts.push(`skip "${h.skip}"`);
|
|
299
|
+
if (h.knownFailing) parts.push(`known failing "${h.knownFailing}"`);
|
|
300
|
+
if (h.orderRandom) parts.push("order random");
|
|
301
|
+
// Python runs a parallel unit's tests at the same time only with
|
|
302
|
+
// pytest-xdist; without it, parallel changes nothing there.
|
|
303
|
+
if (h.parallel) parts.push("parallel (on Python only with pytest-xdist; otherwise one after another)");
|
|
304
|
+
const lines = [` ${item.declaration.replace("::law::", "::")}: ${parts.join("; ") || "no harness settings"}`];
|
|
305
|
+
for (const run of item.adequacy ?? []) {
|
|
306
|
+
lines.push(` last run: ${run.cases} generated case(s)` +
|
|
307
|
+
(run.cover ?? []).map((c) => `; cover "${c.label}" ${c.observed}% (needs ${c.required}%)${c.met ? "" : " NOT MET"}`).join(""));
|
|
308
|
+
const counts = { ...(run.classes ?? {}), ...Object.fromEntries(Object.entries(run.labels ?? {}).map(([k, v]) => [`label ${k}`, v])) };
|
|
309
|
+
if (Object.keys(counts).length)
|
|
310
|
+
lines.push(` ${Object.entries(counts).map(([k, v]) => `${k}: ${run.cases ? (100 * v / run.cases).toFixed(1) : 0}%`).join(", ")}`);
|
|
311
|
+
}
|
|
312
|
+
return lines.join("\n");
|
|
313
|
+
}
|
|
246
314
|
// The compiler keeps work between runs in .lawspec/cache, in one folder per
|
|
247
315
|
// compiler build, so a different build never reads another's entries. The
|
|
248
316
|
// WebAssembly compiler sees only the working directory. A command that fails
|
|
@@ -272,12 +340,19 @@ async function buildDigest() {
|
|
|
272
340
|
return buildDigestValue;
|
|
273
341
|
}
|
|
274
342
|
// lawspec test: run the tests of the laws whose results may have changed,
|
|
275
|
-
// and record each passing law's key and seed (see test-command.mjs).
|
|
343
|
+
// and record each passing law's key and seed (see test-command.mjs). The
|
|
344
|
+
// harness plane chooses which: --tag and --exclude-tag select laws by their
|
|
345
|
+
// harness tags, and a skipped law runs nothing. A law whose last run failed
|
|
346
|
+
// runs first, and its failing inputs are replayed first (.lawspec/failures).
|
|
276
347
|
async function runTests(compiler, input, selected, roots, config) {
|
|
348
|
+
// Recorded values live beside lawspec.json, under recorded/<unit>/<name>.
|
|
349
|
+
const recordedFolder = path.join(configRoot, "recorded");
|
|
277
350
|
const seed = options.seed ?? String(1 + Math.floor(Math.random() * 2147483646));
|
|
278
351
|
const offline = process.env.LAWSPEC_OFFLINE === "1";
|
|
279
352
|
const build = await buildDigest();
|
|
280
353
|
const summaries = [];
|
|
354
|
+
const junit = [];
|
|
355
|
+
const report = options.report === undefined ? null : parseReport(options.report);
|
|
281
356
|
for (const [i, target] of selected.entries()) {
|
|
282
357
|
const root = roots[i];
|
|
283
358
|
const planned = diagnostics(await compiler.planGeneration({
|
|
@@ -287,53 +362,179 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
287
362
|
const pending = await planWrites(root, planned.files);
|
|
288
363
|
if (pending.changes.length)
|
|
289
364
|
throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
|
|
290
|
-
const
|
|
291
|
-
if (!
|
|
365
|
+
const report_ = await doctor(target, root, planned.files);
|
|
366
|
+
if (!report_.ok) throw new Error(`${target.language}: ${report_.message}\n${report_.instructions}`);
|
|
292
367
|
const generated = new Set(planned.files.filter((f) => f.ownership === "generated").map((f) => f.path));
|
|
293
368
|
const keys = lawKeys({
|
|
294
369
|
build, target, machineBits: input.machineBits, minify: options.minify === true,
|
|
295
370
|
tests: planned.tests, files: planned.files,
|
|
296
|
-
|
|
371
|
+
// Recorded values are spec data: a changed recording runs its laws again.
|
|
372
|
+
environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
|
|
297
373
|
project: await projectDigest(root, generated),
|
|
298
374
|
});
|
|
299
375
|
const resultsFile = path.join(configRoot, ".lawspec", "results",
|
|
300
376
|
`${target.language}-${createHash("sha256").update(root).digest("hex").slice(0, 12)}.json`);
|
|
301
377
|
const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
|
|
302
|
-
|
|
378
|
+
// Recording again runs every law, so each records what it sees now.
|
|
379
|
+
const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
|
|
380
|
+
const stale = chosen.filter((entry) => options.fresh || options.coverage || options["update-recorded"] || previous.laws[entry.law]?.key !== keys.get(entry.law));
|
|
381
|
+
// The failure database: each law whose last run failed, with the seed
|
|
382
|
+
// that exposed it. Those laws run first, with that seed, so the failing
|
|
383
|
+
// inputs are generated again; the runtimes keep their own counterexample
|
|
384
|
+
// databases beside it (Hypothesis, proptest).
|
|
385
|
+
const failures = path.join(configRoot, ".lawspec", "failures", target.language);
|
|
386
|
+
const databaseFile = path.join(failures, "laws.json");
|
|
387
|
+
const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
|
|
388
|
+
const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
|
|
389
|
+
const batches = [
|
|
390
|
+
...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
|
|
391
|
+
({ seed: String(s), entries: replayed.filter((entry) => database[entry.law].seed === s) })),
|
|
392
|
+
{ seed, entries: stale.filter((entry) => !replayed.includes(entry)) },
|
|
393
|
+
].filter((batch) => batch.entries.length);
|
|
303
394
|
const passed = [];
|
|
304
395
|
const unrun = [];
|
|
305
396
|
let failed = false;
|
|
306
397
|
const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
|
|
398
|
+
const stats = path.join(scratch, "statistics");
|
|
399
|
+
const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language) : null;
|
|
307
400
|
await rm(scratch, { recursive: true, force: true });
|
|
308
|
-
await mkdir(
|
|
401
|
+
await mkdir(stats, { recursive: true });
|
|
402
|
+
await mkdir(failures, { recursive: true });
|
|
309
403
|
await writeFile(path.join(path.dirname(scratch), ".gitignore"), "*\n");
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
404
|
+
if (coverage) {
|
|
405
|
+
await mkdir(coverage, { recursive: true });
|
|
406
|
+
const missing = await coverageMissing(target, root);
|
|
407
|
+
if (missing) console.error(`${target.language}: --coverage needs ${missing.tool}, which is not available; ${missing.install}. Running without coverage.`);
|
|
408
|
+
}
|
|
409
|
+
const useCoverage = coverage && !(await coverageMissing(target, root));
|
|
410
|
+
// parallel on Python needs pytest-xdist; without it the tests run one
|
|
411
|
+
// after another, and lawspec test says so.
|
|
412
|
+
let xdist = false;
|
|
413
|
+
if (target.language === "python" && stale.some((entry) => entry.parallel)) {
|
|
414
|
+
xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
|
|
415
|
+
if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
|
|
416
|
+
}
|
|
417
|
+
// A skipped law runs nothing, and a known-failing law's one test is
|
|
418
|
+
// expected to fail, so neither is required to show as run.
|
|
419
|
+
const expected = (entry) => !entry.skip && !entry.knownFailing;
|
|
420
|
+
const seedOf = new Map();
|
|
421
|
+
batches: for (const batch of batches) {
|
|
422
|
+
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null, xdist })) {
|
|
423
|
+
const since = Date.now() - 1000;
|
|
424
|
+
const { ok, output } = await spawned(run.command, run.args, root,
|
|
425
|
+
{ ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
|
|
426
|
+
LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
|
|
427
|
+
...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
|
|
428
|
+
const executed = await executedTests(run.report, output, root, since);
|
|
429
|
+
for (const law of run.laws) seedOf.set(law.law, batch.seed);
|
|
430
|
+
if (report) junit.push({ target: target.language, xml: await junitOf(run, executed, root, since) });
|
|
431
|
+
if (!ok) { failed = true; break batches; }
|
|
432
|
+
// A runner whose filter matched nothing reports success, so a law
|
|
433
|
+
// counts as passed only if the runner's report shows its tests ran.
|
|
434
|
+
const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
|
|
435
|
+
passed.push(...run.laws.filter((law) => ran.has(law) || !expected(law)));
|
|
436
|
+
unrun.push(...run.laws.filter((law) => !ran.has(law) && expected(law)));
|
|
437
|
+
}
|
|
320
438
|
}
|
|
321
439
|
if (unrun.length) failed = true;
|
|
440
|
+
// --benchmarks: the harness's benchmarks run after the laws, every time;
|
|
441
|
+
// they are measured, never asserted, and never cached.
|
|
442
|
+
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
443
|
+
if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
|
|
444
|
+
const { ok } = await spawned(run.command, run.args, root,
|
|
445
|
+
{ ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
|
|
446
|
+
if (!ok) { failed = true; break; }
|
|
447
|
+
}
|
|
448
|
+
const statistics = await harnessStatistics(stats);
|
|
322
449
|
const laws = Object.fromEntries(planned.tests.filter((entry) => previous.laws[entry.law]).map((entry) => [entry.law, previous.laws[entry.law]]));
|
|
323
|
-
for (const entry of passed)
|
|
450
|
+
for (const entry of passed) {
|
|
451
|
+
const label = entry.label;
|
|
452
|
+
const observed = statistics.filter((s) => s.law === label);
|
|
453
|
+
laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
|
|
454
|
+
...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
|
|
455
|
+
...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
|
|
456
|
+
.map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
|
|
457
|
+
}
|
|
458
|
+
// A law that failed is recorded with its seed; one that passed leaves.
|
|
459
|
+
for (const entry of stale) {
|
|
460
|
+
if (passed.includes(entry)) delete database[entry.law];
|
|
461
|
+
else if (seedOf.has(entry.law)) database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
|
|
462
|
+
}
|
|
463
|
+
await writeFile(path.join(path.dirname(failures), ".gitignore"), "*\n");
|
|
464
|
+
await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
|
|
324
465
|
await mkdir(path.dirname(resultsFile), { recursive: true });
|
|
325
466
|
await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
|
|
326
467
|
await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
|
|
468
|
+
const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
|
|
469
|
+
const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
|
|
470
|
+
const benchmarks = statistics.filter((s) => s.benchmark);
|
|
471
|
+
// How each parallel unit's tests actually ran at the same time.
|
|
472
|
+
const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
|
|
327
473
|
summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
328
|
-
unchanged:
|
|
329
|
-
...(
|
|
474
|
+
unchanged: chosen.length - stale.length, ok: !failed,
|
|
475
|
+
...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
|
|
476
|
+
...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
|
|
477
|
+
...(flaky.length ? { flaky: [...new Set(flaky)] } : {}),
|
|
478
|
+
...(unmet.length ? { unmetCover: unmet } : {}),
|
|
479
|
+
...(benchmarks.length ? { benchmarks } : {}),
|
|
480
|
+
...(parallelism.length ? { parallelism } : {}),
|
|
481
|
+
...(useCoverage ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
|
|
482
|
+
}
|
|
483
|
+
if (report) {
|
|
484
|
+
await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
|
|
485
|
+
await writeFile(path.resolve(report.path), mergeJunit(junit));
|
|
330
486
|
}
|
|
331
487
|
output(options.json ? summaries : summaries.map((s) =>
|
|
332
488
|
`${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
333
489
|
`, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
|
|
334
|
-
(s.
|
|
490
|
+
(s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
|
|
491
|
+
(s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
|
|
492
|
+
(s.flaky ? `\nFlaky (failed, then passed on a retry): ${s.flaky.join(", ")}` : "") +
|
|
493
|
+
(s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
|
|
494
|
+
(s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
|
|
495
|
+
(s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
|
|
496
|
+
(s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
|
|
335
497
|
if (summaries.some((s) => !s.ok)) process.exitCode = 1;
|
|
336
498
|
}
|
|
499
|
+
// --report junit=path: one JUnit report merged across targets.
|
|
500
|
+
function parseReport(value) {
|
|
501
|
+
const match = value.match(/^junit=(.+)$/);
|
|
502
|
+
if (!match) throw new Error("--report takes junit=<path>");
|
|
503
|
+
return { kind: "junit", path: match[1] };
|
|
504
|
+
}
|
|
505
|
+
// A run's JUnit XML: the runner's own where it writes one, otherwise made
|
|
506
|
+
// from what it printed.
|
|
507
|
+
async function junitOf(run, executed, root, since) {
|
|
508
|
+
if (run.report?.kind === "junit") {
|
|
509
|
+
const files = run.report.files ?? (await readdir(path.join(root, run.report.directory)).catch(() => []))
|
|
510
|
+
.filter((n) => n.endsWith(".xml")).map((n) => path.join(root, run.report.directory, n));
|
|
511
|
+
const xml = [];
|
|
512
|
+
for (const file of files) {
|
|
513
|
+
const info = await stat(path.resolve(root, file)).catch(() => null);
|
|
514
|
+
if (info && info.mtimeMs >= since) xml.push(await readFile(path.resolve(root, file), "utf8"));
|
|
515
|
+
}
|
|
516
|
+
return xml.join("\n");
|
|
517
|
+
}
|
|
518
|
+
return junitFromTests(executed);
|
|
519
|
+
}
|
|
520
|
+
// The tool --coverage needs, if it is missing.
|
|
521
|
+
async function coverageMissing(target, root) {
|
|
522
|
+
const tool = coverageTools[target.language];
|
|
523
|
+
if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
|
|
524
|
+
const probe = (command, args) => new Promise((resolve) => {
|
|
525
|
+
const child = spawn(command, args, { cwd: root, stdio: "ignore" });
|
|
526
|
+
child.on("error", () => resolve(false));
|
|
527
|
+
child.on("close", (code) => resolve(code === 0));
|
|
528
|
+
});
|
|
529
|
+
if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
|
|
530
|
+
if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
|
|
531
|
+
if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
|
|
532
|
+
if (target.language === "kotlin") {
|
|
533
|
+
const build = await readFile(path.join(root, "build.gradle.kts"), "utf8").catch(() => "");
|
|
534
|
+
return build.includes("kover") ? null : tool;
|
|
535
|
+
}
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
337
538
|
// Run a native test command, showing its output (on stderr with --json) and
|
|
338
539
|
// keeping it. Go's JSON events are shown as the text they carry.
|
|
339
540
|
function spawned(command, args, cwd, env, goJson = false) {
|
|
@@ -374,7 +575,7 @@ async function main() {
|
|
|
374
575
|
}
|
|
375
576
|
if (!verb || ["help", "--help", "-h"].includes(verb)) {
|
|
376
577
|
output(
|
|
377
|
-
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
578
|
+
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
378
579
|
targets.join(", "),
|
|
379
580
|
);
|
|
380
581
|
return;
|
|
@@ -418,8 +619,9 @@ async function main() {
|
|
|
418
619
|
if (options.example) throw new Error("--example is only supported by examples");
|
|
419
620
|
if (options.minify && !["init", "generate", "examples", "test"].includes(verb))
|
|
420
621
|
throw new Error("--minify applies to init, generate, test and examples");
|
|
421
|
-
if ((options.fresh || options.seed !== undefined
|
|
422
|
-
|
|
622
|
+
if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
|
|
623
|
+
options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
|
|
624
|
+
throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
|
|
423
625
|
if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
|
|
424
626
|
throw new Error("--seed must be a whole number");
|
|
425
627
|
if (verb === "init") return init();
|
|
@@ -476,13 +678,24 @@ async function main() {
|
|
|
476
678
|
["proved", "PROVED"],
|
|
477
679
|
["exhaustively-checked", "EXHAUSTIVELY CHECKED"],
|
|
478
680
|
["property-tested", "PROPERTY TESTED"],
|
|
681
|
+
["measured", "MEASURED"],
|
|
479
682
|
["runtime-checked", "RUNTIME CHECKED"],
|
|
683
|
+
["default-handler", "DEFAULT HANDLER"],
|
|
480
684
|
["assumed", "ASSUMED / EXTERNAL"],
|
|
685
|
+
// The harness plane: a law it marks as known to fail, a law whose last
|
|
686
|
+
// run was flaky, and a law whose tests it skips.
|
|
687
|
+
["known-failing", "KNOWN FAILING"],
|
|
688
|
+
["flaky", "FLAKY"],
|
|
689
|
+
["skipped", "SKIPPED"],
|
|
481
690
|
];
|
|
482
691
|
const evidenceSummary = (evidence) => {
|
|
483
692
|
if (!evidence.length) return "";
|
|
484
693
|
const count = (status) => evidence.filter((item) => item.status === status).length;
|
|
485
|
-
|
|
694
|
+
// The harness statuses (after assumed) are counted only when a harness
|
|
695
|
+
// gives some.
|
|
696
|
+
const harnessFrom = statuses.findIndex(([status]) => status === "known-failing");
|
|
697
|
+
const shown = statuses.filter(([status], i) => i < harnessFrom || count(status));
|
|
698
|
+
return ` Evidence: ${shown.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
|
|
486
699
|
};
|
|
487
700
|
const obligationName = (item) => item.declaration.replace("::law::", "::");
|
|
488
701
|
if (verb === "check") {
|
|
@@ -512,10 +725,21 @@ async function main() {
|
|
|
512
725
|
evidence = evidence.filter((item) => !positional[0] ||
|
|
513
726
|
obligationName(item) === positional[0] || item.owner === positional[0]);
|
|
514
727
|
if (positional[0] && !evidence.length) throw new Error("No matching obligation");
|
|
728
|
+
// The last run's outcome (lawspec test): a law that failed, then passed
|
|
729
|
+
// on a retry, is flaky; what its generated cases covered is its adequacy.
|
|
730
|
+
const runs = await lastRuns();
|
|
731
|
+
evidence = evidence.map((item) => {
|
|
732
|
+
const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
|
|
733
|
+
if (!run) return item;
|
|
734
|
+
return { ...item, ...(run.adequacy ? { adequacy: run.adequacy } : {}),
|
|
735
|
+
...(run.flaky && ["property-tested", "exhaustively-checked"].includes(item.status)
|
|
736
|
+
? { status: "flaky", reason: `${item.reason}; its last run failed, then passed on a retry` } : {}) };
|
|
737
|
+
});
|
|
738
|
+
const harnessed = evidence.filter((item) => item.harness || item.adequacy);
|
|
515
739
|
output(
|
|
516
740
|
options.json
|
|
517
741
|
? evidence
|
|
518
|
-
: statuses
|
|
742
|
+
: [...statuses
|
|
519
743
|
.map(([status, label]) => {
|
|
520
744
|
const items = evidence.filter((item) => item.status === status);
|
|
521
745
|
if (!items.length) return null;
|
|
@@ -523,7 +747,10 @@ async function main() {
|
|
|
523
747
|
` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
|
|
524
748
|
(item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
|
|
525
749
|
})
|
|
526
|
-
.filter((section) => section !== null)
|
|
750
|
+
.filter((section) => section !== null),
|
|
751
|
+
// How each law is discharged: the harness plane, apart from what
|
|
752
|
+
// the laws say.
|
|
753
|
+
...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
|
|
527
754
|
.join("\n\n"),
|
|
528
755
|
);
|
|
529
756
|
return;
|
|
@@ -542,7 +769,8 @@ async function main() {
|
|
|
542
769
|
: indices
|
|
543
770
|
.map(
|
|
544
771
|
({ e, i }) =>
|
|
545
|
-
`${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}
|
|
772
|
+
`${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}` +
|
|
773
|
+
explainAbilities(e, result.expansions[i], result.units) + explainExamples(e),
|
|
546
774
|
)
|
|
547
775
|
.join("\n\n"),
|
|
548
776
|
);
|