lawspec 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +377 -1
- package/README.md +9 -3
- package/api.mjs +1 -0
- package/beam-doctor.mjs +2 -2
- package/bin/lawspec.mjs +93 -95
- package/build.json +497 -405
- package/component-packages.mjs +339 -0
- package/component.d.ts +46 -0
- package/component.mjs +208 -0
- package/core.wasm +2 -2
- package/core_jsffi.js +5 -5
- package/examples/native-payments/clojure/lawspec.edn +48 -0
- package/examples/native-payments/clojure/src/domain/payments.clj +1 -1
- package/examples/native-payments/clojure/test/domain/payment_generators.clj +1 -1
- package/examples/native-payments/csharp/lawspec.edn +48 -0
- package/examples/native-payments/elixir/lawspec.edn +123 -0
- package/examples/native-payments/erlang/lawspec.edn +128 -0
- package/examples/native-payments/fsharp/lawspec.edn +123 -0
- package/examples/native-payments/gleam/lawspec.edn +123 -0
- package/examples/native-payments/go/lawspec.edn +123 -0
- package/examples/native-payments/groovy/lawspec.edn +123 -0
- package/examples/native-payments/haskell/lawspec.edn +123 -0
- package/examples/native-payments/java/lawspec.edn +123 -0
- package/examples/native-payments/javascript/lawspec.edn +123 -0
- package/examples/native-payments/kotlin/lawspec.edn +123 -0
- package/examples/native-payments/prolog/lawspec.edn +48 -0
- package/examples/native-payments/python/lawspec.edn +123 -0
- package/examples/native-payments/rust/lawspec.edn +124 -0
- package/examples/native-payments/scala/lawspec.edn +123 -0
- package/examples/native-payments/swift/lawspec.edn +123 -0
- package/examples/native-payments/typescript/lawspec.edn +123 -0
- package/examples/packages/orders/lawspec.edn +25 -0
- package/examples/packages/shop-domain/lawspec.edn +13 -0
- package/examples/packages/shop-tax-1/lawspec.edn +9 -0
- package/examples/packages/shop-tax-2/lawspec.edn +9 -0
- package/examples/specs/handler_bindings.lawspec +2 -2
- package/files.mjs +2 -1
- package/go.mod +4 -0
- package/index.d.ts +135 -2
- package/input-files.mjs +47 -0
- package/launcher.mjs +33 -13
- package/lsp.mjs +10 -5
- package/native-evidence.mjs +50 -15
- package/native-examples.mjs +8 -7
- package/native-host.mjs +71 -0
- package/package.json +8 -3
- package/proof.d.ts +8 -2
- package/proof.mjs +183 -31
- package/scala-build-state.mjs +59 -0
- package/templates.mjs +242 -2
- package/test-command.mjs +192 -60
- package/builtin-data-names.mjs +0 -73
- package/examples/native-payments/clojure/lawspec.json +0 -62
- package/examples/native-payments/csharp/lawspec.json +0 -62
- package/examples/native-payments/elixir/lawspec.json +0 -160
- package/examples/native-payments/erlang/lawspec.json +0 -159
- package/examples/native-payments/fsharp/lawspec.json +0 -178
- package/examples/native-payments/gleam/lawspec.json +0 -160
- package/examples/native-payments/go/lawspec.json +0 -147
- package/examples/native-payments/groovy/lawspec.json +0 -176
- package/examples/native-payments/haskell/lawspec.json +0 -160
- package/examples/native-payments/java/lawspec.json +0 -176
- package/examples/native-payments/javascript/lawspec.json +0 -160
- package/examples/native-payments/kotlin/lawspec.json +0 -176
- package/examples/native-payments/prolog/lawspec.json +0 -20
- package/examples/native-payments/python/lawspec.json +0 -160
- package/examples/native-payments/rust/lawspec.json +0 -178
- package/examples/native-payments/scala/lawspec.json +0 -176
- package/examples/native-payments/swift/lawspec.json +0 -165
- package/examples/native-payments/typescript/lawspec.json +0 -160
- package/examples/packages/orders/lawspec.json +0 -27
- package/examples/packages/shop-domain/lawspec-package.json +0 -7
- package/examples/packages/shop-tax-1/lawspec-package.json +0 -5
- package/examples/packages/shop-tax-2/lawspec-package.json +0 -5
- package/explain.mjs +0 -114
- package/expression-presentation.mjs +0 -333
- package/positional-fields.mjs +0 -13
- package/typed-presentation.mjs +0 -165
package/test-command.mjs
CHANGED
|
@@ -4,19 +4,20 @@
|
|
|
4
4
|
//
|
|
5
5
|
// A law's result is keyed by everything it depends on: its plan key (the law
|
|
6
6
|
// and the Merkle digests of the types and definitions it reaches), the
|
|
7
|
-
// compiler build, the target's configuration,
|
|
8
|
-
//
|
|
7
|
+
// compiler build, the target's configuration, the project's build and lock
|
|
8
|
+
// files, the toolchain the doctor reports and, for
|
|
9
9
|
// a law that calls adapters, every file in the target project that LawSpec
|
|
10
10
|
// did not generate. A passing run records each law's key and seed; a law runs
|
|
11
11
|
// again when its key changes or with --fresh.
|
|
12
12
|
import { createHash, randomUUID } from "node:crypto";
|
|
13
13
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
14
14
|
import path from "node:path";
|
|
15
|
+
import {presentationChecks,templates} from "./templates.mjs";
|
|
16
|
+
|
|
17
|
+
import {buildDirectories as skipped,cryptoBuildOutput,isAttestation,fingerprintInputs} from './input-files.mjs';
|
|
18
|
+
export {isAttestation} from './input-files.mjs';
|
|
19
|
+
import {nativeHostIdentity} from './native-host.mjs';
|
|
15
20
|
|
|
16
|
-
// Folders that hold dependencies, build output or caches, not code under test.
|
|
17
|
-
const skipped = new Set([".git", ".lawspec", "node_modules", "dist", "build", "target", "out", ".gradle", "bin", "obj", "TestResults",
|
|
18
|
-
".stack-work", "_build", "deps", ".elixir_ls", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode",
|
|
19
|
-
".scala-build", ".bsp", ".bloop", ".metals", ".build", ".swiftpm"]);
|
|
20
21
|
// Build and lock files, which choose the toolchain and dependencies.
|
|
21
22
|
const environmentFiles = ["package.json", "package-lock.json", "tsconfig.json", "pyproject.toml", "go.mod",
|
|
22
23
|
"go.sum", "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts",
|
|
@@ -25,27 +26,11 @@ const environmentFiles = ["package.json", "package-lock.json", "tsconfig.json",
|
|
|
25
26
|
"global.json", "Directory.Build.props", "Directory.Packages.props", "NuGet.config", "project.scala", "Package.swift", "Package.resolved"];
|
|
26
27
|
|
|
27
28
|
const digest = (value) => createHash("sha256").update(value).digest("hex");
|
|
28
|
-
// The crypto builder owns these outputs. Inputs and other priv/ assets still
|
|
29
|
-
// contribute to the adapter digest; rebuilding the NIF cannot invalidate a pass.
|
|
30
|
-
const cryptoBuildOutput = (file) => /^priv\/lawspec_crypto_native\.(?:so|dll)(?:\.build|\.\d+\.tmp)?$/.test(file);
|
|
31
|
-
|
|
32
|
-
// An environment attestation records where a run's dependencies were wired
|
|
33
|
-
// and what was observed; it is reporting only, never an input of the
|
|
34
|
-
// project, so one written inside the project is not digested. It is known by
|
|
35
|
-
// its content, whatever its name or whichever tool wrote it.
|
|
36
|
-
// ref:DEC-completed-run-attestation
|
|
37
|
-
export function isAttestation(content) {
|
|
38
|
-
try {
|
|
39
|
-
const value = JSON.parse(content);
|
|
40
|
-
return value !== null && typeof value === "object" && value.format === "environment-attestation";
|
|
41
|
-
} catch {
|
|
42
|
-
return false;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
29
|
// Every file in a project that LawSpec did not generate, by content, except
|
|
47
30
|
// environment attestations.
|
|
48
|
-
export async function projectDigest(root, generated) {
|
|
31
|
+
export async function projectDigest(root, generated, language, {inputRoot=root,inputs=[]} = {}) {
|
|
32
|
+
const readable=language?templates(language):{};
|
|
33
|
+
const compact=language?templates(language,{minify:true}):{};
|
|
49
34
|
const hash = createHash("sha256");
|
|
50
35
|
async function walk(folder) {
|
|
51
36
|
const entries = (await readdir(path.join(root, folder), { withFileTypes: true }))
|
|
@@ -57,21 +42,38 @@ export async function projectDigest(root, generated) {
|
|
|
57
42
|
} else if (entry.isFile() && !generated.has(relative) && !cryptoBuildOutput(relative)) {
|
|
58
43
|
const content = await readFile(path.join(root, relative));
|
|
59
44
|
if (relative.endsWith(".json") && isAttestation(content.toString("utf8"))) continue;
|
|
60
|
-
hash.update(relative + "\0").update(content).update("\0");
|
|
45
|
+
hash.update(relative + "\0").update(content.toString("utf8")===compact[relative]?readable[relative]:content).update("\0");
|
|
61
46
|
}
|
|
62
47
|
}
|
|
63
48
|
}
|
|
64
49
|
await walk("");
|
|
50
|
+
if(inputs.length) {
|
|
51
|
+
const owned=new Set([...generated].map(file=>path.resolve(root,file)));
|
|
52
|
+
// Target files already contribute above, including exact readable/compact
|
|
53
|
+
// scaffold normalization. Declared parent directories must not hash their
|
|
54
|
+
// raw bytes a second time and make a layout-only edit stale execution.
|
|
55
|
+
const outside = file => {const relative=path.relative(path.resolve(root),file);return relative==='..'||relative.startsWith('..'+path.sep)||path.isAbsolute(relative);};
|
|
56
|
+
const external = (await fingerprintInputs(inputRoot,inputs,{generated:owned}))
|
|
57
|
+
.filter(([relative])=>outside(path.resolve(inputRoot,relative)));
|
|
58
|
+
hash.update("declared-inputs\0").update(JSON.stringify(external)).update("\0");
|
|
59
|
+
}
|
|
65
60
|
return hash.digest("hex");
|
|
66
61
|
}
|
|
67
62
|
|
|
68
|
-
export async function environmentDigest(root, report) {
|
|
63
|
+
export async function environmentDigest(root, report, language) {
|
|
69
64
|
// Coverage locations describe build output; native tools can discover more
|
|
70
65
|
// output directories after the first run. Build configuration is hashed below.
|
|
71
66
|
const {coverage: _coverage, ...toolchain} = report ?? {};
|
|
72
|
-
const parts = [JSON.stringify(report ? toolchain : null)];
|
|
67
|
+
const parts = [await nativeHostIdentity(), JSON.stringify(report ? toolchain : null)];
|
|
73
68
|
const names = (await readdir(root)).filter((name) => environmentFiles.includes(name) || name.endsWith(".cabal")).sort();
|
|
74
|
-
|
|
69
|
+
const readable=language?templates(language):{};
|
|
70
|
+
const compact=language?templates(language,{minify:true}):{};
|
|
71
|
+
for (const name of names) {
|
|
72
|
+
const content=await readFile(path.join(root,name),"utf8");
|
|
73
|
+
// Only byte-exact compiler-owned scaffold alternatives are equivalent.
|
|
74
|
+
// A user edit to build configuration always remains an execution input.
|
|
75
|
+
parts.push(name,content===compact[name]?readable[name]:content);
|
|
76
|
+
}
|
|
75
77
|
return digest(JSON.stringify(parts));
|
|
76
78
|
}
|
|
77
79
|
|
|
@@ -83,6 +85,7 @@ export async function recordedDigest(folder) {
|
|
|
83
85
|
const entries = await readdir(path.join(folder, relative), { withFileTypes: true }).catch(() => []);
|
|
84
86
|
for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
|
|
85
87
|
const next = relative ? `${relative}/${entry.name}` : entry.name;
|
|
88
|
+
if (entry.isSymbolicLink()) throw new Error(`Recording input is a symbolic link: ${path.join(folder,next)}`);
|
|
86
89
|
if (entry.isDirectory()) await walk(next);
|
|
87
90
|
else if (entry.isFile()) {
|
|
88
91
|
any = true;
|
|
@@ -94,20 +97,36 @@ export async function recordedDigest(folder) {
|
|
|
94
97
|
return any ? hash.digest("hex") : "";
|
|
95
98
|
}
|
|
96
99
|
|
|
97
|
-
// The result key of each law in a target's test manifest
|
|
100
|
+
// The result key of each law in a target's test manifest: what the law does
|
|
101
|
+
// (the manifest's semantic key), the compiler build, the target, toolchain
|
|
102
|
+
// and recorded values, and the adapters a law calls. Never the generated
|
|
103
|
+
// file's bytes: a doc edit or a reformat rewrites comments and source links
|
|
104
|
+
// but reruns nothing. ref:DEC-content-identity
|
|
98
105
|
// A law an environment attestation touches is keyed by how its dependencies
|
|
99
106
|
// were wired too, so a run against other endpoints, or the real dependency
|
|
100
107
|
// in place of a stand-in, runs it again; other laws' keys are unchanged.
|
|
101
|
-
export function lawKeys({ build, target, machineBits, minify, tests,
|
|
102
|
-
|
|
103
|
-
|
|
108
|
+
export function lawKeys({ build, target, machineBits, minify, tests, environment, project, recorded = "", attested = new Map() }) {
|
|
109
|
+
// The compiler resolves binding reachability per law. Missing flags remain
|
|
110
|
+
// conservative for an unknown producer; a checked false stays independent
|
|
111
|
+
// of unrelated native bindings in the same target.
|
|
112
|
+
const native = entry => entry.callsAdapters !== false;
|
|
113
|
+
const configuration = entry=>JSON.stringify({ ...target, root: undefined,
|
|
114
|
+
...(!native(entry)?{nativeBindings:undefined}:{}) });
|
|
104
115
|
return new Map(tests.map((entry) => [entry.law, digest(JSON.stringify([
|
|
105
|
-
build, configuration, machineBits,
|
|
106
|
-
environment, entry
|
|
116
|
+
build, configuration(entry), machineBits, entry.key,
|
|
117
|
+
environment, native(entry) ? project : null,
|
|
118
|
+
native(entry) ? recorded : null,
|
|
107
119
|
...(attested.has(entry.law) ? [attested.get(entry.law)] : []),
|
|
108
120
|
]))]));
|
|
109
121
|
}
|
|
110
122
|
|
|
123
|
+
// A law's last result: under its identity, or, when the law was renamed,
|
|
124
|
+
// under any earlier identity with the same key.
|
|
125
|
+
export function previousResult(previous, law, key) {
|
|
126
|
+
if (previous[law]?.key === key) return previous[law];
|
|
127
|
+
return Object.values(previous ?? {}).find((result) => result?.key === key);
|
|
128
|
+
}
|
|
129
|
+
|
|
111
130
|
const regex = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
112
131
|
const without = (file, extension) => file.slice(0, -extension.length);
|
|
113
132
|
const groupBy = (entries, key) => {
|
|
@@ -130,6 +149,27 @@ export function preparationInvocations(target, files) {
|
|
|
130
149
|
: [];
|
|
131
150
|
}
|
|
132
151
|
|
|
152
|
+
// A changed layout must be accepted by the target compiler/parser before a
|
|
153
|
+
// semantic execution result can be reused. No property cases run here.
|
|
154
|
+
export function presentationInvocations(target,files,{offline=false}={}) {
|
|
155
|
+
const overrides={python3:target.python,mvn:target.maven,gradle:target.gradle,
|
|
156
|
+
scala:target.scala,swift:target.swift,clojure:target.clojure};
|
|
157
|
+
return (presentationChecks[target.language]??[]).flatMap(check=>{
|
|
158
|
+
const paths=check.suffix?files.filter(file=>file.path.endsWith(check.suffix)).map(file=>file.path):[];
|
|
159
|
+
const extra=offline?({cargo:['--offline'],mvn:['-o'],gradle:['--offline'],dotnet:['--no-restore']}[check.command]??[]):[];
|
|
160
|
+
const args=[...check.args,...extra];
|
|
161
|
+
return (check.each?paths.map(file=>[file]):[paths]).map(paths=>({
|
|
162
|
+
command:overrides[check.command]??check.command,args:[...args,...paths],environment:check.environment}));
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Presentation bytes are deliberately kept out of lawKeys. This separate
|
|
167
|
+
// digest records which current artifacts have passed native source validation.
|
|
168
|
+
export function presentationDigest(files,environment) {
|
|
169
|
+
return digest(JSON.stringify([environment,files.filter(file=>file.ownership==='generated')
|
|
170
|
+
.map(file=>[file.path,file.content])]));
|
|
171
|
+
}
|
|
172
|
+
|
|
133
173
|
// The native invocations that run the harness's benchmarks (lawspec test
|
|
134
174
|
// --benchmarks): each target's test command, selecting the tests that
|
|
135
175
|
// measure them by the names the manifest gives (entry.name). Benchmarks are
|
|
@@ -156,7 +196,7 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
156
196
|
if (language === "gleam")
|
|
157
197
|
return [{ command: "gleam", args: ["test", "--target", "erlang"], env: {
|
|
158
198
|
LAWSPEC_BEAM_BENCHMARKS: JSON.stringify(entries.map((entry) => ({
|
|
159
|
-
module: path.posix.basename(entry.file, ".gleam"), function: `${entry.name}
|
|
199
|
+
module: path.posix.basename(entry.file, ".gleam"), function: `${entry.name}_test`
|
|
160
200
|
})))
|
|
161
201
|
} }];
|
|
162
202
|
if (language === "python")
|
|
@@ -173,7 +213,7 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
173
213
|
args: ["test", "-count=1", "-v", `./${directory}`, "-run", `^(${marks.map((e) => regex(e.name)).join("|")})$`] }));
|
|
174
214
|
if (language === "java") {
|
|
175
215
|
const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
|
|
176
|
-
return [{ command: target.maven || "mvn", args: [...(offline ? ["-o"] : []), "-B", "test",
|
|
216
|
+
return [{ command: target.maven || "mvn", env: { LAWSPEC_CASE: "" }, args: [...(offline ? ["-o"] : []), "-B", "test",
|
|
177
217
|
`-Dtest=${byClass.map(([name, marks]) => `${name}#${marks.map((e) => e.name).join("+")}`).join(",")}`] }];
|
|
178
218
|
}
|
|
179
219
|
if (language === "kotlin")
|
|
@@ -191,7 +231,7 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
191
231
|
}
|
|
192
232
|
// Spock names each feature in words; Gradle's filters select it by name.
|
|
193
233
|
if (language === "groovy")
|
|
194
|
-
return [{ command: target.gradle || "gradle",
|
|
234
|
+
return [{ command: target.gradle || "gradle", env: { LAWSPEC_CASE: "" },
|
|
195
235
|
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun",
|
|
196
236
|
...entries.flatMap((e) => ["--tests", `${className(e.file, "src/test/groovy", ".groovy")}.${e.name}`])] }];
|
|
197
237
|
if (language === "rust")
|
|
@@ -206,7 +246,11 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
206
246
|
// Swift Testing selects tests by a regular expression over their ids
|
|
207
247
|
// (Module.Suite/function()), one --filter per benchmark, OR'd.
|
|
208
248
|
if (language === "swift")
|
|
209
|
-
return [{ command: target.swift || "swift", args: ["test", ...entries.flatMap((e) => [
|
|
249
|
+
return [{ command: target.swift || "swift", args: ["test", ...entries.flatMap((e) => [
|
|
250
|
+
"--filter", `/${regex(e.name)}\\(`,
|
|
251
|
+
"--filter", `${regex(path.posix.basename(e.file, ".swift"))}/ordered\\(`])],
|
|
252
|
+
env: { LAWSPEC_SWIFT_LAWS: JSON.stringify(entries.map(({unit,name}) => ({unit,name}))),
|
|
253
|
+
LAWSPEC_SWIFT_REPORT: "", LAWSPEC_SWIFT_RUN: "" } }];
|
|
210
254
|
if (language === "haskell") {
|
|
211
255
|
const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
|
|
212
256
|
// hspec matches by substring, and a benchmark's name has spaces, so each
|
|
@@ -316,7 +360,7 @@ function nativeInvocations(target, entries, { offline = false, scratch = ".", co
|
|
|
316
360
|
});
|
|
317
361
|
if (language === "javascript" || language === "typescript") {
|
|
318
362
|
const pattern = (e) => model(e)?`^${regex(e.label)}$`:
|
|
319
|
-
`^${regex(e.label)}( example: .*| property| known discrepancy| skipped| replay| search)?$`;
|
|
363
|
+
`^${regex(e.label)}( example: .*| boundary \\d+| property| known discrepancy| skipped| replay| search)?$`;
|
|
320
364
|
const files = [...new Set(entries.map((e) => language === "typescript"
|
|
321
365
|
? `dist/${without(e.file, ".ts")}.js` : e.file))];
|
|
322
366
|
const file = path.join(scratch, "node-events.jsonl");
|
|
@@ -345,10 +389,10 @@ function nativeInvocations(target, entries, { offline = false, scratch = ".", co
|
|
|
345
389
|
`^(${[...new Set(laws.map(e=>regex(e.name)+(model(e)?"$":"_")))].join("|")})`:
|
|
346
390
|
`^(${laws.map((e) => regex(e.name)).join("|")})_`],
|
|
347
391
|
report: { kind: "go-json" },
|
|
348
|
-
ran: (test) => laws.filter((e) => named(e).test(test.name)) }));
|
|
392
|
+
ran: (test) => laws.filter((e) => named(e).test(test.name) && test.name !== `${e.name}_Boundaries`) }));
|
|
349
393
|
if (language === "java") {
|
|
350
394
|
const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
|
|
351
|
-
return [{ laws: entries, command: target.maven || "mvn",
|
|
395
|
+
return [{ laws: entries, command: target.maven || "mvn", env: { LAWSPEC_CASE: "" },
|
|
352
396
|
args: [...(offline ? ["-o"] : []), "-B", ...(coverage ? ["org.jacoco:jacoco-maven-plugin:prepare-agent"] : []), "test",
|
|
353
397
|
...(coverage ? ["org.jacoco:jacoco-maven-plugin:report"] : []),
|
|
354
398
|
`-Dtest=${byClass.map(([name, laws]) => `${name}#${[...new Set(laws.map((e) => model(e)?e.name:`${e.name}_*`))].join("+")}`).join(",")}`],
|
|
@@ -405,27 +449,30 @@ function nativeInvocations(target, entries, { offline = false, scratch = ".", co
|
|
|
405
449
|
// (LawSpec.TestNames), so "name: *" selects one law's features only.
|
|
406
450
|
if (language === "groovy") {
|
|
407
451
|
const name = (e) => className(e.file, "src/test/groovy", ".groovy");
|
|
408
|
-
return [{ laws: entries, command: target.gradle || "gradle",
|
|
452
|
+
return [{ laws: entries, command: target.gradle || "gradle", env: { LAWSPEC_CASE: "" },
|
|
409
453
|
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["jacocoTestReport"] : []), "--rerun",
|
|
410
454
|
...[...new Set(entries.map(e=>`${e.scope??name(e)}.${e.name}${model(e)?"":": *"}`))].flatMap(name=>["--tests",name])],
|
|
411
455
|
report: { kind: "junit", directory: "build/test-results/test" },
|
|
412
456
|
ran: (test) => entries.filter((e) => test.classname === (e.scope??name(e)) &&
|
|
413
457
|
(model(e)?test.name===e.name:test.name.startsWith(`${e.name}: `))) }];
|
|
414
458
|
}
|
|
459
|
+
// A law's tests are a module of its name (LawSpec.TestNames), selected
|
|
460
|
+
// by the module's path; a model's test is selected by its exact name.
|
|
415
461
|
if (language === "rust")
|
|
416
|
-
return groupBy(entries, (e) => e.file).
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
...
|
|
420
|
-
|
|
421
|
-
|
|
462
|
+
return groupBy(entries, (e) => e.file).flatMap(([file, entries]) =>
|
|
463
|
+
[entries.filter((e) => !model(e)), entries.filter(model)].filter((laws) => laws.length).map((laws) => ({
|
|
464
|
+
laws, command: "cargo",
|
|
465
|
+
args: [...(coverage ? ["llvm-cov", "--no-report"] : []), "test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--",
|
|
466
|
+
...(model(laws[0]) ? ["--exact", ...new Set(laws.map((e) => e.name))] : [...new Set(laws.map((e) => `${e.name}::`))])],
|
|
467
|
+
report: { kind: "lines", pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/ },
|
|
468
|
+
ran: (test) => laws.filter((e) => model(e) ? test.name === e.name : test.name.startsWith(`${e.name}::`)) })));
|
|
422
469
|
// xUnit v3 on Microsoft.Testing.Platform selects test methods with its
|
|
423
470
|
// query filter (assembly/namespace/class/method, one per law, OR'd), and
|
|
424
471
|
// reports each test by its display name: the law's label, then its kind.
|
|
425
472
|
if (language === "fsharp" || language === "csharp") {
|
|
426
473
|
const report = "lawspec-junit.xml";
|
|
427
474
|
const results = path.resolve(scratch, "dotnet-results");
|
|
428
|
-
const kinds = "example \\d+: .*|boundary
|
|
475
|
+
const kinds = "example \\d+: .*|boundary(?: \\d+(?: — .*)?(?:\\(caseIndex: \\d+\\))?)?|statistics|property|search|replay|skipped|known discrepancy";
|
|
429
476
|
const pattern = (e) => new RegExp(`^${regex(e.label)}( (${kinds}))?$`);
|
|
430
477
|
const measured = coverage ? path.resolve(coverage, "dotnet.cobertura.xml") : null;
|
|
431
478
|
return [{ laws: entries, command: "dotnet",
|
|
@@ -435,19 +482,31 @@ function nativeInvocations(target, entries, { offline = false, scratch = ".", co
|
|
|
435
482
|
report: { kind: "junit", files: [path.join(results, report)] },
|
|
436
483
|
ran: (test) => entries.filter((e) => pattern(e).test(test.name)) }];
|
|
437
484
|
}
|
|
438
|
-
// Swift
|
|
439
|
-
//
|
|
440
|
-
// given, with -swift-testing added. A law's tests are its unit's suite
|
|
441
|
-
// (named after the test file) and its name, then a kind (lawName_property).
|
|
485
|
+
// Ordered Swift groups expose their parameterized aggregate in JUnit;
|
|
486
|
+
// the native lifecycle report identifies each completed invocation inside it.
|
|
442
487
|
if (language === "swift") {
|
|
443
488
|
const results = path.resolve(scratch);
|
|
444
489
|
const suite = (e) => path.posix.basename(e.file, ".swift");
|
|
445
|
-
const
|
|
490
|
+
const ordered = entries.filter((e) => e.scope === "ordered");
|
|
491
|
+
const token = randomUUID();
|
|
492
|
+
const output = ordered.length ? `swift-lawspec-${token}` : "swift-lawspec";
|
|
493
|
+
const files = [path.join(results, `${output}-swift-testing.xml`)];
|
|
494
|
+
const events = path.join(results, `${output}.jsonl`);
|
|
495
|
+
const test = (e) => e.scope === "ordered" ? `${regex(suite(e))}/ordered\\(` :
|
|
496
|
+
`${regex(suite(e))}/${regex(e.name)}(_[A-Za-z0-9]+)?\\(`;
|
|
446
497
|
return [{ laws: entries, command: target.swift || "swift",
|
|
447
|
-
args: ["test", ...(coverage ? ["--enable-code-coverage"] : []),
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
498
|
+
args: ["test", ...(coverage ? ["--enable-code-coverage"] : []),
|
|
499
|
+
...[...new Set(entries.map(test))].flatMap((selector) => ["--filter", selector]),
|
|
500
|
+
"--xunit-output", path.join(results, `${output}.xml`)],
|
|
501
|
+
env: { LAWSPEC_SWIFT_LAWS: JSON.stringify(ordered.map(({unit,name}) => ({unit,name}))),
|
|
502
|
+
LAWSPEC_SWIFT_REPORT: ordered.length ? events : "", LAWSPEC_SWIFT_RUN: ordered.length ? token : "" },
|
|
503
|
+
report: ordered.length ? { kind: "swift-events", files, file: events, run: token,
|
|
504
|
+
groups: ordered.map((e) => ({unit:e.unit,law:e.name,suite:suite(e),
|
|
505
|
+
minimum:typeof e.knownDiscrepancy === "string" ? 1 : Math.max(1,e.coverage?.cases ?? 1)})) } :
|
|
506
|
+
{ kind: "junit", files },
|
|
507
|
+
ran: (result) => entries.filter((e) => e.scope === "ordered" ?
|
|
508
|
+
result.unit === e.unit && result.nativeName === e.name :
|
|
509
|
+
result.classname?.split(".").pop() === suite(e) &&
|
|
451
510
|
new RegExp(`^${regex(e.name)}(_[A-Za-z0-9]+)?(\\(|$)`).test(result.name)) }];
|
|
452
511
|
}
|
|
453
512
|
if (language === "haskell") {
|
|
@@ -494,7 +553,7 @@ export function selectByTags(entries, include = [], exclude = []) {
|
|
|
494
553
|
// including when an adapter is repaired by restoring its previous contents.
|
|
495
554
|
export function testsToRun(entries, keys, previous, failures, {fresh = false, coverage = false, updateRecorded = false} = {}) {
|
|
496
555
|
return entries.filter((entry) => fresh || coverage || updateRecorded || failures[entry.law] ||
|
|
497
|
-
previous
|
|
556
|
+
!previousResult(previous, entry.law, keys.get(entry.law)));
|
|
498
557
|
}
|
|
499
558
|
|
|
500
559
|
// Read complete outer elements, preserving nested suites and native payloads.
|
|
@@ -575,6 +634,7 @@ export async function harnessStatistics(directory) {
|
|
|
575
634
|
// The tests a run executed, as {name, classname}, from the runner's report or
|
|
576
635
|
// output. Skipped tests are not counted.
|
|
577
636
|
export async function executedTests(report, output, root, since) {
|
|
637
|
+
if (report?.kind === "swift-events") return swiftExecutedTests(report, output, root, since);
|
|
578
638
|
if(report?.kind==='node-events') {
|
|
579
639
|
const raw=await readFile(path.resolve(root,report.file),'utf8').catch(()=>'');
|
|
580
640
|
let rows;
|
|
@@ -662,6 +722,78 @@ export async function executedTests(report, output, root, since) {
|
|
|
662
722
|
return [];
|
|
663
723
|
}
|
|
664
724
|
|
|
725
|
+
// Swift's XML seals the native aggregate; runtime events refine that completed
|
|
726
|
+
// aggregate into individually attributable cases. Neither channel suffices alone.
|
|
727
|
+
async function swiftExecutedTests(report, output, root, since) {
|
|
728
|
+
const native = await executedTests({kind:"junit",files:report.files}, output, root, since);
|
|
729
|
+
const ordinary = native.filter((test) => test.name !== "ordered(_:)");
|
|
730
|
+
const raw = await readFile(path.resolve(root,report.file), "utf8").catch(() => "");
|
|
731
|
+
let rows;
|
|
732
|
+
try { rows = raw.trim().split("\n").map((line) => JSON.parse(line)); } catch { return ordinary; }
|
|
733
|
+
if (!rows.length || rows.some((row) => !row || row.version !== 1 || row.run !== report.run)) return ordinary;
|
|
734
|
+
const key = (unit,name) => JSON.stringify([unit,name]);
|
|
735
|
+
const expected = new Map(report.groups.map((group) => [key(group.unit,group.law),group]));
|
|
736
|
+
const plans = new Map(), cases = new Map(), starts = new Set(), ends = new Map();
|
|
737
|
+
for (const row of rows) {
|
|
738
|
+
if (typeof row.unit !== "string") return ordinary;
|
|
739
|
+
if (row.event === "plan") {
|
|
740
|
+
if (!Array.isArray(row.cases) || !row.cases.length) return ordinary;
|
|
741
|
+
const previous = plans.get(row.unit);
|
|
742
|
+
if (previous !== undefined) {
|
|
743
|
+
if (JSON.stringify(previous) !== JSON.stringify(row.cases)) return ordinary;
|
|
744
|
+
continue;
|
|
745
|
+
}
|
|
746
|
+
plans.set(row.unit,row.cases);
|
|
747
|
+
for (const item of row.cases) {
|
|
748
|
+
if (!item || typeof item.id !== "string" || !item.id || typeof item.name !== "string" || !item.name ||
|
|
749
|
+
typeof item.law !== "string" || !expected.has(key(row.unit,item.law))) return ordinary;
|
|
750
|
+
const id = key(row.unit,item.id);
|
|
751
|
+
if (cases.has(id)) return ordinary;
|
|
752
|
+
cases.set(id,{...item,unit:row.unit});
|
|
753
|
+
}
|
|
754
|
+
} else if (row.event === "case-start" || row.event === "case-end") {
|
|
755
|
+
const id = key(row.unit,row.id), item = cases.get(id);
|
|
756
|
+
if (!item || item.name !== row.name || item.law !== row.law) return ordinary;
|
|
757
|
+
if (row.event === "case-start") {
|
|
758
|
+
if (starts.has(id)) return ordinary;
|
|
759
|
+
starts.add(id);
|
|
760
|
+
} else {
|
|
761
|
+
if (!starts.has(id) || ends.has(id) || !["passed","failed"].includes(row.status) ||
|
|
762
|
+
!Number.isFinite(row.time) || row.time < 0 ||
|
|
763
|
+
(row.failure !== undefined && typeof row.failure !== "string")) return ordinary;
|
|
764
|
+
ends.set(id,row);
|
|
765
|
+
}
|
|
766
|
+
} else return ordinary;
|
|
767
|
+
}
|
|
768
|
+
if (cases.size !== starts.size || cases.size !== ends.size) return ordinary;
|
|
769
|
+
const completed = [];
|
|
770
|
+
for (const group of expected.values()) {
|
|
771
|
+
const members = [...cases.values()].filter((item) => item.unit === group.unit && item.law === group.law);
|
|
772
|
+
if (members.length < group.minimum) return ordinary;
|
|
773
|
+
const aggregates = native.filter((test) => test.name === "ordered(_:)" &&
|
|
774
|
+
test.classname.split(".").pop() === group.suite);
|
|
775
|
+
if (aggregates.length !== 1 || !["passed","failed"].includes(aggregates[0].status)) return ordinary;
|
|
776
|
+
const aggregate = aggregates[0];
|
|
777
|
+
// A framework failure outside the invocation body cannot credit that body's
|
|
778
|
+
// earlier successes. Preserve them as incomplete observations instead.
|
|
779
|
+
const suiteFailed = [...cases.values()].some((item) => {
|
|
780
|
+
const expectedGroup = expected.get(key(item.unit,item.law));
|
|
781
|
+
return expectedGroup.suite === group.suite && ends.get(key(item.unit,item.id)).status === "failed";
|
|
782
|
+
});
|
|
783
|
+
if (aggregate.status === "passed" && suiteFailed) return ordinary;
|
|
784
|
+
for (const item of members) {
|
|
785
|
+
const result = ends.get(key(item.unit,item.id));
|
|
786
|
+
const incomplete = aggregate.status === "failed" && !suiteFailed;
|
|
787
|
+
completed.push({name:item.id,classname:aggregate.classname,identity:`${item.unit}::${item.law}`,
|
|
788
|
+
unit:item.unit,nativeName:item.law,
|
|
789
|
+
status:incomplete ? "incomplete" : result.status,time:result.time,
|
|
790
|
+
...(incomplete ? {failure:aggregate.failure ?? "Swift aggregate failed outside its invocation bodies"} :
|
|
791
|
+
result.failure === undefined ? {} : {failure:result.failure})});
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
return [...ordinary,...completed];
|
|
795
|
+
}
|
|
796
|
+
|
|
665
797
|
// JUnit XML for a runner that writes none, from the tests it printed.
|
|
666
798
|
export function junitFromTests(tests) {
|
|
667
799
|
const failures = tests.filter((t) => t.status === "failed").length;
|
package/builtin-data-names.mjs
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
// Generated from templates/npm/builtin-data-names.mjs by lawspec-dev generate. Do not edit.
|
|
2
|
-
// Names and field order projected from the compiler's primitive data registry.
|
|
3
|
-
// This is presentation metadata, not another type checker or public wire view.
|
|
4
|
-
export const builtinDataNames =
|
|
5
|
-
[
|
|
6
|
-
{
|
|
7
|
-
"id": "List",
|
|
8
|
-
"name": "List",
|
|
9
|
-
"constructors": [
|
|
10
|
-
{
|
|
11
|
-
"id": "List::Nil",
|
|
12
|
-
"name": "Nil",
|
|
13
|
-
"fields": []
|
|
14
|
-
},
|
|
15
|
-
{
|
|
16
|
-
"id": "List::Cons",
|
|
17
|
-
"name": "Cons",
|
|
18
|
-
"fields": [
|
|
19
|
-
{
|
|
20
|
-
"name": "head"
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
"name": "tail"
|
|
24
|
-
}
|
|
25
|
-
]
|
|
26
|
-
}
|
|
27
|
-
]
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
"id": "Maybe",
|
|
31
|
-
"name": "Maybe",
|
|
32
|
-
"constructors": [
|
|
33
|
-
{
|
|
34
|
-
"id": "Maybe::Absent",
|
|
35
|
-
"name": "Absent",
|
|
36
|
-
"fields": []
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"id": "Maybe::Present",
|
|
40
|
-
"name": "Present",
|
|
41
|
-
"fields": [
|
|
42
|
-
{
|
|
43
|
-
"name": "value"
|
|
44
|
-
}
|
|
45
|
-
]
|
|
46
|
-
}
|
|
47
|
-
]
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"id": "Either",
|
|
51
|
-
"name": "Either",
|
|
52
|
-
"constructors": [
|
|
53
|
-
{
|
|
54
|
-
"id": "Either::Left",
|
|
55
|
-
"name": "Left",
|
|
56
|
-
"fields": [
|
|
57
|
-
{
|
|
58
|
-
"name": "value"
|
|
59
|
-
}
|
|
60
|
-
]
|
|
61
|
-
},
|
|
62
|
-
{
|
|
63
|
-
"id": "Either::Right",
|
|
64
|
-
"name": "Right",
|
|
65
|
-
"fields": [
|
|
66
|
-
{
|
|
67
|
-
"name": "value"
|
|
68
|
-
}
|
|
69
|
-
]
|
|
70
|
-
}
|
|
71
|
-
]
|
|
72
|
-
}
|
|
73
|
-
];
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 2,
|
|
3
|
-
"sources": [
|
|
4
|
-
"laws"
|
|
5
|
-
],
|
|
6
|
-
"machineBits": 64,
|
|
7
|
-
"targets": [
|
|
8
|
-
{
|
|
9
|
-
"language": "clojure",
|
|
10
|
-
"root": ".",
|
|
11
|
-
"nativeBindings": {
|
|
12
|
-
"functions": [
|
|
13
|
-
{
|
|
14
|
-
"declaration": "example.payments::addFee",
|
|
15
|
-
"native": [
|
|
16
|
-
"domain",
|
|
17
|
-
"payments",
|
|
18
|
-
"apply_fee"
|
|
19
|
-
]
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
"declaration": "example.payments::roundTrip",
|
|
23
|
-
"native": [
|
|
24
|
-
"domain",
|
|
25
|
-
"payments",
|
|
26
|
-
"restore"
|
|
27
|
-
]
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
"declaration": "example.payments::archive",
|
|
31
|
-
"native": [
|
|
32
|
-
"domain",
|
|
33
|
-
"payments",
|
|
34
|
-
"store"
|
|
35
|
-
]
|
|
36
|
-
}
|
|
37
|
-
],
|
|
38
|
-
"generators": [
|
|
39
|
-
{
|
|
40
|
-
"type": "example.payments::type::Money",
|
|
41
|
-
"factory": [
|
|
42
|
-
"domain",
|
|
43
|
-
"payment_generators",
|
|
44
|
-
"prices"
|
|
45
|
-
]
|
|
46
|
-
}
|
|
47
|
-
]
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
],
|
|
51
|
-
"languageProfile": "1.0-rc.2",
|
|
52
|
-
"verification": {
|
|
53
|
-
"layers": [
|
|
54
|
-
"proof",
|
|
55
|
-
"exhaustive",
|
|
56
|
-
"property",
|
|
57
|
-
"examples",
|
|
58
|
-
"runtime",
|
|
59
|
-
"model"
|
|
60
|
-
]
|
|
61
|
-
}
|
|
62
|
-
}
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 2,
|
|
3
|
-
"sources": [
|
|
4
|
-
"laws"
|
|
5
|
-
],
|
|
6
|
-
"machineBits": 64,
|
|
7
|
-
"targets": [
|
|
8
|
-
{
|
|
9
|
-
"language": "csharp",
|
|
10
|
-
"root": ".",
|
|
11
|
-
"nativeBindings": {
|
|
12
|
-
"functions": [
|
|
13
|
-
{
|
|
14
|
-
"declaration": "example.payments::addFee",
|
|
15
|
-
"native": [
|
|
16
|
-
"Domain",
|
|
17
|
-
"Payments",
|
|
18
|
-
"ApplyFee"
|
|
19
|
-
]
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
"declaration": "example.payments::roundTrip",
|
|
23
|
-
"native": [
|
|
24
|
-
"Domain",
|
|
25
|
-
"Payments",
|
|
26
|
-
"Restore"
|
|
27
|
-
]
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
"declaration": "example.payments::archive",
|
|
31
|
-
"native": [
|
|
32
|
-
"Domain",
|
|
33
|
-
"Payments",
|
|
34
|
-
"Store"
|
|
35
|
-
]
|
|
36
|
-
}
|
|
37
|
-
],
|
|
38
|
-
"generators": [
|
|
39
|
-
{
|
|
40
|
-
"type": "example.payments::type::Money",
|
|
41
|
-
"factory": [
|
|
42
|
-
"Domain",
|
|
43
|
-
"PaymentGenerators",
|
|
44
|
-
"Prices"
|
|
45
|
-
]
|
|
46
|
-
}
|
|
47
|
-
]
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
],
|
|
51
|
-
"languageProfile": "1.0-rc.2",
|
|
52
|
-
"verification": {
|
|
53
|
-
"layers": [
|
|
54
|
-
"proof",
|
|
55
|
-
"exhaustive",
|
|
56
|
-
"property",
|
|
57
|
-
"examples",
|
|
58
|
-
"runtime",
|
|
59
|
-
"model"
|
|
60
|
-
]
|
|
61
|
-
}
|
|
62
|
-
}
|