lawspec 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +71 -31
- package/build.json +232 -38
- package/compatibility.json +17 -0
- package/core.wasm +2 -2
- package/doctor.mjs +7 -1
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +10 -1
- package/native-examples.mjs +7 -2
- package/package.json +1 -1
- package/templates.mjs +92 -3
- package/test-command.mjs +161 -15
package/index.d.ts
CHANGED
|
@@ -7,7 +7,10 @@ export type Target =
|
|
|
7
7
|
'go' |
|
|
8
8
|
'haskell' |
|
|
9
9
|
'kotlin' |
|
|
10
|
-
'rust'
|
|
10
|
+
'rust' |
|
|
11
|
+
'erlang' |
|
|
12
|
+
'elixir' |
|
|
13
|
+
'gleam';
|
|
11
14
|
|
|
12
15
|
export interface Source {
|
|
13
16
|
path: string;
|
|
@@ -207,12 +210,18 @@ export interface NativeGoImport {
|
|
|
207
210
|
path: string;
|
|
208
211
|
}
|
|
209
212
|
|
|
213
|
+
export interface NativeErlangInclude {
|
|
214
|
+
path: string;
|
|
215
|
+
library?: boolean;
|
|
216
|
+
}
|
|
217
|
+
|
|
210
218
|
export interface NativeBindings {
|
|
211
219
|
types?: NativeTypeBinding[];
|
|
212
220
|
generators?: NativeGeneratorBinding[];
|
|
213
221
|
functions?: NativeFunctionBinding[];
|
|
214
222
|
rustCrate?: string;
|
|
215
223
|
goImports?: NativeGoImport[];
|
|
224
|
+
erlangIncludes?: NativeErlangInclude[];
|
|
216
225
|
}
|
|
217
226
|
|
|
218
227
|
export type DischargeStatus =
|
package/native-examples.mjs
CHANGED
|
@@ -33,6 +33,9 @@ export async function exportNativePayments(options, selected) {
|
|
|
33
33
|
const config = JSON.parse(files['lawspec.json']);
|
|
34
34
|
config.machineBits = options.machineBits ?? 64;
|
|
35
35
|
files['lawspec.json'] = JSON.stringify(config, null, options.minify ? undefined : 2) + '\n';
|
|
36
|
+
const instructions = target === 'gleam' ? setup[target].replace(
|
|
37
|
+
'Prepare dependencies with gleam build, then run gleam test.',
|
|
38
|
+
'Prepare dependencies with node prepare.mjs before generation, then run gleam test.') : setup[target];
|
|
36
39
|
files['README.md'] = `# Native payments: ${target}
|
|
37
40
|
|
|
38
41
|
This project checks application-owned payment types against the shared LawSpec
|
|
@@ -45,11 +48,13 @@ Install LawSpec and the native dependencies described below, then run:
|
|
|
45
48
|
|
|
46
49
|
\`\`\`sh
|
|
47
50
|
lawspec check
|
|
48
|
-
lawspec generate
|
|
51
|
+
${target === 'gleam' ? 'node prepare.mjs\n' : ''}lawspec generate
|
|
49
52
|
${commands[target]}
|
|
50
53
|
\`\`\`
|
|
51
54
|
|
|
52
|
-
${
|
|
55
|
+
${instructions}
|
|
56
|
+
|
|
57
|
+
${target === 'gleam' ? 'The preparation command compiles dependencies in a temporary project using\nyour dependency requirements and lock file. It leaves application sources in\nplace, so their imports can be resolved after LawSpec generates the runtime.\n' : ''}
|
|
53
58
|
|
|
54
59
|
JavaScript and TypeScript projects require \`npm install\` before testing.
|
|
55
60
|
The specification includes exact decimal arithmetic and explicit USD/GBP examples.
|
package/package.json
CHANGED
package/templates.mjs
CHANGED
|
@@ -9,8 +9,57 @@ export const targets = [
|
|
|
9
9
|
"go",
|
|
10
10
|
"haskell",
|
|
11
11
|
"kotlin",
|
|
12
|
-
"rust"
|
|
12
|
+
"rust",
|
|
13
|
+
"erlang",
|
|
14
|
+
"elixir",
|
|
15
|
+
"gleam"
|
|
13
16
|
];
|
|
17
|
+
export const targetLabels = [
|
|
18
|
+
[
|
|
19
|
+
"java",
|
|
20
|
+
"Java"
|
|
21
|
+
],
|
|
22
|
+
[
|
|
23
|
+
"python",
|
|
24
|
+
"Python"
|
|
25
|
+
],
|
|
26
|
+
[
|
|
27
|
+
"javascript",
|
|
28
|
+
"JavaScript"
|
|
29
|
+
],
|
|
30
|
+
[
|
|
31
|
+
"typescript",
|
|
32
|
+
"TypeScript"
|
|
33
|
+
],
|
|
34
|
+
[
|
|
35
|
+
"go",
|
|
36
|
+
"Go"
|
|
37
|
+
],
|
|
38
|
+
[
|
|
39
|
+
"haskell",
|
|
40
|
+
"Haskell"
|
|
41
|
+
],
|
|
42
|
+
[
|
|
43
|
+
"kotlin",
|
|
44
|
+
"Kotlin"
|
|
45
|
+
],
|
|
46
|
+
[
|
|
47
|
+
"rust",
|
|
48
|
+
"Rust"
|
|
49
|
+
],
|
|
50
|
+
[
|
|
51
|
+
"erlang",
|
|
52
|
+
"Erlang"
|
|
53
|
+
],
|
|
54
|
+
[
|
|
55
|
+
"elixir",
|
|
56
|
+
"Elixir"
|
|
57
|
+
],
|
|
58
|
+
[
|
|
59
|
+
"gleam",
|
|
60
|
+
"Gleam"
|
|
61
|
+
]
|
|
62
|
+
];
|
|
14
63
|
|
|
15
64
|
export const commands = {
|
|
16
65
|
"java": "mvn test",
|
|
@@ -20,7 +69,10 @@ export const commands = {
|
|
|
20
69
|
"go": "go test ./...",
|
|
21
70
|
"haskell": "stack test",
|
|
22
71
|
"kotlin": "gradle test",
|
|
23
|
-
"rust": "cargo test"
|
|
72
|
+
"rust": "cargo test",
|
|
73
|
+
"erlang": "rebar3 eunit",
|
|
74
|
+
"elixir": "mix test",
|
|
75
|
+
"gleam": "gleam test"
|
|
24
76
|
};
|
|
25
77
|
|
|
26
78
|
export const setup = {
|
|
@@ -31,7 +83,10 @@ export const setup = {
|
|
|
31
83
|
"go": "Use Go 1.25+ and go get pgregory.net/rapid@v1.2.0 (and github.com/cloudflare/circl@v1.6.5 for a program that imports lawspec.crypto or lawspec.network), then go mod download.",
|
|
32
84
|
"haskell": "Use Stack with lts-24.58, directory and time as dependencies (and, for a program that imports lawspec.crypto, lawspec.network or lawspec.randomness, extra-deps crypton-1.1.5, ram-0.22.1, mlkem-0.2.3.0 and mldsa-0.1.1.0 as dependencies too), and test dependencies hspec, hedgehog, hspec-hedgehog, hspec-discover, and a test/Spec.hs using hspec-discover. Run stack build --test --no-run-tests.",
|
|
33
85
|
"kotlin": "Use JDK 25, Gradle 9.3.0, Kotlin plugin 2.3.21, JVM target 25, Kotest 5.9.1 (runner, assertions, property), and useJUnitPlatform(); a program that imports lawspec.crypto or lawspec.network also needs Bouncy Castle bcprov-jdk18on 1.86. Run gradle testClasses.",
|
|
34
|
-
"rust": "Use Rust 1.85+ with edition 2024, Proptest 1.11.0, num-bigint 0.4.8, num-rational 0.4.2, num-complex 0.4.6 and num-traits 0.2.19; a program that imports lawspec.crypto, lawspec.network or lawspec.randomness also needs sha3 0.12.0, shake 0.1.0, ml-kem 0.3.2, ml-dsa 0.1.1, slh-dsa 0.2.0-rc.5, aes-gcm 0.11.1 and getrandom 0.4.3. Run cargo test."
|
|
86
|
+
"rust": "Use Rust 1.85+ with edition 2024, Proptest 1.11.0, num-bigint 0.4.8, num-rational 0.4.2, num-complex 0.4.6 and num-traits 0.2.19; a program that imports lawspec.crypto, lawspec.network or lawspec.randomness also needs sha3 0.12.0, shake 0.1.0, ml-kem 0.3.2, ml-dsa 0.1.1, slh-dsa 0.2.0-rc.5, aes-gcm 0.11.1 and getrandom 0.4.3. Run cargo test.",
|
|
87
|
+
"erlang": "Use Erlang/OTP 29, Rebar3 3.27.1 and PropEr 1.5.0 as a Hex dependency in the test profile. Configure {eunit_opts, [{print_depth, 100}, {report, {lawspec_beam_report, []}}]}. Run rebar3 as test compile to prepare dependencies, then rebar3 eunit. Custom test directories must be in extra_src_dirs. Programs importing lawspec.crypto or lawspec.network need OTP's post-quantum crypto support and a Unix C toolchain (WSL on Windows). Install OpenSSL 3.5+ development headers with the same major version as OTP, then run escript lawspec_crypto_build.escript before compiling. The builder uses pkg-config or Homebrew; LAWSPEC_OPENSSL_PREFIX overrides the installation. Ship the compiled priv directory with the application.",
|
|
88
|
+
"elixir": "Use Erlang/OTP 29, Elixir 1.20.x and StreamData 1.4.0 as a direct Hex test dependency. Run mix deps.get and MIX_ENV=test mix deps.compile. In MIX_ENV=test include src and test/support in erlc_paths, and lib and test/support in elixirc_paths. Add LawSpec.Beam.ExUnitFormatter to ExUnit's formatters in test/test_helper.exs and use unfiltered discovery. Run mix test. Programs importing lawspec.crypto or lawspec.network need OTP's post-quantum crypto support and a Unix C toolchain (WSL on Windows). Install OpenSSL 3.5+ development headers with the same major version as OTP, then run escript lawspec_crypto_build.escript before compiling. The builder uses pkg-config or Homebrew; LAWSPEC_OPENSSL_PREFIX overrides the installation. Ship the compiled priv directory with the application.",
|
|
89
|
+
"gleam": "Use Erlang/OTP 29, Gleam 1.18 or 1.19, gleam_stdlib 1.0.5, gleeunit 1.11.0 and qcheck 1.0.5. Select target=erlang, src/test directories, and the lawspec_test_support development dependency at ./test-support. In test/<package>_test.gleam use @external(erlang, \"lawspec_beam_test_run\", \"gleam_main\") pub fn main() -> Nil. Prepare dependencies with gleam build, then run gleam test. Programs importing lawspec.crypto or lawspec.network need OTP's post-quantum crypto support and a Unix C toolchain (WSL on Windows). Install OpenSSL 3.5+ development headers with the same major version as OTP, then run escript lawspec_crypto_build.escript before compiling. The builder uses pkg-config or Homebrew; LAWSPEC_OPENSSL_PREFIX overrides the installation. Ship the compiled priv directory with the application."
|
|
35
90
|
};
|
|
36
91
|
|
|
37
92
|
// Each target's build files in the readable and the compact (minified) layout.
|
|
@@ -109,6 +164,40 @@ const scaffolds = {
|
|
|
109
164
|
"src/lib.rs": "// Application library. LawSpec maintains the included module declarations.\ninclude!(\"lawspec_modules.rs\");\n",
|
|
110
165
|
"Cargo.toml": "[package]\nname = \"lawspec-example\"\nversion = \"0.1.0\"\nedition = \"2024\"\nrust-version = \"1.85\"\npublish = false\n\n[dependencies]\nnum-bigint = \"=0.4.8\"\nnum-rational = \"=0.4.2\"\nnum-complex = \"=0.4.6\"\nnum-traits = \"=0.2.19\"\n\n[dev-dependencies]\nproptest = \"=1.11.0\"\n"
|
|
111
166
|
}
|
|
167
|
+
},
|
|
168
|
+
"erlang": {
|
|
169
|
+
"readable": {
|
|
170
|
+
"rebar.config": "{erl_opts, [debug_info]}.\n{eunit_opts, [{print_depth, 100}, {report, {lawspec_beam_report, []}}]}.\n{profiles, [{test, [{deps, [{proper, \"1.5.0\"}]}]}]}.\n",
|
|
171
|
+
"src/lawspec_example.app.src": "{application, lawspec_example, [\n {description, \"LawSpec example\"},\n {vsn, \"0.1.0\"},\n {modules, []},\n {registered, []},\n {applications, [kernel, stdlib]}\n]}.\n"
|
|
172
|
+
},
|
|
173
|
+
"compact": {
|
|
174
|
+
"rebar.config": "{erl_opts, [debug_info]}.\n{eunit_opts, [{print_depth, 100}, {report, {lawspec_beam_report, []}}]}.\n{profiles, [{test, [{deps, [{proper, \"1.5.0\"}]}]}]}.\n",
|
|
175
|
+
"src/lawspec_example.app.src": "{application, lawspec_example, [\n {description, \"LawSpec example\"},\n {vsn, \"0.1.0\"},\n {modules, []},\n {registered, []},\n {applications, [kernel, stdlib]}\n]}.\n"
|
|
176
|
+
}
|
|
177
|
+
},
|
|
178
|
+
"elixir": {
|
|
179
|
+
"readable": {
|
|
180
|
+
"mix.exs": "defmodule LawSpecExample.MixProject do\n use Mix.Project\n\n def project do\n [app: :lawspec_example, version: \"0.1.0\", elixir: \"~> 1.20\",\n erlc_paths: [\"src\"] ++ test_paths(), elixirc_paths: [\"lib\"] ++ test_paths(),\n deps: [{:stream_data, \"== 1.4.0\", only: :test}]]\n end\n\n defp test_paths, do: if(Mix.env() == :test, do: [\"test/support\"], else: [])\n\n def application do\n [extra_applications: [:logger]]\n end\nend\n",
|
|
181
|
+
"test/test_helper.exs": "case System.get_env(\"LAWSPEC_SEED\") do\n nil -> ExUnit.start()\n seed -> ExUnit.start(seed: String.to_integer(seed))\nend\nif Code.ensure_loaded?(LawSpec.Beam.ExUnitFormatter) do\n ExUnit.configure(formatters: Enum.uniq(ExUnit.configuration()[:formatters] ++ [LawSpec.Beam.ExUnitFormatter]))\nend\nif Code.ensure_loaded?(:lawspec_beam_resources) do\n :lawspec_beam_resources.configure_suite(Process.whereis(ExUnit.Server))\n ExUnit.after_suite(fn _ -> :lawspec_beam_resources.stop_suite() end)\nend\n"
|
|
182
|
+
},
|
|
183
|
+
"compact": {
|
|
184
|
+
"mix.exs": "defmodule LawSpecExample.MixProject do\n use Mix.Project\n\n def project do\n [app: :lawspec_example, version: \"0.1.0\", elixir: \"~> 1.20\",\n erlc_paths: [\"src\"] ++ test_paths(), elixirc_paths: [\"lib\"] ++ test_paths(),\n deps: [{:stream_data, \"== 1.4.0\", only: :test}]]\n end\n\n defp test_paths, do: if(Mix.env() == :test, do: [\"test/support\"], else: [])\n\n def application do\n [extra_applications: [:logger]]\n end\nend\n",
|
|
185
|
+
"test/test_helper.exs": "case System.get_env(\"LAWSPEC_SEED\") do\n nil -> ExUnit.start()\n seed -> ExUnit.start(seed: String.to_integer(seed))\nend\nif Code.ensure_loaded?(LawSpec.Beam.ExUnitFormatter) do\n ExUnit.configure(formatters: Enum.uniq(ExUnit.configuration()[:formatters] ++ [LawSpec.Beam.ExUnitFormatter]))\nend\nif Code.ensure_loaded?(:lawspec_beam_resources) do\n :lawspec_beam_resources.configure_suite(Process.whereis(ExUnit.Server))\n ExUnit.after_suite(fn _ -> :lawspec_beam_resources.stop_suite() end)\nend\n"
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
"gleam": {
|
|
189
|
+
"readable": {
|
|
190
|
+
"gleam.toml": "name = \"lawspec_example\"\nversion = \"0.1.0\"\ngleam = \">= 1.18.0\"\ntarget = \"erlang\"\n\n[dependencies]\ngleam_stdlib = \"== 1.0.5\"\n\n[dev-dependencies]\ngleeunit = \"== 1.11.0\"\nqcheck = \"== 1.0.5\"\nlawspec_test_support = { path = \"./test-support\" }\n",
|
|
191
|
+
"test-support/gleam.toml": "# Generated test helpers are a development-only dependency.\nname = \"lawspec_test_support\"\nversion = \"0.1.0\"\ntarget = \"erlang\"\n\n[dependencies]\nqcheck = \"== 1.0.5\"\n",
|
|
192
|
+
"test-support/src/.gitkeep": "",
|
|
193
|
+
"test/lawspec_example_test.gleam": "@external(erlang, \"lawspec_beam_test_run\", \"gleam_main\")\npub fn main() -> Nil\n"
|
|
194
|
+
},
|
|
195
|
+
"compact": {
|
|
196
|
+
"gleam.toml": "name = \"lawspec_example\"\nversion = \"0.1.0\"\ngleam = \">= 1.18.0\"\ntarget = \"erlang\"\n\n[dependencies]\ngleam_stdlib = \"== 1.0.5\"\n\n[dev-dependencies]\ngleeunit = \"== 1.11.0\"\nqcheck = \"== 1.0.5\"\nlawspec_test_support = { path = \"./test-support\" }\n",
|
|
197
|
+
"test-support/gleam.toml": "# Generated test helpers are a development-only dependency.\nname = \"lawspec_test_support\"\nversion = \"0.1.0\"\ntarget = \"erlang\"\n\n[dependencies]\nqcheck = \"== 1.0.5\"\n",
|
|
198
|
+
"test-support/src/.gitkeep": "",
|
|
199
|
+
"test/lawspec_example_test.gleam": "@external(erlang, \"lawspec_beam_test_run\", \"gleam_main\")\npub fn main() -> Nil\n"
|
|
200
|
+
}
|
|
112
201
|
}
|
|
113
202
|
};
|
|
114
203
|
|
package/test-command.mjs
CHANGED
|
@@ -9,19 +9,23 @@
|
|
|
9
9
|
// a law that calls adapters, every file in the target project that LawSpec
|
|
10
10
|
// did not generate. A passing run records each law's key and seed; a law runs
|
|
11
11
|
// again when its key changes or with --fresh.
|
|
12
|
-
import { createHash } from "node:crypto";
|
|
12
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
13
13
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
|
|
16
16
|
// Folders that hold dependencies, build output or caches, not code under test.
|
|
17
17
|
const skipped = new Set([".git", ".lawspec", "node_modules", "dist", "build", "target", "out", ".gradle",
|
|
18
|
-
".stack-work", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
|
|
18
|
+
".stack-work", "_build", "deps", ".elixir_ls", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
|
|
19
19
|
// Build and lock files, which choose the toolchain and dependencies.
|
|
20
20
|
const environmentFiles = ["package.json", "package-lock.json", "tsconfig.json", "pyproject.toml", "go.mod",
|
|
21
21
|
"go.sum", "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts",
|
|
22
|
-
"Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml"
|
|
22
|
+
"Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml",
|
|
23
|
+
"rebar.config", "rebar.config.script", "rebar.lock", "mix.exs", "mix.lock", "gleam.toml", "manifest.toml"];
|
|
23
24
|
|
|
24
25
|
const digest = (value) => createHash("sha256").update(value).digest("hex");
|
|
26
|
+
// The crypto builder owns these outputs. Inputs and other priv/ assets still
|
|
27
|
+
// contribute to the adapter digest; rebuilding the NIF cannot invalidate a pass.
|
|
28
|
+
const cryptoBuildOutput = (file) => /^priv\/lawspec_crypto_native\.(?:so|dll)(?:\.build|\.\d+\.tmp)?$/.test(file);
|
|
25
29
|
|
|
26
30
|
// Every file in a project that LawSpec did not generate, by content.
|
|
27
31
|
export async function projectDigest(root, generated) {
|
|
@@ -33,7 +37,7 @@ export async function projectDigest(root, generated) {
|
|
|
33
37
|
const relative = folder ? `${folder}/${entry.name}` : entry.name;
|
|
34
38
|
if (entry.isDirectory()) {
|
|
35
39
|
if (!skipped.has(entry.name) && !relative.endsWith("testdata/rapid")) await walk(relative);
|
|
36
|
-
} else if (entry.isFile() && !generated.has(relative)) {
|
|
40
|
+
} else if (entry.isFile() && !generated.has(relative) && !cryptoBuildOutput(relative)) {
|
|
37
41
|
hash.update(relative + "\0").update(await readFile(path.join(root, relative))).update("\0");
|
|
38
42
|
}
|
|
39
43
|
}
|
|
@@ -43,7 +47,10 @@ export async function projectDigest(root, generated) {
|
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
export async function environmentDigest(root, report) {
|
|
46
|
-
|
|
50
|
+
// Coverage locations describe build output; native tools can discover more
|
|
51
|
+
// output directories after the first run. Build configuration is hashed below.
|
|
52
|
+
const {coverage: _coverage, ...toolchain} = report ?? {};
|
|
53
|
+
const parts = [JSON.stringify(report ? toolchain : null)];
|
|
47
54
|
const names = (await readdir(root)).filter((name) => environmentFiles.includes(name) || name.endsWith(".cabal")).sort();
|
|
48
55
|
for (const name of names) parts.push(name, await readFile(path.join(root, name), "utf8"));
|
|
49
56
|
return digest(JSON.stringify(parts));
|
|
@@ -86,6 +93,14 @@ const groupBy = (entries, key) => {
|
|
|
86
93
|
return [...groups.entries()];
|
|
87
94
|
};
|
|
88
95
|
|
|
96
|
+
// Rebar and Mix run their configured compile hooks. Gleam has no corresponding
|
|
97
|
+
// hook, so build its generated crypto bridge before starting a native test VM.
|
|
98
|
+
export function preparationInvocations(target, files) {
|
|
99
|
+
return target.language === "gleam" && files.some((file) => file.path === "priv/lawspec_crypto_native.c")
|
|
100
|
+
? [{command: "escript", args: ["lawspec_crypto_build.escript"]}]
|
|
101
|
+
: [];
|
|
102
|
+
}
|
|
103
|
+
|
|
89
104
|
// The native invocations that run the harness's benchmarks (lawspec test
|
|
90
105
|
// --benchmarks): each target's test command, selecting the tests that
|
|
91
106
|
// measure them by the names the manifest gives (entry.name). Benchmarks are
|
|
@@ -100,6 +115,18 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
100
115
|
return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
|
|
101
116
|
};
|
|
102
117
|
const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
|
|
118
|
+
if (language === "erlang")
|
|
119
|
+
return entries.map((entry) => ({ command: "rebar3", args: ["eunit", "--generator",
|
|
120
|
+
`${path.posix.basename(entry.file, ".erl")}:${entry.name}_cases`] }));
|
|
121
|
+
if (language === "elixir")
|
|
122
|
+
return groupBy(entries, (entry) => entry.file).map(([file, marks]) => ({ command: "mix",
|
|
123
|
+
args: ["test", file, ...marks.flatMap((entry) => ["--only", `lawspec:${entry.name}`])] }));
|
|
124
|
+
if (language === "gleam")
|
|
125
|
+
return [{ command: "gleam", args: ["test", "--target", "erlang"], env: {
|
|
126
|
+
LAWSPEC_BEAM_BENCHMARKS: JSON.stringify(entries.map((entry) => ({
|
|
127
|
+
module: path.posix.basename(entry.file, ".gleam"), function: `${entry.name}__case_0_test`
|
|
128
|
+
})))
|
|
129
|
+
} }];
|
|
103
130
|
if (language === "python")
|
|
104
131
|
return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: target.python || "python3",
|
|
105
132
|
args: ["-m", "pytest", "-q", "-s", file, "-k", marks.map((e) => e.name).join(" or ")] }));
|
|
@@ -143,7 +170,9 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
143
170
|
// LawSpec.TestNames), then a kind after a separator no name contains, so one
|
|
144
171
|
// law's name never selects another's. `scratch` is a folder for reports;
|
|
145
172
|
// `coverage` asks each runner to measure coverage too (see coverageSetup).
|
|
146
|
-
export function invocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false
|
|
173
|
+
export function invocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false,
|
|
174
|
+
root = process.cwd(), coverageConfig } = {}) {
|
|
175
|
+
if (!entries.length) return [];
|
|
147
176
|
const language = target.language;
|
|
148
177
|
const testDir = target.testDir;
|
|
149
178
|
const relativeTo = (file, defaultDir) => {
|
|
@@ -152,6 +181,42 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
152
181
|
};
|
|
153
182
|
const named = (entry) => new RegExp(`^${regex(entry.name)}_`);
|
|
154
183
|
const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
|
|
184
|
+
if (["erlang", "elixir", "gleam"].includes(language)) {
|
|
185
|
+
const run = randomUUID();
|
|
186
|
+
const file = path.join(scratch, `beam-${run}.jsonl`);
|
|
187
|
+
const identity = (entry) => `${entry.unit}::${entry.name}`;
|
|
188
|
+
const env = { LAWSPEC_BEAM_REPORT: file, LAWSPEC_BEAM_RUN: run };
|
|
189
|
+
const measured = coverage ? {run, file: path.resolve(coverage, "runs", `${run}.coverdata`),
|
|
190
|
+
...(language === "elixir" ? {beamDirectory: coverageConfig?.compilePath} : {}),
|
|
191
|
+
...(language === "erlang" ? {tool: "rebar3"} : {}),
|
|
192
|
+
...(language === "gleam" ? {metadata: path.resolve(coverage, "runs", `${run}.json`)} : {})} : null;
|
|
193
|
+
let command, args;
|
|
194
|
+
if (language === "elixir") {
|
|
195
|
+
command = "mix";
|
|
196
|
+
args = ["test", ...new Set(entries.map((entry) => entry.file)),
|
|
197
|
+
...entries.flatMap((entry) => ["--only", `lawspec_identity:${identity(entry)}`])];
|
|
198
|
+
if (measured) {
|
|
199
|
+
if (coverageConfig?.tool !== "Mix.Tasks.Test.Coverage" || !coverageConfig.output || !coverageConfig.compilePath)
|
|
200
|
+
throw new Error("BEAM coverage requires Mix.Tasks.Test.Coverage and its effective output directory from lawspec doctor");
|
|
201
|
+
args.push("--cover", "--export-coverage",
|
|
202
|
+
path.relative(path.resolve(root, coverageConfig.output), without(measured.file, ".coverdata")));
|
|
203
|
+
}
|
|
204
|
+
if (offline) env.HEX_OFFLINE = "1";
|
|
205
|
+
} else {
|
|
206
|
+
env.LAWSPEC_BEAM_LAWS = JSON.stringify(entries.map(({unit, name}) => ({unit, name})));
|
|
207
|
+
if (language === "erlang") {
|
|
208
|
+
command = "rebar3";
|
|
209
|
+
args = ["eunit", "--generator", "lawspec_generated_tests:lawspec_test_"];
|
|
210
|
+
if (measured) args.push("--cover", "--cover_export_name", without(measured.file, ".coverdata"));
|
|
211
|
+
} else {
|
|
212
|
+
command = "gleam";
|
|
213
|
+
args = ["test", "--target", "erlang"];
|
|
214
|
+
if (measured) env.LAWSPEC_BEAM_COVERAGE = JSON.stringify(measured);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return [{ laws: entries, command, args, env, ...(measured ? {coverage: measured} : {}), report: {kind: "beam-events", file, run},
|
|
218
|
+
ran: (test) => test.status === "passed" ? entries.filter((entry) => test.identity === identity(entry)) : [] }];
|
|
219
|
+
}
|
|
155
220
|
if (language === "python")
|
|
156
221
|
return groupBy(entries, (e) => e.file).map(([file, laws], n) => {
|
|
157
222
|
const report = path.join(scratch, `pytest-${n}.xml`);
|
|
@@ -232,6 +297,9 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
232
297
|
|
|
233
298
|
// What --coverage needs on each target, and how to say it is missing.
|
|
234
299
|
export const coverageTools = {
|
|
300
|
+
erlang: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
301
|
+
elixir: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
302
|
+
gleam: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
235
303
|
python: { tool: "coverage.py", check: ["-c", "import coverage"], install: "python -m pip install coverage" },
|
|
236
304
|
javascript: { tool: "c8", install: "npm install --save-dev c8" },
|
|
237
305
|
typescript: { tool: "c8", install: "npm install --save-dev c8" },
|
|
@@ -248,18 +316,73 @@ export function selectByTags(entries, include = [], exclude = []) {
|
|
|
248
316
|
!exclude.some((t) => (e.tags ?? []).includes(t)));
|
|
249
317
|
}
|
|
250
318
|
|
|
319
|
+
// A recorded failure takes precedence over any older passing cache entry,
|
|
320
|
+
// including when an adapter is repaired by restoring its previous contents.
|
|
321
|
+
export function testsToRun(entries, keys, previous, failures, {fresh = false, coverage = false, updateRecorded = false} = {}) {
|
|
322
|
+
return entries.filter((entry) => fresh || coverage || updateRecorded || failures[entry.law] ||
|
|
323
|
+
previous[entry.law]?.key !== keys.get(entry.law));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Read complete outer elements, preserving nested suites and native payloads.
|
|
327
|
+
// Comments and CDATA can contain apparent XML tags; they are never elements.
|
|
328
|
+
function junitElements(xml, names) {
|
|
329
|
+
const elements = [];
|
|
330
|
+
let current;
|
|
331
|
+
const tags = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z_][\w:.-]*)(?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
|
|
332
|
+
for (const tag of xml.matchAll(tags)) {
|
|
333
|
+
const [, closing, name] = tag;
|
|
334
|
+
if (!name || !names.includes(name)) continue;
|
|
335
|
+
const empty = /\/\s*>$/.test(tag[0]);
|
|
336
|
+
if (!current) {
|
|
337
|
+
if (closing) continue;
|
|
338
|
+
current = {name, start: tag.index, opening: tag[0], depth: 0};
|
|
339
|
+
}
|
|
340
|
+
if (name !== current.name) continue;
|
|
341
|
+
current.depth += closing ? -1 : empty ? 0 : 1;
|
|
342
|
+
if (current.depth === 0) {
|
|
343
|
+
elements.push({...current, xml: xml.slice(current.start, tag.index + tag[0].length)});
|
|
344
|
+
current = undefined;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return elements;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function junitCounts(cases) {
|
|
351
|
+
const statuses = cases.map(test => new Set(junitElements(test.xml, ["failure", "error", "skipped"]).map(e => e.name)));
|
|
352
|
+
return {tests: cases.length, ...Object.fromEntries(["failure", "error", "skipped"].map(name =>
|
|
353
|
+
[name === "skipped" ? name : `${name}s`, statuses.filter(status => status.has(name)).length]))};
|
|
354
|
+
}
|
|
355
|
+
|
|
251
356
|
// One JUnit report from every target's: each target's suites, renamed with
|
|
252
357
|
// the target, in one <testsuites>.
|
|
253
358
|
export function mergeJunit(reports) {
|
|
254
359
|
const suites = [];
|
|
360
|
+
const totals = {tests: 0, failures: 0, errors: 0, skipped: 0};
|
|
255
361
|
for (const { target, xml } of reports) {
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
362
|
+
const elements = junitElements(xml, ["testsuite", "testcase"]);
|
|
363
|
+
const found = elements.filter(e => e.name === "testsuite");
|
|
364
|
+
// Node's native JUnit reporter puts top-level tests directly in testsuites.
|
|
365
|
+
// Keep those cases (and their native failure payloads) instead of silently
|
|
366
|
+
// dropping them because they have no intervening testsuite element.
|
|
367
|
+
const loose = elements.filter(e => e.name === "testcase");
|
|
368
|
+
if (loose.length) {
|
|
369
|
+
const opening = `<testsuite name="native" ${Object.entries(junitCounts(loose)).map(([key, value]) => `${key}="${value}"`).join(" ")}>`;
|
|
370
|
+
found.push({opening, xml: opening + "\n" + loose.map(e => e.xml).join("\n") + "\n</testsuite>"});
|
|
371
|
+
}
|
|
372
|
+
for (const suite of found) {
|
|
373
|
+
const attributes = [...suite.opening.matchAll(/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)];
|
|
374
|
+
const values = Object.fromEntries(attributes.map(match => [match[1], match[2] ?? match[3]]));
|
|
375
|
+
const counts = junitCounts(junitElements(suite.xml, ["testcase"]));
|
|
376
|
+
for (const key of Object.keys(totals))
|
|
377
|
+
totals[key] += /^\d+$/.test(values[key] ?? "") ? Number(values[key]) : counts[key];
|
|
378
|
+
const name = attributes.find(match => match[1] === "name");
|
|
379
|
+
const opening = name
|
|
380
|
+
? suite.opening.slice(0, name.index) + ` name="${escapeXml(target)}: ${values.name.replaceAll('"', '"')}"` + suite.opening.slice(name.index + name[0].length)
|
|
381
|
+
: suite.opening.replace(/^<testsuite\b/, `<testsuite name="${escapeXml(target)}"`);
|
|
382
|
+
suites.push(opening + suite.xml.slice(suite.opening.length));
|
|
383
|
+
}
|
|
260
384
|
}
|
|
261
|
-
|
|
262
|
-
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${count("tests")}" failures="${count("failures")}" errors="${count("errors")}" skipped="${count("skipped")}">\n` +
|
|
385
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${totals.tests}" failures="${totals.failures}" errors="${totals.errors}" skipped="${totals.skipped}">\n` +
|
|
263
386
|
suites.join("\n") + "\n</testsuites>\n";
|
|
264
387
|
}
|
|
265
388
|
|
|
@@ -278,6 +401,20 @@ export async function harnessStatistics(directory) {
|
|
|
278
401
|
// The tests a run executed, as {name, classname}, from the runner's report or
|
|
279
402
|
// output. Skipped tests are not counted.
|
|
280
403
|
export async function executedTests(report, output, root, since) {
|
|
404
|
+
if (report?.kind === "beam-events") {
|
|
405
|
+
// A native listener seals the invocation after its final event. Missing,
|
|
406
|
+
// stale, malformed or interrupted reports are not execution evidence.
|
|
407
|
+
const raw = await readFile(path.resolve(root, report.file), "utf8").catch(() => "");
|
|
408
|
+
let rows;
|
|
409
|
+
try { rows = raw.trim().split("\n").map((line) => JSON.parse(line)); } catch { return []; }
|
|
410
|
+
if (rows.length < 2 || rows[0]?.event !== "start" || rows.at(-1)?.event !== "end" ||
|
|
411
|
+
rows.some((row) => row?.run !== report.run)) return [];
|
|
412
|
+
const tests = rows.slice(1, -1);
|
|
413
|
+
if (tests.some((row) => row.event !== "test" || !["passed", "failed"].includes(row.status) ||
|
|
414
|
+
typeof row.name !== "string" || typeof row.classname !== "string" || typeof row.identity !== "string" ||
|
|
415
|
+
!Number.isFinite(row.time) || row.time < 0)) return [];
|
|
416
|
+
return tests;
|
|
417
|
+
}
|
|
281
418
|
if (report?.kind === "junit") {
|
|
282
419
|
const files = report.files ?? await reportFiles(path.join(root, report.directory), since);
|
|
283
420
|
const tests = [];
|
|
@@ -321,9 +458,18 @@ export async function executedTests(report, output, root, since) {
|
|
|
321
458
|
// JUnit XML for a runner that writes none, from the tests it printed.
|
|
322
459
|
export function junitFromTests(tests) {
|
|
323
460
|
const failures = tests.filter((t) => t.status === "failed").length;
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
461
|
+
const skipped = tests.filter((t) => t.status === "skipped").length;
|
|
462
|
+
// A BEAM scheduler wraps native functions from several units. Its module
|
|
463
|
+
// is not the declaring class: retain the unit from the validated identity.
|
|
464
|
+
const classname = (test) => {
|
|
465
|
+
const separator = typeof test.identity === "string" ? test.identity.lastIndexOf("::") : -1;
|
|
466
|
+
return separator > 0 ? test.identity.slice(0, separator) : test.classname ?? "";
|
|
467
|
+
};
|
|
468
|
+
return `<testsuite name="lawspec" tests="${tests.length}" failures="${failures}"${skipped ? ` skipped="${skipped}"` : ""}>\n` +
|
|
469
|
+
tests.map((t) => ` <testcase classname="${escapeXml(classname(t))}" name="${escapeXml(t.name)}"` +
|
|
470
|
+
(Number.isFinite(t.time) ? ` time="${t.time}"` : "") +
|
|
471
|
+
(t.status === "failed" ? `><failure message="${escapeXml(t.failure ?? "failed")}"/></testcase>` :
|
|
472
|
+
t.status === "skipped" ? `><skipped message="${escapeXml(t.reason ?? "skipped")}"/></testcase>` : "/>")).join("\n") + "\n</testsuite>";
|
|
327
473
|
}
|
|
328
474
|
|
|
329
475
|
async function reportFiles(directory, since) {
|