lawspec 0.31.1 → 0.31.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -2
- package/README.md +85 -9
- package/api.mjs +14 -9
- package/authoring.mjs +899 -0
- package/beam-coverage.mjs +190 -65
- package/beam-doctor.mjs +343 -124
- package/bin/lawspec.mjs +1949 -661
- package/build.json +134 -128
- package/component-packages.mjs +662 -176
- package/component.d.ts +82 -20
- package/component.mjs +317 -94
- package/core.wasm +2 -2
- package/core_jsffi.js +5 -5
- package/doctor.mjs +558 -299
- package/examples-command.mjs +44 -23
- package/failure-database.mjs +60 -17
- package/files.mjs +122 -62
- package/index.d.ts +585 -312
- package/input-files.mjs +90 -25
- package/launcher.mjs +72 -31
- package/learn/coverage.json +184 -112
- package/learn/curriculum.json +4 -1
- package/learn/vendor/fantomas/{LawSpec.Learn.Formatter.5arcua4vy0.wasm → LawSpec.Learn.Formatter.poiijdbsl9.wasm} +0 -0
- package/learn/vendor/fantomas/dotnet.js +3 -3
- package/learn/vendor/manifest.json +4 -8
- package/lsp.mjs +444 -133
- package/mcp.mjs +250 -0
- package/native-evidence.mjs +668 -182
- package/native-examples.mjs +56 -19
- package/native-host.mjs +136 -42
- package/native-parsers/clojure.LICENSE +121 -0
- package/native-parsers/clojure.wasm +0 -0
- package/native-parsers/csharp.LICENSE +21 -0
- package/native-parsers/csharp.wasm +0 -0
- package/native-parsers/elixir.LICENSE +176 -0
- package/native-parsers/elixir.wasm +0 -0
- package/native-parsers/erlang.LICENSE +201 -0
- package/native-parsers/erlang.wasm +0 -0
- package/native-parsers/fsharp.LICENSE +21 -0
- package/native-parsers/fsharp.wasm +0 -0
- package/native-parsers/gleam.LICENSE +202 -0
- package/native-parsers/gleam.wasm +0 -0
- package/native-parsers/go.LICENSE +21 -0
- package/native-parsers/go.wasm +0 -0
- package/native-parsers/groovy.LICENSE +21 -0
- package/native-parsers/groovy.wasm +0 -0
- package/native-parsers/haskell.LICENSE +21 -0
- package/native-parsers/haskell.wasm +0 -0
- package/native-parsers/java.LICENSE +21 -0
- package/native-parsers/java.wasm +0 -0
- package/native-parsers/javascript.LICENSE +21 -0
- package/native-parsers/javascript.wasm +0 -0
- package/native-parsers/kotlin.LICENSE +9 -0
- package/native-parsers/kotlin.wasm +0 -0
- package/native-parsers/lock.json +114 -0
- package/native-parsers/prolog.LICENSE +21 -0
- package/native-parsers/prolog.wasm +0 -0
- package/native-parsers/python.LICENSE +21 -0
- package/native-parsers/python.wasm +0 -0
- package/native-parsers/rust.LICENSE +21 -0
- package/native-parsers/rust.wasm +0 -0
- package/native-parsers/scala.LICENSE +21 -0
- package/native-parsers/scala.wasm +0 -0
- package/native-parsers/swift.LICENSE +21 -0
- package/native-parsers/swift.wasm +0 -0
- package/native-parsers/typescript.LICENSE +21 -0
- package/native-parsers/typescript.wasm +0 -0
- package/node-events-reporter.mjs +41 -17
- package/package.json +4 -3
- package/project.mjs +338 -0
- package/proof.d.ts +93 -19
- package/proof.mjs +1538 -447
- package/read-only-hpack.mjs +6 -5
- package/scala-build-state.mjs +131 -41
- package/scala-coverage.mjs +57 -34
- package/scalars.mjs +23 -8
- package/templates.mjs +566 -617
- package/test-command.mjs +1671 -521
- package/learn/vendor/fantomas/LawSpec.Learn.Formatter.u1ne3a6xd2.wasm +0 -0
package/test-command.mjs
CHANGED
|
@@ -9,92 +9,186 @@
|
|
|
9
9
|
// a law that calls adapters, every file in the target project that LawSpec
|
|
10
10
|
// did not generate. A passing run records each law's key and seed; a law runs
|
|
11
11
|
// again when its key changes or with --fresh.
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import path from
|
|
15
|
-
import {presentationChecks,templates} from
|
|
12
|
+
import {createHash, randomUUID} from 'node:crypto';
|
|
13
|
+
import {readdir, readFile, stat} from 'node:fs/promises';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
import {presentationChecks, templates} from './templates.mjs';
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
buildDirectories as skipped,
|
|
19
|
+
cryptoBuildOutput,
|
|
20
|
+
isAttestation,
|
|
21
|
+
fingerprintInputs,
|
|
22
|
+
} from './input-files.mjs';
|
|
18
23
|
export {isAttestation} from './input-files.mjs';
|
|
19
24
|
import {nativeHostIdentity} from './native-host.mjs';
|
|
20
25
|
|
|
21
26
|
// Build and lock files, which choose the toolchain and dependencies.
|
|
22
|
-
const environmentFiles = [
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
+
const environmentFiles = [
|
|
28
|
+
'package.json',
|
|
29
|
+
'package-lock.json',
|
|
30
|
+
'tsconfig.json',
|
|
31
|
+
'pyproject.toml',
|
|
32
|
+
'go.mod',
|
|
33
|
+
'go.sum',
|
|
34
|
+
'pom.xml',
|
|
35
|
+
'build.gradle',
|
|
36
|
+
'build.gradle.kts',
|
|
37
|
+
'settings.gradle',
|
|
38
|
+
'settings.gradle.kts',
|
|
39
|
+
'Cargo.toml',
|
|
40
|
+
'Cargo.lock',
|
|
41
|
+
'stack.yaml',
|
|
42
|
+
'stack.yaml.lock',
|
|
43
|
+
'package.yaml',
|
|
44
|
+
'rebar.config',
|
|
45
|
+
'rebar.config.script',
|
|
46
|
+
'rebar.lock',
|
|
47
|
+
'mix.exs',
|
|
48
|
+
'mix.lock',
|
|
49
|
+
'gleam.toml',
|
|
50
|
+
'manifest.toml',
|
|
51
|
+
'global.json',
|
|
52
|
+
'Directory.Build.props',
|
|
53
|
+
'Directory.Packages.props',
|
|
54
|
+
'NuGet.config',
|
|
55
|
+
'project.scala',
|
|
56
|
+
'Package.swift',
|
|
57
|
+
'Package.resolved',
|
|
58
|
+
];
|
|
27
59
|
|
|
28
|
-
const digest =
|
|
60
|
+
const digest = value => createHash('sha256').update(value).digest('hex');
|
|
29
61
|
// Every file in a project that LawSpec did not generate, by content, except
|
|
30
62
|
// environment attestations.
|
|
31
|
-
export async function projectDigest(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
63
|
+
export async function projectDigest(
|
|
64
|
+
root,
|
|
65
|
+
generated,
|
|
66
|
+
language,
|
|
67
|
+
{inputRoot = root, inputs = []} = {},
|
|
68
|
+
)
|
|
69
|
+
{
|
|
70
|
+
const readable = language ? templates(language) : {};
|
|
71
|
+
const compact = language ? templates(language, {minify: true}) : {};
|
|
72
|
+
const hash = createHash('sha256');
|
|
73
|
+
async function walk(folder)
|
|
74
|
+
{
|
|
75
|
+
const entries = (
|
|
76
|
+
await readdir(path.join(root, folder), {withFileTypes: true})
|
|
77
|
+
).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
78
|
+
for (const entry of entries)
|
|
79
|
+
{
|
|
39
80
|
const relative = folder ? `${folder}/${entry.name}` : entry.name;
|
|
40
|
-
if (entry.isDirectory())
|
|
41
|
-
|
|
42
|
-
|
|
81
|
+
if (entry.isDirectory())
|
|
82
|
+
{
|
|
83
|
+
if (!skipped.has(entry.name) && !relative.endsWith('testdata/rapid'))
|
|
84
|
+
await walk(relative);
|
|
85
|
+
}
|
|
86
|
+
else if (
|
|
87
|
+
entry.isFile() &&
|
|
88
|
+
!generated.has(relative) &&
|
|
89
|
+
!cryptoBuildOutput(relative)
|
|
90
|
+
)
|
|
91
|
+
{
|
|
43
92
|
const content = await readFile(path.join(root, relative));
|
|
44
|
-
if (
|
|
45
|
-
|
|
93
|
+
if (
|
|
94
|
+
relative.endsWith('.json') &&
|
|
95
|
+
isAttestation(content.toString('utf8'))
|
|
96
|
+
)
|
|
97
|
+
continue;
|
|
98
|
+
hash
|
|
99
|
+
.update(relative + '\0')
|
|
100
|
+
.update(
|
|
101
|
+
content.toString('utf8') === compact[relative]
|
|
102
|
+
? readable[relative]
|
|
103
|
+
: content,
|
|
104
|
+
)
|
|
105
|
+
.update('\0');
|
|
46
106
|
}
|
|
47
107
|
}
|
|
48
108
|
}
|
|
49
|
-
await walk(
|
|
50
|
-
if(inputs.length)
|
|
51
|
-
|
|
109
|
+
await walk('');
|
|
110
|
+
if (inputs.length)
|
|
111
|
+
{
|
|
112
|
+
const owned = new Set([...generated].map(file => path.resolve(root, file)));
|
|
52
113
|
// Target files already contribute above, including exact readable/compact
|
|
53
114
|
// scaffold normalization. Declared parent directories must not hash their
|
|
54
115
|
// raw bytes a second time and make a layout-only edit stale execution.
|
|
55
|
-
const outside = file =>
|
|
56
|
-
|
|
57
|
-
.
|
|
58
|
-
|
|
116
|
+
const outside = file =>
|
|
117
|
+
{
|
|
118
|
+
const relative = path.relative(path.resolve(root), file);
|
|
119
|
+
return (
|
|
120
|
+
relative === '..' ||
|
|
121
|
+
relative.startsWith('..' + path.sep) ||
|
|
122
|
+
path.isAbsolute(relative)
|
|
123
|
+
);
|
|
124
|
+
};
|
|
125
|
+
const external = (
|
|
126
|
+
await fingerprintInputs(inputRoot, inputs, {generated: owned})
|
|
127
|
+
).filter(([relative]) => outside(path.resolve(inputRoot, relative)));
|
|
128
|
+
hash
|
|
129
|
+
.update('declared-inputs\0')
|
|
130
|
+
.update(JSON.stringify(external))
|
|
131
|
+
.update('\0');
|
|
59
132
|
}
|
|
60
|
-
return hash.digest(
|
|
133
|
+
return hash.digest('hex');
|
|
61
134
|
}
|
|
62
135
|
|
|
63
|
-
export async function environmentDigest(root, report, language)
|
|
136
|
+
export async function environmentDigest(root, report, language)
|
|
137
|
+
{
|
|
64
138
|
// Coverage locations describe build output; native tools can discover more
|
|
65
139
|
// output directories after the first run. Build configuration is hashed below.
|
|
66
140
|
const {coverage: _coverage, ...toolchain} = report ?? {};
|
|
67
|
-
const parts = [
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
141
|
+
const parts = [
|
|
142
|
+
await nativeHostIdentity(),
|
|
143
|
+
JSON.stringify(report ? toolchain : null),
|
|
144
|
+
];
|
|
145
|
+
const names = (await readdir(root))
|
|
146
|
+
.filter(name => environmentFiles.includes(name) || name.endsWith('.cabal'))
|
|
147
|
+
.sort();
|
|
148
|
+
const readable = language ? templates(language) : {};
|
|
149
|
+
const compact = language ? templates(language, {minify: true}) : {};
|
|
150
|
+
for (const name of names)
|
|
151
|
+
{
|
|
152
|
+
const content = await readFile(path.join(root, name), 'utf8');
|
|
73
153
|
// Only byte-exact compiler-owned scaffold alternatives are equivalent.
|
|
74
154
|
// A user edit to build configuration always remains an execution input.
|
|
75
|
-
parts.push(name,content===compact[name]?readable[name]:content);
|
|
155
|
+
parts.push(name, content === compact[name] ? readable[name] : content);
|
|
76
156
|
}
|
|
77
157
|
return digest(JSON.stringify(parts));
|
|
78
158
|
}
|
|
79
159
|
|
|
80
160
|
// Every recorded value under recorded/, by content: "" when there are none.
|
|
81
|
-
export async function recordedDigest(folder)
|
|
82
|
-
|
|
161
|
+
export async function recordedDigest(folder)
|
|
162
|
+
{
|
|
163
|
+
const hash = createHash('sha256');
|
|
83
164
|
let any = false;
|
|
84
|
-
async function walk(relative)
|
|
85
|
-
|
|
86
|
-
|
|
165
|
+
async function walk(relative)
|
|
166
|
+
{
|
|
167
|
+
const entries = await readdir(path.join(folder, relative), {
|
|
168
|
+
withFileTypes: true,
|
|
169
|
+
}).catch(() => []);
|
|
170
|
+
for (const entry of entries.sort((a, b) =>
|
|
171
|
+
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
|
|
172
|
+
))
|
|
173
|
+
{
|
|
87
174
|
const next = relative ? `${relative}/${entry.name}` : entry.name;
|
|
88
|
-
if (entry.isSymbolicLink())
|
|
175
|
+
if (entry.isSymbolicLink())
|
|
176
|
+
throw new Error(
|
|
177
|
+
`Recording input is a symbolic link: ${path.join(folder, next)}`,
|
|
178
|
+
);
|
|
89
179
|
if (entry.isDirectory()) await walk(next);
|
|
90
|
-
else if (entry.isFile())
|
|
180
|
+
else if (entry.isFile())
|
|
181
|
+
{
|
|
91
182
|
any = true;
|
|
92
|
-
hash
|
|
183
|
+
hash
|
|
184
|
+
.update(next + '\0')
|
|
185
|
+
.update(await readFile(path.join(folder, next)))
|
|
186
|
+
.update('\0');
|
|
93
187
|
}
|
|
94
188
|
}
|
|
95
189
|
}
|
|
96
|
-
await walk(
|
|
97
|
-
return any ? hash.digest(
|
|
190
|
+
await walk('');
|
|
191
|
+
return any ? hash.digest('hex') : '';
|
|
98
192
|
}
|
|
99
193
|
|
|
100
194
|
// The result key of each law in a target's test manifest: what the law does
|
|
@@ -105,69 +199,132 @@ export async function recordedDigest(folder) {
|
|
|
105
199
|
// A law an environment attestation touches is keyed by how its dependencies
|
|
106
200
|
// were wired too, so a run against other endpoints, or the real dependency
|
|
107
201
|
// in place of a stand-in, runs it again; other laws' keys are unchanged.
|
|
108
|
-
export function lawKeys({
|
|
202
|
+
export function lawKeys({
|
|
203
|
+
build,
|
|
204
|
+
target,
|
|
205
|
+
machineBits,
|
|
206
|
+
minify,
|
|
207
|
+
tests,
|
|
208
|
+
environment,
|
|
209
|
+
project,
|
|
210
|
+
recorded = '',
|
|
211
|
+
attested = new Map(),
|
|
212
|
+
})
|
|
213
|
+
{
|
|
109
214
|
// The compiler resolves binding reachability per law. Missing flags remain
|
|
110
215
|
// conservative for an unknown producer; a checked false stays independent
|
|
111
216
|
// of unrelated native bindings in the same target.
|
|
112
217
|
const native = entry => entry.callsAdapters !== false;
|
|
113
|
-
const configuration = entry=>
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
218
|
+
const configuration = entry =>
|
|
219
|
+
JSON.stringify({
|
|
220
|
+
...target,
|
|
221
|
+
root: undefined,
|
|
222
|
+
...(!native(entry) ? {nativeBindings: undefined} : {}),
|
|
223
|
+
});
|
|
224
|
+
return new Map(
|
|
225
|
+
tests.map(entry => [
|
|
226
|
+
entry.law,
|
|
227
|
+
digest(
|
|
228
|
+
JSON.stringify([
|
|
229
|
+
build,
|
|
230
|
+
configuration(entry),
|
|
231
|
+
machineBits,
|
|
232
|
+
entry.key,
|
|
233
|
+
environment,
|
|
234
|
+
native(entry) ? project : null,
|
|
235
|
+
native(entry) ? recorded : null,
|
|
236
|
+
...(attested.has(entry.law) ? [attested.get(entry.law)] : []),
|
|
237
|
+
]),
|
|
238
|
+
),
|
|
239
|
+
]),
|
|
240
|
+
);
|
|
121
241
|
}
|
|
122
242
|
|
|
123
243
|
// A law's last result: under its identity, or, when the law was renamed,
|
|
124
244
|
// under any earlier identity with the same key.
|
|
125
|
-
export function previousResult(previous, law, key)
|
|
245
|
+
export function previousResult(previous, law, key)
|
|
246
|
+
{
|
|
126
247
|
if (previous[law]?.key === key) return previous[law];
|
|
127
|
-
return Object.values(previous ?? {}).find(
|
|
248
|
+
return Object.values(previous ?? {}).find(result => result?.key === key);
|
|
128
249
|
}
|
|
129
250
|
|
|
130
|
-
const regex =
|
|
251
|
+
const regex = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
131
252
|
const without = (file, extension) => file.slice(0, -extension.length);
|
|
132
|
-
const groupBy = (entries, key) =>
|
|
253
|
+
const groupBy = (entries, key) =>
|
|
254
|
+
{
|
|
133
255
|
const groups = new Map();
|
|
134
|
-
for (const entry of entries)
|
|
256
|
+
for (const entry of entries)
|
|
257
|
+
groups.set(key(entry), [...(groups.get(key(entry)) ?? []), entry]);
|
|
135
258
|
return [...groups.entries()];
|
|
136
259
|
};
|
|
137
260
|
|
|
138
261
|
// Hpack discovers generated modules before native result keys are computed.
|
|
139
262
|
// Stack owns this metadata transformation; no compilation or tests run here.
|
|
140
|
-
export function configurationInvocations(target)
|
|
141
|
-
|
|
263
|
+
export function configurationInvocations(target)
|
|
264
|
+
{
|
|
265
|
+
return target.language === 'haskell'
|
|
266
|
+
? [{command: 'stack', args: ['--no-terminal', 'config', 'build-files']}]
|
|
267
|
+
: [];
|
|
142
268
|
}
|
|
143
269
|
|
|
144
270
|
// Rebar and Mix run their configured compile hooks. Gleam has no corresponding
|
|
145
271
|
// hook, so build its generated crypto bridge before starting a native test VM.
|
|
146
|
-
export function preparationInvocations(target, files)
|
|
147
|
-
|
|
148
|
-
|
|
272
|
+
export function preparationInvocations(target, files)
|
|
273
|
+
{
|
|
274
|
+
return target.language === 'gleam' &&
|
|
275
|
+
files.some(file => file.path === 'priv/lawspec_crypto_native.c')
|
|
276
|
+
? [{command: 'escript', args: ['lawspec_crypto_build.escript']}]
|
|
149
277
|
: [];
|
|
150
278
|
}
|
|
151
279
|
|
|
152
280
|
// A changed layout must be accepted by the target compiler/parser before a
|
|
153
281
|
// semantic execution result can be reused. No property cases run here.
|
|
154
|
-
export function presentationInvocations(target,files,{offline=false}={})
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
282
|
+
export function presentationInvocations(target, files, {offline = false} = {})
|
|
283
|
+
{
|
|
284
|
+
const overrides = {
|
|
285
|
+
python3: target.python,
|
|
286
|
+
mvn: target.maven,
|
|
287
|
+
gradle: target.gradle,
|
|
288
|
+
scala: target.scala,
|
|
289
|
+
swift: target.swift,
|
|
290
|
+
clojure: target.clojure,
|
|
291
|
+
};
|
|
292
|
+
return (presentationChecks[target.language] ?? []).flatMap(check =>
|
|
293
|
+
{
|
|
294
|
+
const paths = check.suffix
|
|
295
|
+
? files
|
|
296
|
+
.filter(file => file.path.endsWith(check.suffix))
|
|
297
|
+
.map(file => file.path)
|
|
298
|
+
: [];
|
|
299
|
+
const extra = offline
|
|
300
|
+
? ({
|
|
301
|
+
cargo: ['--offline'],
|
|
302
|
+
mvn: ['-o'],
|
|
303
|
+
gradle: ['--offline'],
|
|
304
|
+
dotnet: ['--no-restore'],
|
|
305
|
+
}[check.command] ?? [])
|
|
306
|
+
: [];
|
|
307
|
+
const args = [...check.args, ...extra];
|
|
308
|
+
return (check.each ? paths.map(file => [file]) : [paths]).map(paths => ({
|
|
309
|
+
command: overrides[check.command] ?? check.command,
|
|
310
|
+
args: [...args, ...paths],
|
|
311
|
+
environment: check.environment,
|
|
312
|
+
}));
|
|
163
313
|
});
|
|
164
314
|
}
|
|
165
315
|
|
|
166
316
|
// Presentation bytes are deliberately kept out of lawKeys. This separate
|
|
167
317
|
// digest records which current artifacts have passed native source validation.
|
|
168
|
-
export function presentationDigest(files,environment)
|
|
169
|
-
|
|
170
|
-
|
|
318
|
+
export function presentationDigest(files, environment)
|
|
319
|
+
{
|
|
320
|
+
return digest(
|
|
321
|
+
JSON.stringify([
|
|
322
|
+
environment,
|
|
323
|
+
files
|
|
324
|
+
.filter(file => file.ownership === 'generated')
|
|
325
|
+
.map(file => [file.path, file.content]),
|
|
326
|
+
]),
|
|
327
|
+
);
|
|
171
328
|
}
|
|
172
329
|
|
|
173
330
|
// The native invocations that run the harness's benchmarks (lawspec test
|
|
@@ -175,88 +332,269 @@ export function presentationDigest(files,environment) {
|
|
|
175
332
|
// measure them by the names the manifest gives (entry.name). Benchmarks are
|
|
176
333
|
// measured, never asserted; their timings reach the summary through the
|
|
177
334
|
// harness statistics.
|
|
178
|
-
export function benchmarkInvocations(target, entries, {
|
|
335
|
+
export function benchmarkInvocations(target, entries, {offline = false} = {})
|
|
336
|
+
{
|
|
179
337
|
if (!entries.length) return [];
|
|
180
338
|
const language = target.language;
|
|
181
|
-
if (language ===
|
|
182
|
-
return [
|
|
183
|
-
|
|
339
|
+
if (language === 'prolog')
|
|
340
|
+
return [
|
|
341
|
+
{
|
|
342
|
+
command: target.python || 'python3',
|
|
343
|
+
args: [
|
|
344
|
+
'lawspec_test.py',
|
|
345
|
+
...entries.flatMap(entry => [
|
|
346
|
+
'--select',
|
|
347
|
+
`${entry.file}::${entry.name}`,
|
|
348
|
+
]),
|
|
349
|
+
],
|
|
350
|
+
},
|
|
351
|
+
];
|
|
184
352
|
const testDir = target.testDir;
|
|
185
|
-
const relativeTo = (file, defaultDir) =>
|
|
353
|
+
const relativeTo = (file, defaultDir) =>
|
|
354
|
+
{
|
|
186
355
|
const directory = testDir || defaultDir;
|
|
187
|
-
return file.startsWith(directory +
|
|
356
|
+
return file.startsWith(directory + '/')
|
|
357
|
+
? file.slice(directory.length + 1)
|
|
358
|
+
: file;
|
|
188
359
|
};
|
|
189
|
-
const className = (file, defaultDir, extension) =>
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
360
|
+
const className = (file, defaultDir, extension) =>
|
|
361
|
+
without(relativeTo(file, defaultDir), extension).replaceAll('/', '.');
|
|
362
|
+
if (language === 'erlang')
|
|
363
|
+
return entries.map(entry => ({
|
|
364
|
+
command: 'rebar3',
|
|
365
|
+
args: [
|
|
366
|
+
'eunit',
|
|
367
|
+
'--generator',
|
|
368
|
+
`${path.posix.basename(entry.file, '.erl')}:${entry.name}_cases`,
|
|
369
|
+
],
|
|
370
|
+
}));
|
|
371
|
+
if (language === 'elixir')
|
|
372
|
+
return groupBy(entries, entry => entry.file).map(([file, marks]) => ({
|
|
373
|
+
command: 'mix',
|
|
374
|
+
args: [
|
|
375
|
+
'test',
|
|
376
|
+
file,
|
|
377
|
+
...marks.flatMap(entry => ['--only', `lawspec:${entry.name}`]),
|
|
378
|
+
],
|
|
379
|
+
}));
|
|
380
|
+
if (language === 'gleam')
|
|
381
|
+
return [
|
|
382
|
+
{
|
|
383
|
+
command: 'gleam',
|
|
384
|
+
args: ['test', '--target', 'erlang'],
|
|
385
|
+
env: {
|
|
386
|
+
LAWSPEC_BEAM_BENCHMARKS: JSON.stringify(
|
|
387
|
+
entries.map(entry => ({
|
|
388
|
+
module: path.posix.basename(entry.file, '.gleam'),
|
|
389
|
+
function: `${entry.name}_test`,
|
|
390
|
+
})),
|
|
391
|
+
),
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
];
|
|
395
|
+
if (language === 'python')
|
|
396
|
+
return groupBy(entries, e => e.file).map(([file, marks]) => ({
|
|
397
|
+
command: target.python || 'python3',
|
|
398
|
+
args: [
|
|
399
|
+
'-m',
|
|
400
|
+
'pytest',
|
|
401
|
+
'-q',
|
|
402
|
+
'-s',
|
|
403
|
+
file,
|
|
404
|
+
'-k',
|
|
405
|
+
marks.map(e => e.name).join(' or '),
|
|
406
|
+
],
|
|
407
|
+
}));
|
|
408
|
+
if (language === 'javascript' || language === 'typescript')
|
|
409
|
+
{
|
|
410
|
+
const files = [
|
|
411
|
+
...new Set(
|
|
412
|
+
entries.map(e =>
|
|
413
|
+
language === 'typescript'
|
|
414
|
+
? `dist/${without(e.file, '.ts')}.js`
|
|
415
|
+
: e.file,
|
|
416
|
+
),
|
|
417
|
+
),
|
|
418
|
+
];
|
|
419
|
+
const run = {
|
|
420
|
+
command: process.execPath,
|
|
421
|
+
args: [
|
|
422
|
+
'--test',
|
|
423
|
+
`--test-name-pattern=^(${entries.map(e => regex(e.name)).join('|')})$`,
|
|
424
|
+
...files,
|
|
425
|
+
],
|
|
426
|
+
};
|
|
427
|
+
return language === 'typescript'
|
|
428
|
+
? [
|
|
429
|
+
{command: 'npm', args: ['exec', '--', 'tsc', '-p', 'tsconfig.json']},
|
|
430
|
+
run,
|
|
431
|
+
]
|
|
432
|
+
: [run];
|
|
210
433
|
}
|
|
211
|
-
if (language ===
|
|
212
|
-
return groupBy(entries,
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
434
|
+
if (language === 'go')
|
|
435
|
+
return groupBy(entries, e => path.posix.dirname(e.file)).map(
|
|
436
|
+
([directory, marks]) => ({
|
|
437
|
+
command: 'go',
|
|
438
|
+
args: [
|
|
439
|
+
'test',
|
|
440
|
+
'-count=1',
|
|
441
|
+
'-v',
|
|
442
|
+
`./${directory}`,
|
|
443
|
+
'-run',
|
|
444
|
+
`^(${marks.map(e => regex(e.name)).join('|')})$`,
|
|
445
|
+
],
|
|
446
|
+
}),
|
|
447
|
+
);
|
|
448
|
+
if (language === 'java')
|
|
449
|
+
{
|
|
450
|
+
const byClass = groupBy(entries, e =>
|
|
451
|
+
className(e.file, 'src/test/java', '.java'),
|
|
452
|
+
);
|
|
453
|
+
return [
|
|
454
|
+
{
|
|
455
|
+
command: target.maven || 'mvn',
|
|
456
|
+
env: {LAWSPEC_CASE: ''},
|
|
457
|
+
args: [
|
|
458
|
+
...(offline ? ['-o'] : []),
|
|
459
|
+
'-B',
|
|
460
|
+
'test',
|
|
461
|
+
`-Dtest=${byClass.map(([name, marks]) => `${name}#${marks.map(e => e.name).join('+')}`).join(',')}`,
|
|
462
|
+
],
|
|
463
|
+
},
|
|
464
|
+
];
|
|
218
465
|
}
|
|
219
|
-
if (language ===
|
|
220
|
-
return groupBy(entries,
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
466
|
+
if (language === 'kotlin')
|
|
467
|
+
return groupBy(entries, e => e.file).map(([file, marks]) =>
|
|
468
|
+
{
|
|
469
|
+
const filter = `(${marks.map(e => e.name).join('|')})`;
|
|
470
|
+
return {
|
|
471
|
+
command: target.gradle || 'gradle',
|
|
472
|
+
args: [
|
|
473
|
+
...(offline ? ['--offline'] : []),
|
|
474
|
+
'--console=plain',
|
|
475
|
+
'test',
|
|
476
|
+
'--rerun',
|
|
477
|
+
'--tests',
|
|
478
|
+
className(file, 'src/test/kotlin', '.kt'),
|
|
479
|
+
],
|
|
480
|
+
env: {'kotest.filter.tests': filter, kotest_filter_tests: filter},
|
|
481
|
+
};
|
|
225
482
|
});
|
|
226
483
|
// The Clojure runner selects tests by namespace and name prefix.
|
|
227
|
-
if (language ===
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
484
|
+
if (language === 'clojure')
|
|
485
|
+
{
|
|
486
|
+
const ns = file =>
|
|
487
|
+
without(relativeTo(file, 'test'), '.clj')
|
|
488
|
+
.replaceAll('/', '.')
|
|
489
|
+
.replaceAll('_', '-');
|
|
490
|
+
return [
|
|
491
|
+
{
|
|
492
|
+
command: target.clojure || 'clojure',
|
|
493
|
+
args: [
|
|
494
|
+
'-M:test',
|
|
495
|
+
...entries.flatMap(e => ['--select', `${ns(e.file)}/${e.name}`]),
|
|
496
|
+
],
|
|
497
|
+
},
|
|
498
|
+
];
|
|
231
499
|
}
|
|
232
500
|
// Spock names each feature in words; Gradle's filters select it by name.
|
|
233
|
-
if (language ===
|
|
234
|
-
return [
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
501
|
+
if (language === 'groovy')
|
|
502
|
+
return [
|
|
503
|
+
{
|
|
504
|
+
command: target.gradle || 'gradle',
|
|
505
|
+
env: {LAWSPEC_CASE: ''},
|
|
506
|
+
args: [
|
|
507
|
+
...(offline ? ['--offline'] : []),
|
|
508
|
+
'--console=plain',
|
|
509
|
+
'test',
|
|
510
|
+
'--rerun',
|
|
511
|
+
...entries.flatMap(e => [
|
|
512
|
+
'--tests',
|
|
513
|
+
`${className(e.file, 'src/test/groovy', '.groovy')}.${e.name}`,
|
|
514
|
+
]),
|
|
515
|
+
],
|
|
516
|
+
},
|
|
517
|
+
];
|
|
518
|
+
if (language === 'rust')
|
|
519
|
+
return groupBy(entries, e => e.file).map(([file, marks]) => ({
|
|
520
|
+
command: 'cargo',
|
|
521
|
+
args: [
|
|
522
|
+
'test',
|
|
523
|
+
...(offline ? ['--offline'] : []),
|
|
524
|
+
'--test',
|
|
525
|
+
path.posix.basename(file, '.rs'),
|
|
526
|
+
'--',
|
|
527
|
+
'--exact',
|
|
528
|
+
'--nocapture',
|
|
529
|
+
...marks.map(e => e.name),
|
|
530
|
+
],
|
|
531
|
+
}));
|
|
532
|
+
if (language === 'fsharp' || language === 'csharp')
|
|
533
|
+
return [
|
|
534
|
+
{
|
|
535
|
+
command: 'dotnet',
|
|
536
|
+
args: [
|
|
537
|
+
'test',
|
|
538
|
+
...entries.flatMap(e => ['--filter-query', `/*/*/*/${e.name}`]),
|
|
539
|
+
],
|
|
540
|
+
},
|
|
541
|
+
];
|
|
542
|
+
if (language === 'scala')
|
|
543
|
+
return groupBy(entries, e => e.file).map(([file, marks]) => ({
|
|
544
|
+
command: target.scala || 'scala',
|
|
545
|
+
args: [
|
|
546
|
+
'test',
|
|
547
|
+
'.',
|
|
548
|
+
...(offline ? ['--offline'] : []),
|
|
549
|
+
'--test-only',
|
|
550
|
+
className(file, 'src/test/scala', '.scala'),
|
|
551
|
+
'--',
|
|
552
|
+
...marks.flatMap(e => ['-z', e.name]),
|
|
553
|
+
],
|
|
554
|
+
}));
|
|
246
555
|
// Swift Testing selects tests by a regular expression over their ids
|
|
247
556
|
// (Module.Suite/function()), one --filter per benchmark, OR'd.
|
|
248
|
-
if (language ===
|
|
249
|
-
return [
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
557
|
+
if (language === 'swift')
|
|
558
|
+
return [
|
|
559
|
+
{
|
|
560
|
+
command: target.swift || 'swift',
|
|
561
|
+
args: [
|
|
562
|
+
'test',
|
|
563
|
+
...entries.flatMap(e => [
|
|
564
|
+
'--filter',
|
|
565
|
+
`/${regex(e.name)}\\(`,
|
|
566
|
+
'--filter',
|
|
567
|
+
`${regex(path.posix.basename(e.file, '.swift'))}/ordered\\(`,
|
|
568
|
+
]),
|
|
569
|
+
],
|
|
570
|
+
env: {
|
|
571
|
+
LAWSPEC_SWIFT_LAWS: JSON.stringify(
|
|
572
|
+
entries.map(({unit, name}) => ({unit, name})),
|
|
573
|
+
),
|
|
574
|
+
LAWSPEC_SWIFT_REPORT: '',
|
|
575
|
+
LAWSPEC_SWIFT_RUN: '',
|
|
576
|
+
},
|
|
577
|
+
},
|
|
578
|
+
];
|
|
579
|
+
if (language === 'haskell')
|
|
580
|
+
{
|
|
581
|
+
const module = e =>
|
|
582
|
+
without(relativeTo(e.file, 'test'), 'Spec.hs').replaceAll('/', '.');
|
|
256
583
|
// hspec matches by substring, and a benchmark's name has spaces, so each
|
|
257
584
|
// module's benchmarks are selected together.
|
|
258
|
-
return [
|
|
259
|
-
|
|
585
|
+
return [
|
|
586
|
+
{
|
|
587
|
+
command: 'stack',
|
|
588
|
+
args: [
|
|
589
|
+
'--no-terminal',
|
|
590
|
+
'test',
|
|
591
|
+
'--test-arguments',
|
|
592
|
+
[...new Set(entries.map(e => `--match ${module(e)}/benchmark`))].join(
|
|
593
|
+
' ',
|
|
594
|
+
),
|
|
595
|
+
],
|
|
596
|
+
},
|
|
597
|
+
];
|
|
260
598
|
}
|
|
261
599
|
throw new Error(`lawspec test does not support ${language}`);
|
|
262
600
|
}
|
|
@@ -267,455 +605,1160 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
267
605
|
// LawSpec.TestNames), then a kind after a separator no name contains, so one
|
|
268
606
|
// law's name never selects another's. `scratch` is a folder for reports;
|
|
269
607
|
// `coverage` asks each runner to measure coverage too (see coverageSetup).
|
|
270
|
-
export function invocations(...arguments_)
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
608
|
+
export function invocations(...arguments_)
|
|
609
|
+
{
|
|
610
|
+
return nativeInvocations(...arguments_).map(run =>
|
|
611
|
+
{
|
|
612
|
+
const matches = run.matches ?? run.ran;
|
|
613
|
+
return matches
|
|
614
|
+
? {
|
|
615
|
+
...run,
|
|
616
|
+
matches,
|
|
617
|
+
ran: test => (test.status === 'passed' ? matches(test) : []),
|
|
618
|
+
}
|
|
619
|
+
: run;
|
|
274
620
|
});
|
|
275
621
|
}
|
|
276
622
|
|
|
277
|
-
function nativeInvocations(
|
|
278
|
-
|
|
623
|
+
function nativeInvocations(
|
|
624
|
+
target,
|
|
625
|
+
entries,
|
|
626
|
+
{
|
|
627
|
+
offline = false,
|
|
628
|
+
scratch = '.',
|
|
629
|
+
coverage = null,
|
|
630
|
+
xdist = false,
|
|
631
|
+
root = process.cwd(),
|
|
632
|
+
coverageConfig,
|
|
633
|
+
} = {},
|
|
634
|
+
)
|
|
635
|
+
{
|
|
279
636
|
if (!entries.length) return [];
|
|
280
637
|
const language = target.language;
|
|
281
|
-
if (language ===
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
638
|
+
if (language === 'prolog')
|
|
639
|
+
{
|
|
640
|
+
if (coverage)
|
|
641
|
+
throw new Error('Prolog coverage instrumentation is not integrated');
|
|
642
|
+
const report = path.join(scratch, 'prolog.xml');
|
|
643
|
+
const matching = test =>
|
|
644
|
+
entries.filter(entry =>
|
|
645
|
+
entry.kind === 'model'
|
|
646
|
+
? test.name === entry.name
|
|
647
|
+
: test.name.startsWith(`${entry.name}__`),
|
|
648
|
+
);
|
|
649
|
+
const selections = [
|
|
650
|
+
...new Set(entries.map(entry => `${entry.file}::${entry.name}`)),
|
|
651
|
+
];
|
|
652
|
+
return [
|
|
653
|
+
{
|
|
654
|
+
laws: entries,
|
|
655
|
+
command: target.python || 'python3',
|
|
656
|
+
args: [
|
|
657
|
+
'lawspec_test.py',
|
|
658
|
+
...selections.flatMap(slot => ['--select', slot]),
|
|
659
|
+
'--junitxml',
|
|
660
|
+
report,
|
|
661
|
+
],
|
|
662
|
+
report: {kind: 'junit', files: [report]},
|
|
663
|
+
ran: matching,
|
|
664
|
+
},
|
|
665
|
+
];
|
|
290
666
|
}
|
|
291
667
|
const testDir = target.testDir;
|
|
292
|
-
const relativeTo = (file, defaultDir) =>
|
|
668
|
+
const relativeTo = (file, defaultDir) =>
|
|
669
|
+
{
|
|
293
670
|
const directory = testDir || defaultDir;
|
|
294
|
-
return file.startsWith(directory +
|
|
671
|
+
return file.startsWith(directory + '/')
|
|
672
|
+
? file.slice(directory.length + 1)
|
|
673
|
+
: file;
|
|
295
674
|
};
|
|
296
|
-
const model =
|
|
297
|
-
const named =
|
|
298
|
-
|
|
299
|
-
|
|
675
|
+
const model = entry => entry.kind === 'model';
|
|
676
|
+
const named = entry =>
|
|
677
|
+
new RegExp(`^${regex(entry.name)}${model(entry) ? '$' : '_'}`);
|
|
678
|
+
const className = (file, defaultDir, extension) =>
|
|
679
|
+
without(relativeTo(file, defaultDir), extension).replaceAll('/', '.');
|
|
680
|
+
if (['erlang', 'elixir', 'gleam'].includes(language))
|
|
681
|
+
{
|
|
300
682
|
const run = randomUUID();
|
|
301
683
|
const file = path.join(scratch, `beam-${run}.jsonl`);
|
|
302
|
-
const identity =
|
|
303
|
-
const matching = test =>
|
|
304
|
-
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
684
|
+
const identity = entry => `${entry.unit}::${entry.name}`;
|
|
685
|
+
const matching = test =>
|
|
686
|
+
entries.filter(entry => test.identity === identity(entry));
|
|
687
|
+
const env = {LAWSPEC_BEAM_REPORT: file, LAWSPEC_BEAM_RUN: run};
|
|
688
|
+
const measured = coverage
|
|
689
|
+
? {
|
|
690
|
+
run,
|
|
691
|
+
file: path.resolve(coverage, 'runs', `${run}.coverdata`),
|
|
692
|
+
...(language === 'elixir'
|
|
693
|
+
? {beamDirectory: coverageConfig?.compilePath}
|
|
694
|
+
: {}),
|
|
695
|
+
...(language === 'erlang' ? {tool: 'rebar3'} : {}),
|
|
696
|
+
...(language === 'gleam'
|
|
697
|
+
? {metadata: path.resolve(coverage, 'runs', `${run}.json`)}
|
|
698
|
+
: {}),
|
|
699
|
+
}
|
|
700
|
+
: null;
|
|
309
701
|
let command, args;
|
|
310
|
-
if (language ===
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
702
|
+
if (language === 'elixir')
|
|
703
|
+
{
|
|
704
|
+
command = 'mix';
|
|
705
|
+
args = [
|
|
706
|
+
'test',
|
|
707
|
+
...new Set(entries.map(entry => entry.file)),
|
|
708
|
+
...entries.flatMap(entry => [
|
|
709
|
+
'--only',
|
|
710
|
+
`lawspec_identity:${identity(entry)}`,
|
|
711
|
+
]),
|
|
712
|
+
];
|
|
713
|
+
if (measured)
|
|
714
|
+
{
|
|
715
|
+
if (
|
|
716
|
+
coverageConfig?.tool !== 'Mix.Tasks.Test.Coverage' ||
|
|
717
|
+
!coverageConfig.output ||
|
|
718
|
+
!coverageConfig.compilePath
|
|
719
|
+
)
|
|
720
|
+
throw new Error(
|
|
721
|
+
'BEAM coverage requires Mix.Tasks.Test.Coverage and its effective output directory from lawspec doctor',
|
|
722
|
+
);
|
|
723
|
+
args.push(
|
|
724
|
+
'--cover',
|
|
725
|
+
'--export-coverage',
|
|
726
|
+
path.relative(
|
|
727
|
+
path.resolve(root, coverageConfig.output),
|
|
728
|
+
without(measured.file, '.coverdata'),
|
|
729
|
+
),
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
if (offline) env.HEX_OFFLINE = '1';
|
|
733
|
+
}
|
|
734
|
+
else
|
|
735
|
+
{
|
|
736
|
+
env.LAWSPEC_BEAM_LAWS = JSON.stringify(
|
|
737
|
+
entries.map(({unit, name}) => ({unit, name})),
|
|
738
|
+
);
|
|
739
|
+
if (language === 'erlang')
|
|
740
|
+
{
|
|
741
|
+
command = 'rebar3';
|
|
742
|
+
args = [
|
|
743
|
+
'eunit',
|
|
744
|
+
'--generator',
|
|
745
|
+
'lawspec_generated_tests:lawspec_test_',
|
|
746
|
+
];
|
|
747
|
+
if (measured)
|
|
748
|
+
args.push(
|
|
749
|
+
'--cover',
|
|
750
|
+
'--cover_export_name',
|
|
751
|
+
without(measured.file, '.coverdata'),
|
|
752
|
+
);
|
|
319
753
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
command = "rebar3";
|
|
325
|
-
args = ["eunit", "--generator", "lawspec_generated_tests:lawspec_test_"];
|
|
326
|
-
if (measured) args.push("--cover", "--cover_export_name", without(measured.file, ".coverdata"));
|
|
327
|
-
} else {
|
|
328
|
-
command = "gleam";
|
|
329
|
-
args = ["test", "--target", "erlang"];
|
|
754
|
+
else
|
|
755
|
+
{
|
|
756
|
+
command = 'gleam';
|
|
757
|
+
args = ['test', '--target', 'erlang'];
|
|
330
758
|
if (measured) env.LAWSPEC_BEAM_COVERAGE = JSON.stringify(measured);
|
|
331
759
|
}
|
|
332
760
|
}
|
|
333
|
-
return [
|
|
334
|
-
|
|
761
|
+
return [
|
|
762
|
+
{
|
|
763
|
+
laws: entries,
|
|
764
|
+
command,
|
|
765
|
+
args,
|
|
766
|
+
env,
|
|
767
|
+
...(measured ? {coverage: measured} : {}),
|
|
768
|
+
report: {kind: 'beam-events', file, run},
|
|
769
|
+
matches: matching,
|
|
770
|
+
ran: test => (test.status === 'passed' ? matching(test) : []),
|
|
771
|
+
},
|
|
772
|
+
];
|
|
335
773
|
}
|
|
336
|
-
if (language ===
|
|
337
|
-
return groupBy(entries,
|
|
338
|
-
|
|
339
|
-
const
|
|
340
|
-
const
|
|
774
|
+
if (language === 'python')
|
|
775
|
+
return groupBy(entries, e => e.file).flatMap(([file, selected], n) =>
|
|
776
|
+
{
|
|
777
|
+
const laws = selected.filter(entry => !model(entry));
|
|
778
|
+
const models = selected.filter(model);
|
|
779
|
+
const invocations = [];
|
|
341
780
|
const report = path.join(scratch, `pytest-${n}.xml`);
|
|
342
781
|
// parallel: pytest-xdist runs the tests on several workers, when it
|
|
343
782
|
// is installed (lawspec test checks); otherwise one after another.
|
|
344
|
-
const workers =
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
783
|
+
const workers =
|
|
784
|
+
xdist && selected.some(e => e.parallel)
|
|
785
|
+
? ['-p', 'xdist', '-n', 'auto']
|
|
786
|
+
: [];
|
|
787
|
+
const pytest = [
|
|
788
|
+
'-m',
|
|
789
|
+
'pytest',
|
|
790
|
+
'-q',
|
|
791
|
+
...workers,
|
|
792
|
+
file,
|
|
793
|
+
'-k',
|
|
794
|
+
laws.map(e => `${e.name}__`).join(' or '),
|
|
795
|
+
`--junitxml=${report}`,
|
|
796
|
+
];
|
|
797
|
+
if (laws.length)
|
|
798
|
+
invocations.push({
|
|
799
|
+
laws,
|
|
800
|
+
command: target.python || 'python3',
|
|
801
|
+
args: coverage
|
|
802
|
+
? [
|
|
803
|
+
'-m',
|
|
804
|
+
'coverage',
|
|
805
|
+
'run',
|
|
806
|
+
'--append',
|
|
807
|
+
`--data-file=${path.join(coverage, '.coverage')}`,
|
|
808
|
+
...pytest.slice(1),
|
|
809
|
+
]
|
|
810
|
+
: pytest,
|
|
811
|
+
report: {kind: 'junit', files: [report]},
|
|
812
|
+
ran: test => laws.filter(e => test.name.startsWith(`${e.name}__`)),
|
|
813
|
+
});
|
|
814
|
+
if (models.length)
|
|
815
|
+
{
|
|
816
|
+
const modelReport = path.join(scratch, `pytest-model-${n}.xml`);
|
|
817
|
+
const arguments_ = [
|
|
818
|
+
'-m',
|
|
819
|
+
'pytest',
|
|
820
|
+
'-q',
|
|
821
|
+
...workers,
|
|
822
|
+
...new Set(models.map(entry => `${file}::${entry.name}`)),
|
|
823
|
+
`--junitxml=${modelReport}`,
|
|
824
|
+
];
|
|
825
|
+
invocations.push({
|
|
826
|
+
laws: models,
|
|
827
|
+
command: target.python || 'python3',
|
|
828
|
+
args: coverage
|
|
829
|
+
? [
|
|
830
|
+
'-m',
|
|
831
|
+
'coverage',
|
|
832
|
+
'run',
|
|
833
|
+
'--append',
|
|
834
|
+
`--data-file=${path.join(coverage, '.coverage')}`,
|
|
835
|
+
...arguments_.slice(1),
|
|
836
|
+
]
|
|
837
|
+
: arguments_,
|
|
838
|
+
report: {kind: 'junit', files: [modelReport]},
|
|
839
|
+
ran: test => models.filter(entry => test.name === entry.name),
|
|
840
|
+
});
|
|
358
841
|
}
|
|
359
842
|
return invocations;
|
|
360
843
|
});
|
|
361
|
-
if (language ===
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
const
|
|
368
|
-
|
|
844
|
+
if (language === 'javascript' || language === 'typescript')
|
|
845
|
+
{
|
|
846
|
+
const pattern = e =>
|
|
847
|
+
model(e)
|
|
848
|
+
? `^${regex(e.label)}$`
|
|
849
|
+
: `^${regex(e.label)}( example: .*| boundary \\d+| property| known discrepancy| skipped| replay| search)?$`;
|
|
850
|
+
const files = [
|
|
851
|
+
...new Set(
|
|
852
|
+
entries.map(e =>
|
|
853
|
+
language === 'typescript'
|
|
854
|
+
? `dist/${without(e.file, '.ts')}.js`
|
|
855
|
+
: e.file,
|
|
856
|
+
),
|
|
857
|
+
),
|
|
858
|
+
];
|
|
859
|
+
const file = path.join(scratch, 'node-events.jsonl');
|
|
860
|
+
const runId = randomUUID();
|
|
861
|
+
const matching = test =>
|
|
862
|
+
entries.filter(entry => new RegExp(pattern(entry)).test(test.name));
|
|
369
863
|
// parallel: test files run at the same time too.
|
|
370
|
-
const node = [
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
864
|
+
const node = [
|
|
865
|
+
'--test',
|
|
866
|
+
...(entries.some(e => e.parallel)
|
|
867
|
+
? [
|
|
868
|
+
'--test-concurrency=' +
|
|
869
|
+
Math.max(2, globalThis.navigator?.hardwareConcurrency ?? 2),
|
|
870
|
+
]
|
|
871
|
+
: []),
|
|
872
|
+
`--test-name-pattern=(${entries.map(pattern).join('|')})`,
|
|
873
|
+
'--test-reporter=spec',
|
|
874
|
+
'--test-reporter-destination=stdout',
|
|
875
|
+
`--test-reporter=${new URL('./node-events-reporter.mjs', import.meta.url).href}`,
|
|
876
|
+
`--test-reporter-destination=${file}`,
|
|
877
|
+
...files,
|
|
878
|
+
];
|
|
879
|
+
const run = {
|
|
880
|
+
laws: entries,
|
|
881
|
+
command: coverage ? 'npx' : process.execPath,
|
|
882
|
+
args: coverage
|
|
883
|
+
? [
|
|
884
|
+
'--no-install',
|
|
885
|
+
'c8',
|
|
886
|
+
'--reporter=text',
|
|
887
|
+
`--reports-dir=${coverage}`,
|
|
888
|
+
process.execPath,
|
|
889
|
+
...node,
|
|
890
|
+
]
|
|
891
|
+
: node,
|
|
377
892
|
// A CLI launched from node:test must start an independent native run.
|
|
378
|
-
env:{LAWSPEC_NODE_RUN:runId,NODE_TEST_CONTEXT:undefined},
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
893
|
+
env: {LAWSPEC_NODE_RUN: runId, NODE_TEST_CONTEXT: undefined},
|
|
894
|
+
report: {kind: 'node-events', file, run: runId},
|
|
895
|
+
matches: matching,
|
|
896
|
+
ran: test => (test.status === 'passed' ? matching(test) : []),
|
|
897
|
+
};
|
|
898
|
+
return language === 'typescript'
|
|
899
|
+
? [
|
|
900
|
+
{
|
|
901
|
+
laws: [],
|
|
902
|
+
command: 'npm',
|
|
903
|
+
args: ['exec', '--', 'tsc', '-p', 'tsconfig.json'],
|
|
904
|
+
},
|
|
905
|
+
run,
|
|
906
|
+
]
|
|
382
907
|
: [run];
|
|
383
908
|
}
|
|
384
|
-
if (language ===
|
|
385
|
-
return groupBy(entries,
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
909
|
+
if (language === 'go')
|
|
910
|
+
return groupBy(entries, e => path.posix.dirname(e.file)).map(
|
|
911
|
+
([directory, laws]) => ({
|
|
912
|
+
laws,
|
|
913
|
+
command: 'go',
|
|
914
|
+
args: [
|
|
915
|
+
'test',
|
|
916
|
+
'-json',
|
|
917
|
+
'-count=1',
|
|
918
|
+
...(coverage
|
|
919
|
+
? [
|
|
920
|
+
'-cover',
|
|
921
|
+
`-coverprofile=${path.join(coverage, `go-${directory.replaceAll('/', '_') || 'root'}.out`)}`,
|
|
922
|
+
]
|
|
923
|
+
: []),
|
|
924
|
+
`./${directory}`,
|
|
925
|
+
'-run',
|
|
926
|
+
laws.some(model)
|
|
927
|
+
? `^(${[...new Set(laws.map(e => regex(e.name) + (model(e) ? '$' : '_')))].join('|')})`
|
|
928
|
+
: `^(${laws.map(e => regex(e.name)).join('|')})_`,
|
|
929
|
+
],
|
|
930
|
+
report: {kind: 'go-json'},
|
|
931
|
+
ran: test =>
|
|
932
|
+
laws.filter(
|
|
933
|
+
e =>
|
|
934
|
+
named(e).test(test.name) && test.name !== `${e.name}_Boundaries`,
|
|
935
|
+
),
|
|
936
|
+
}),
|
|
937
|
+
);
|
|
938
|
+
if (language === 'java')
|
|
939
|
+
{
|
|
940
|
+
const byClass = groupBy(entries, e =>
|
|
941
|
+
className(e.file, 'src/test/java', '.java'),
|
|
942
|
+
);
|
|
943
|
+
return [
|
|
944
|
+
{
|
|
945
|
+
laws: entries,
|
|
946
|
+
command: target.maven || 'mvn',
|
|
947
|
+
env: {LAWSPEC_CASE: ''},
|
|
948
|
+
args: [
|
|
949
|
+
...(offline ? ['-o'] : []),
|
|
950
|
+
'-B',
|
|
951
|
+
...(coverage ? ['org.jacoco:jacoco-maven-plugin:prepare-agent'] : []),
|
|
952
|
+
'test',
|
|
953
|
+
...(coverage ? ['org.jacoco:jacoco-maven-plugin:report'] : []),
|
|
954
|
+
`-Dtest=${byClass.map(([name, laws]) => `${name}#${[...new Set(laws.map(e => (model(e) ? e.name : `${e.name}_*`)))].join('+')}`).join(',')}`,
|
|
955
|
+
],
|
|
956
|
+
report: {kind: 'junit', directory: 'target/surefire-reports'},
|
|
957
|
+
ran: test =>
|
|
958
|
+
entries.filter(
|
|
959
|
+
e =>
|
|
960
|
+
test.classname ===
|
|
961
|
+
(e.scope ?? className(e.file, 'src/test/java', '.java')) &&
|
|
962
|
+
named(e).test(test.name),
|
|
963
|
+
),
|
|
964
|
+
},
|
|
965
|
+
];
|
|
401
966
|
}
|
|
402
967
|
// Kotest names tests by string, which Gradle's method filters cannot
|
|
403
968
|
// select. Kotest's own filter selects them within each class: one pattern,
|
|
404
969
|
// matched against whole names, whose * becomes .*? (several comma-separated
|
|
405
970
|
// patterns must all match, so alternatives go in one group).
|
|
406
|
-
if (language ===
|
|
407
|
-
return groupBy(entries,
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
`(${laws.map(
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
971
|
+
if (language === 'kotlin')
|
|
972
|
+
return groupBy(entries, e => e.file).map(([file, laws]) =>
|
|
973
|
+
{
|
|
974
|
+
const filter = laws.some(model)
|
|
975
|
+
? `(${[...new Set(laws.map(e => regex(e.name) + (model(e) ? '' : '_.*')))].join('|')})`
|
|
976
|
+
: `(${laws.map(e => `${e.name}_`).join('|')})*`;
|
|
977
|
+
const name = className(file, 'src/test/kotlin', '.kt');
|
|
978
|
+
return {
|
|
979
|
+
laws,
|
|
980
|
+
command: target.gradle || 'gradle',
|
|
981
|
+
args: [
|
|
982
|
+
...(offline ? ['--offline'] : []),
|
|
983
|
+
'--console=plain',
|
|
984
|
+
'test',
|
|
985
|
+
...(coverage ? ['koverXmlReport'] : []),
|
|
986
|
+
'--rerun',
|
|
987
|
+
'--tests',
|
|
988
|
+
name,
|
|
989
|
+
],
|
|
990
|
+
env: {'kotest.filter.tests': filter, kotest_filter_tests: filter},
|
|
991
|
+
report: {kind: 'junit', directory: 'build/test-results/test'},
|
|
992
|
+
ran: test =>
|
|
993
|
+
laws.filter(
|
|
994
|
+
e =>
|
|
995
|
+
test.classname === (e.scope ?? name) && named(e).test(test.name),
|
|
996
|
+
),
|
|
997
|
+
};
|
|
417
998
|
});
|
|
418
999
|
// ScalaTest selects tests by substring (-z): each law's tests are named
|
|
419
1000
|
// after it, then a kind after a separator no name contains. Its JUnit
|
|
420
1001
|
// reporter writes one file per suite (-u).
|
|
421
|
-
if (language ===
|
|
422
|
-
return groupBy(entries,
|
|
423
|
-
|
|
1002
|
+
if (language === 'scala')
|
|
1003
|
+
return groupBy(entries, e => e.file).map(([file, laws], n) =>
|
|
1004
|
+
{
|
|
1005
|
+
const name = className(file, 'src/test/scala', '.scala');
|
|
424
1006
|
const directory = path.join(scratch, `scalatest-${n}`);
|
|
425
1007
|
// --coverage: Scala 3 instruments the code (scoverage's format).
|
|
426
1008
|
// The tests themselves are not measured.
|
|
427
|
-
const testRoot = target.testDir ||
|
|
428
|
-
const measured = coverage
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
.
|
|
434
|
-
|
|
435
|
-
|
|
1009
|
+
const testRoot = target.testDir || 'src/test/scala';
|
|
1010
|
+
const measured = coverage
|
|
1011
|
+
? [
|
|
1012
|
+
'-O',
|
|
1013
|
+
`-coverage-out:${path.resolve(coverage, 'data')}`,
|
|
1014
|
+
'-O',
|
|
1015
|
+
`-coverage-exclude-files:.*/${testRoot.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/.*`,
|
|
1016
|
+
]
|
|
1017
|
+
: [];
|
|
1018
|
+
return {
|
|
1019
|
+
laws,
|
|
1020
|
+
command: target.scala || 'scala',
|
|
1021
|
+
args: [
|
|
1022
|
+
'test',
|
|
1023
|
+
'.',
|
|
1024
|
+
...(offline ? ['--power', '--offline'] : []),
|
|
1025
|
+
...measured,
|
|
1026
|
+
'--test-only',
|
|
1027
|
+
name,
|
|
1028
|
+
'--',
|
|
1029
|
+
...[
|
|
1030
|
+
...new Map(
|
|
1031
|
+
laws.map(e => [(model(e) ? 'exact:' : 'prefix:') + e.name, e]),
|
|
1032
|
+
).values(),
|
|
1033
|
+
].flatMap(e => [
|
|
1034
|
+
model(e) ? '-t' : '-z',
|
|
1035
|
+
model(e) ? e.name : `${e.name}_`,
|
|
1036
|
+
]),
|
|
1037
|
+
'-u',
|
|
1038
|
+
directory,
|
|
1039
|
+
],
|
|
1040
|
+
report: {
|
|
1041
|
+
kind: 'junit',
|
|
1042
|
+
directory: path.relative(root, directory) || '.',
|
|
1043
|
+
},
|
|
1044
|
+
ran: test =>
|
|
1045
|
+
laws.filter(
|
|
1046
|
+
e =>
|
|
1047
|
+
test.classname === (e.scope ?? name) && named(e).test(test.name),
|
|
1048
|
+
),
|
|
1049
|
+
};
|
|
436
1050
|
});
|
|
437
1051
|
// A Clojure test is the law's words, a double dash and the kind of test
|
|
438
1052
|
// (LawSpec.TestNames); the runner selects tests by namespace and prefix.
|
|
439
|
-
if (language ===
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
1053
|
+
if (language === 'clojure')
|
|
1054
|
+
{
|
|
1055
|
+
const ns = e =>
|
|
1056
|
+
e.scope ??
|
|
1057
|
+
without(relativeTo(e.file, 'test'), '.clj')
|
|
1058
|
+
.replaceAll('/', '.')
|
|
1059
|
+
.replaceAll('_', '-');
|
|
1060
|
+
return [
|
|
1061
|
+
{
|
|
1062
|
+
laws: entries,
|
|
1063
|
+
command: target.clojure || 'clojure',
|
|
1064
|
+
args: [
|
|
1065
|
+
'-M:test',
|
|
1066
|
+
...[
|
|
1067
|
+
...new Map(
|
|
1068
|
+
entries.map(e => [
|
|
1069
|
+
(model(e) ? 'exact:' : 'prefix:') + ns(e) + '/' + e.name,
|
|
1070
|
+
e,
|
|
1071
|
+
]),
|
|
1072
|
+
).values(),
|
|
1073
|
+
].flatMap(e => [
|
|
1074
|
+
model(e) ? '--select-exact' : '--select',
|
|
1075
|
+
`${ns(e)}/${e.name}${model(e) ? '' : '--'}`,
|
|
1076
|
+
]),
|
|
1077
|
+
],
|
|
1078
|
+
report: {kind: 'junit', directory: 'target/test-results'},
|
|
1079
|
+
ran: test =>
|
|
1080
|
+
entries.filter(
|
|
1081
|
+
e =>
|
|
1082
|
+
test.classname === ns(e) &&
|
|
1083
|
+
(model(e)
|
|
1084
|
+
? test.name === e.name
|
|
1085
|
+
: test.name.startsWith(`${e.name}--`)),
|
|
1086
|
+
),
|
|
1087
|
+
},
|
|
1088
|
+
];
|
|
447
1089
|
}
|
|
448
1090
|
// A Spock feature is the law's words, a colon and the kind of test
|
|
449
1091
|
// (LawSpec.TestNames), so "name: *" selects one law's features only.
|
|
450
|
-
if (language ===
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
1092
|
+
if (language === 'groovy')
|
|
1093
|
+
{
|
|
1094
|
+
const name = e => className(e.file, 'src/test/groovy', '.groovy');
|
|
1095
|
+
return [
|
|
1096
|
+
{
|
|
1097
|
+
laws: entries,
|
|
1098
|
+
command: target.gradle || 'gradle',
|
|
1099
|
+
env: {LAWSPEC_CASE: ''},
|
|
1100
|
+
args: [
|
|
1101
|
+
...(offline ? ['--offline'] : []),
|
|
1102
|
+
'--console=plain',
|
|
1103
|
+
'test',
|
|
1104
|
+
...(coverage ? ['jacocoTestReport'] : []),
|
|
1105
|
+
'--rerun',
|
|
1106
|
+
...[
|
|
1107
|
+
...new Set(
|
|
1108
|
+
entries.map(
|
|
1109
|
+
e => `${e.scope ?? name(e)}.${e.name}${model(e) ? '' : ': *'}`,
|
|
1110
|
+
),
|
|
1111
|
+
),
|
|
1112
|
+
].flatMap(name => ['--tests', name]),
|
|
1113
|
+
],
|
|
1114
|
+
report: {kind: 'junit', directory: 'build/test-results/test'},
|
|
1115
|
+
ran: test =>
|
|
1116
|
+
entries.filter(
|
|
1117
|
+
e =>
|
|
1118
|
+
test.classname === (e.scope ?? name(e)) &&
|
|
1119
|
+
(model(e)
|
|
1120
|
+
? test.name === e.name
|
|
1121
|
+
: test.name.startsWith(`${e.name}: `)),
|
|
1122
|
+
),
|
|
1123
|
+
},
|
|
1124
|
+
];
|
|
458
1125
|
}
|
|
459
1126
|
// A law's tests are a module of its name (LawSpec.TestNames), selected
|
|
460
1127
|
// by the module's path; a model's test is selected by its exact name.
|
|
461
|
-
if (language ===
|
|
462
|
-
return groupBy(entries,
|
|
463
|
-
[entries.filter(
|
|
464
|
-
laws
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
1128
|
+
if (language === 'rust')
|
|
1129
|
+
return groupBy(entries, e => e.file).flatMap(([file, entries]) =>
|
|
1130
|
+
[entries.filter(e => !model(e)), entries.filter(model)]
|
|
1131
|
+
.filter(laws => laws.length)
|
|
1132
|
+
.map(laws => ({
|
|
1133
|
+
laws,
|
|
1134
|
+
command: 'cargo',
|
|
1135
|
+
args: [
|
|
1136
|
+
...(coverage ? ['llvm-cov', '--no-report'] : []),
|
|
1137
|
+
'test',
|
|
1138
|
+
...(offline ? ['--offline'] : []),
|
|
1139
|
+
'--test',
|
|
1140
|
+
path.posix.basename(file, '.rs'),
|
|
1141
|
+
'--',
|
|
1142
|
+
...(model(laws[0])
|
|
1143
|
+
? ['--exact', ...new Set(laws.map(e => e.name))]
|
|
1144
|
+
: [...new Set(laws.map(e => `${e.name}::`))]),
|
|
1145
|
+
],
|
|
1146
|
+
report: {
|
|
1147
|
+
kind: 'lines',
|
|
1148
|
+
pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/,
|
|
1149
|
+
},
|
|
1150
|
+
ran: test =>
|
|
1151
|
+
laws.filter(e =>
|
|
1152
|
+
model(e)
|
|
1153
|
+
? test.name === e.name
|
|
1154
|
+
: test.name.startsWith(`${e.name}::`),
|
|
1155
|
+
),
|
|
1156
|
+
})),
|
|
1157
|
+
);
|
|
469
1158
|
// xUnit v3 on Microsoft.Testing.Platform selects test methods with its
|
|
470
1159
|
// query filter (assembly/namespace/class/method, one per law, OR'd), and
|
|
471
1160
|
// reports each test by its display name: the law's label, then its kind.
|
|
472
|
-
if (language ===
|
|
473
|
-
|
|
474
|
-
const
|
|
475
|
-
const
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
1161
|
+
if (language === 'fsharp' || language === 'csharp')
|
|
1162
|
+
{
|
|
1163
|
+
const report = 'lawspec-junit.xml';
|
|
1164
|
+
const results = path.resolve(scratch, 'dotnet-results');
|
|
1165
|
+
const kinds =
|
|
1166
|
+
'example \\d+: .*|boundary(?: \\d+(?: — .*)?(?:\\(caseIndex: \\d+\\))?)?|statistics|property|search|replay|skipped|known discrepancy';
|
|
1167
|
+
const pattern = e => new RegExp(`^${regex(e.label)}( (${kinds}))?$`);
|
|
1168
|
+
const measured = coverage
|
|
1169
|
+
? path.resolve(coverage, 'dotnet.cobertura.xml')
|
|
1170
|
+
: null;
|
|
1171
|
+
return [
|
|
1172
|
+
{
|
|
1173
|
+
laws: entries,
|
|
1174
|
+
command: 'dotnet',
|
|
1175
|
+
args: [
|
|
1176
|
+
'test',
|
|
1177
|
+
...entries.flatMap(e => [
|
|
1178
|
+
'--filter-query',
|
|
1179
|
+
`/*/*/*/${e.name}_*`,
|
|
1180
|
+
'--filter-query',
|
|
1181
|
+
`/*/*/*/${e.name}`,
|
|
1182
|
+
]),
|
|
1183
|
+
'--report-xunit-junit',
|
|
1184
|
+
'--report-xunit-junit-filename',
|
|
1185
|
+
report,
|
|
1186
|
+
'--results-directory',
|
|
1187
|
+
results,
|
|
1188
|
+
...(measured
|
|
1189
|
+
? [
|
|
1190
|
+
'--coverage',
|
|
1191
|
+
'--coverage-output-format',
|
|
1192
|
+
'cobertura',
|
|
1193
|
+
'--coverage-output',
|
|
1194
|
+
measured,
|
|
1195
|
+
]
|
|
1196
|
+
: []),
|
|
1197
|
+
],
|
|
1198
|
+
report: {kind: 'junit', files: [path.join(results, report)]},
|
|
1199
|
+
ran: test => entries.filter(e => pattern(e).test(test.name)),
|
|
1200
|
+
},
|
|
1201
|
+
];
|
|
484
1202
|
}
|
|
485
1203
|
// Ordered Swift groups expose their parameterized aggregate in JUnit;
|
|
486
1204
|
// the native lifecycle report identifies each completed invocation inside it.
|
|
487
|
-
if (language ===
|
|
1205
|
+
if (language === 'swift')
|
|
1206
|
+
{
|
|
488
1207
|
const results = path.resolve(scratch);
|
|
489
|
-
const suite =
|
|
490
|
-
const ordered = entries.filter(
|
|
1208
|
+
const suite = e => path.posix.basename(e.file, '.swift');
|
|
1209
|
+
const ordered = entries.filter(e => e.scope === 'ordered');
|
|
491
1210
|
const token = randomUUID();
|
|
492
|
-
const output = ordered.length ? `swift-lawspec-${token}` :
|
|
1211
|
+
const output = ordered.length ? `swift-lawspec-${token}` : 'swift-lawspec';
|
|
493
1212
|
const files = [path.join(results, `${output}-swift-testing.xml`)];
|
|
494
1213
|
const events = path.join(results, `${output}.jsonl`);
|
|
495
|
-
const test =
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
1214
|
+
const test = e =>
|
|
1215
|
+
e.scope === 'ordered'
|
|
1216
|
+
? `${regex(suite(e))}/ordered\\(`
|
|
1217
|
+
: `${regex(suite(e))}/${regex(e.name)}(_[A-Za-z0-9]+)?\\(`;
|
|
1218
|
+
return [
|
|
1219
|
+
{
|
|
1220
|
+
laws: entries,
|
|
1221
|
+
command: target.swift || 'swift',
|
|
1222
|
+
args: [
|
|
1223
|
+
'test',
|
|
1224
|
+
...(coverage ? ['--enable-code-coverage'] : []),
|
|
1225
|
+
...[...new Set(entries.map(test))].flatMap(selector => [
|
|
1226
|
+
'--filter',
|
|
1227
|
+
selector,
|
|
1228
|
+
]),
|
|
1229
|
+
'--xunit-output',
|
|
1230
|
+
path.join(results, `${output}.xml`),
|
|
1231
|
+
],
|
|
1232
|
+
env: {
|
|
1233
|
+
LAWSPEC_SWIFT_LAWS: JSON.stringify(
|
|
1234
|
+
ordered.map(({unit, name}) => ({unit, name})),
|
|
1235
|
+
),
|
|
1236
|
+
LAWSPEC_SWIFT_REPORT: ordered.length ? events : '',
|
|
1237
|
+
LAWSPEC_SWIFT_RUN: ordered.length ? token : '',
|
|
1238
|
+
},
|
|
1239
|
+
report: ordered.length
|
|
1240
|
+
? {
|
|
1241
|
+
kind: 'swift-events',
|
|
1242
|
+
files,
|
|
1243
|
+
file: events,
|
|
1244
|
+
run: token,
|
|
1245
|
+
groups: ordered.map(e => ({
|
|
1246
|
+
unit: e.unit,
|
|
1247
|
+
law: e.name,
|
|
1248
|
+
suite: suite(e),
|
|
1249
|
+
minimum:
|
|
1250
|
+
typeof e.knownDiscrepancy === 'string'
|
|
1251
|
+
? 1
|
|
1252
|
+
: Math.max(1, e.coverage?.cases ?? 1),
|
|
1253
|
+
})),
|
|
1254
|
+
}
|
|
1255
|
+
: {kind: 'junit', files},
|
|
1256
|
+
ran: result =>
|
|
1257
|
+
entries.filter(e =>
|
|
1258
|
+
e.scope === 'ordered'
|
|
1259
|
+
? result.unit === e.unit && result.nativeName === e.name
|
|
1260
|
+
: result.classname?.split('.').pop() === suite(e) &&
|
|
1261
|
+
new RegExp(`^${regex(e.name)}(_[A-Za-z0-9]+)?(\\(|$)`).test(
|
|
1262
|
+
result.name,
|
|
1263
|
+
),
|
|
1264
|
+
),
|
|
1265
|
+
},
|
|
1266
|
+
];
|
|
511
1267
|
}
|
|
512
|
-
if (language ===
|
|
513
|
-
|
|
514
|
-
const
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
1268
|
+
if (language === 'haskell')
|
|
1269
|
+
{
|
|
1270
|
+
const module = e =>
|
|
1271
|
+
without(relativeTo(e.file, 'test'), 'Spec.hs').replaceAll('/', '.');
|
|
1272
|
+
const matches = [
|
|
1273
|
+
...new Set(
|
|
1274
|
+
entries.map(e =>
|
|
1275
|
+
model(e)
|
|
1276
|
+
? `/${e.scope ?? module(e)}/${e.unit} models/${e.name}/`
|
|
1277
|
+
: `${module(e)}/${e.name}_`,
|
|
1278
|
+
),
|
|
1279
|
+
),
|
|
1280
|
+
];
|
|
1281
|
+
return [
|
|
1282
|
+
{
|
|
1283
|
+
laws: entries,
|
|
1284
|
+
command: 'stack',
|
|
1285
|
+
args: [
|
|
1286
|
+
'--no-terminal',
|
|
1287
|
+
'test',
|
|
1288
|
+
...(coverage ? ['--coverage'] : []),
|
|
1289
|
+
'--test-arguments',
|
|
1290
|
+
matches
|
|
1291
|
+
.map(m => `--match ${m.includes(' ') ? JSON.stringify(m) : m}`)
|
|
1292
|
+
.join(' '),
|
|
1293
|
+
],
|
|
1294
|
+
report: {kind: 'hspec'},
|
|
1295
|
+
ran: test =>
|
|
1296
|
+
entries.filter(
|
|
1297
|
+
e =>
|
|
1298
|
+
test.classname === (e.scope ?? module(e)) &&
|
|
1299
|
+
named(e).test(test.name),
|
|
1300
|
+
),
|
|
1301
|
+
},
|
|
1302
|
+
];
|
|
521
1303
|
}
|
|
522
1304
|
throw new Error(`lawspec test does not support ${language}`);
|
|
523
1305
|
}
|
|
524
1306
|
|
|
525
1307
|
// What --coverage needs on each target, and how to say it is missing.
|
|
526
1308
|
export const coverageTools = {
|
|
527
|
-
erlang: {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
1309
|
+
erlang: {
|
|
1310
|
+
tool: 'OTP cover',
|
|
1311
|
+
install: 'install Erlang/OTP with the tools application',
|
|
1312
|
+
},
|
|
1313
|
+
elixir: {
|
|
1314
|
+
tool: 'OTP cover',
|
|
1315
|
+
install: 'install Erlang/OTP with the tools application',
|
|
1316
|
+
},
|
|
1317
|
+
gleam: {
|
|
1318
|
+
tool: 'OTP cover',
|
|
1319
|
+
install: 'install Erlang/OTP with the tools application',
|
|
1320
|
+
},
|
|
1321
|
+
python: {
|
|
1322
|
+
tool: 'coverage.py',
|
|
1323
|
+
check: ['-c', 'import coverage'],
|
|
1324
|
+
install: 'python -m pip install coverage',
|
|
1325
|
+
},
|
|
1326
|
+
javascript: {tool: 'c8', install: 'npm install --save-dev c8'},
|
|
1327
|
+
typescript: {tool: 'c8', install: 'npm install --save-dev c8'},
|
|
1328
|
+
go: {tool: 'go test -cover', install: '(built in)'},
|
|
1329
|
+
java: {
|
|
1330
|
+
tool: 'JaCoCo',
|
|
1331
|
+
install: '(resolved by Maven as org.jacoco:jacoco-maven-plugin)',
|
|
1332
|
+
},
|
|
1333
|
+
kotlin: {
|
|
1334
|
+
tool: 'Kover',
|
|
1335
|
+
install:
|
|
1336
|
+
'add id("org.jetbrains.kotlinx.kover") to the plugins of build.gradle.kts',
|
|
1337
|
+
},
|
|
1338
|
+
clojure: {
|
|
1339
|
+
tool: 'cloverage',
|
|
1340
|
+
install: 'add cloverage to a :coverage alias in deps.edn',
|
|
1341
|
+
},
|
|
1342
|
+
groovy: {
|
|
1343
|
+
tool: 'JaCoCo',
|
|
1344
|
+
install: "add id 'jacoco' to the plugins of build.gradle",
|
|
1345
|
+
},
|
|
1346
|
+
rust: {tool: 'cargo-llvm-cov', install: 'cargo install cargo-llvm-cov'},
|
|
1347
|
+
haskell: {tool: 'hpc', install: '(built into GHC; stack test --coverage)'},
|
|
1348
|
+
fsharp: {
|
|
1349
|
+
tool: 'Microsoft code coverage',
|
|
1350
|
+
install:
|
|
1351
|
+
'add the Microsoft.Testing.Extensions.CodeCoverage package to the test project',
|
|
1352
|
+
},
|
|
1353
|
+
csharp: {
|
|
1354
|
+
tool: 'Microsoft code coverage',
|
|
1355
|
+
install:
|
|
1356
|
+
'add the Microsoft.Testing.Extensions.CodeCoverage package to the test project',
|
|
1357
|
+
},
|
|
1358
|
+
scala: {
|
|
1359
|
+
tool: 'scoverage',
|
|
1360
|
+
install: '(built into Scala 3; Scala CLI fetches the scoverage reporter)',
|
|
1361
|
+
},
|
|
1362
|
+
swift: {
|
|
1363
|
+
tool: 'llvm-cov',
|
|
1364
|
+
install:
|
|
1365
|
+
'(built into the Swift toolchain; swift test --enable-code-coverage)',
|
|
1366
|
+
},
|
|
544
1367
|
};
|
|
545
1368
|
|
|
546
1369
|
// Laws selected by tag: every --tag, and none of --exclude-tag.
|
|
547
|
-
export function selectByTags(entries, include = [], exclude = [])
|
|
548
|
-
|
|
549
|
-
|
|
1370
|
+
export function selectByTags(entries, include = [], exclude = [])
|
|
1371
|
+
{
|
|
1372
|
+
return entries.filter(
|
|
1373
|
+
e =>
|
|
1374
|
+
(!include.length || include.some(t => (e.tags ?? []).includes(t))) &&
|
|
1375
|
+
!exclude.some(t => (e.tags ?? []).includes(t)),
|
|
1376
|
+
);
|
|
550
1377
|
}
|
|
551
1378
|
|
|
552
1379
|
// A recorded failure takes precedence over any older passing cache entry,
|
|
553
1380
|
// including when an adapter is repaired by restoring its previous contents.
|
|
554
|
-
export function testsToRun(
|
|
555
|
-
|
|
556
|
-
|
|
1381
|
+
export function testsToRun(
|
|
1382
|
+
entries,
|
|
1383
|
+
keys,
|
|
1384
|
+
previous,
|
|
1385
|
+
failures,
|
|
1386
|
+
{fresh = false, coverage = false, updateRecorded = false} = {},
|
|
1387
|
+
)
|
|
1388
|
+
{
|
|
1389
|
+
return entries.filter(
|
|
1390
|
+
entry =>
|
|
1391
|
+
fresh ||
|
|
1392
|
+
coverage ||
|
|
1393
|
+
updateRecorded ||
|
|
1394
|
+
failures[entry.law] ||
|
|
1395
|
+
!previousResult(previous, entry.law, keys.get(entry.law)),
|
|
1396
|
+
);
|
|
557
1397
|
}
|
|
558
1398
|
|
|
559
1399
|
// Read complete outer elements, preserving nested suites and native payloads.
|
|
560
1400
|
// Comments and CDATA can contain apparent XML tags; they are never elements.
|
|
561
|
-
function junitElements(xml, names)
|
|
1401
|
+
function junitElements(xml, names)
|
|
1402
|
+
{
|
|
562
1403
|
const elements = [];
|
|
563
1404
|
let current;
|
|
564
|
-
const tags =
|
|
565
|
-
|
|
1405
|
+
const tags =
|
|
1406
|
+
/<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z_][\w:.-]*)(?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
|
|
1407
|
+
for (const tag of xml.matchAll(tags))
|
|
1408
|
+
{
|
|
566
1409
|
const [, closing, name] = tag;
|
|
567
1410
|
if (!name || !names.includes(name)) continue;
|
|
568
1411
|
const empty = /\/\s*>$/.test(tag[0]);
|
|
569
|
-
if (!current)
|
|
1412
|
+
if (!current)
|
|
1413
|
+
{
|
|
570
1414
|
if (closing) continue;
|
|
571
1415
|
current = {name, start: tag.index, opening: tag[0], depth: 0};
|
|
572
1416
|
}
|
|
573
1417
|
if (name !== current.name) continue;
|
|
574
1418
|
current.depth += closing ? -1 : empty ? 0 : 1;
|
|
575
|
-
if (current.depth === 0)
|
|
576
|
-
|
|
1419
|
+
if (current.depth === 0)
|
|
1420
|
+
{
|
|
1421
|
+
elements.push({
|
|
1422
|
+
...current,
|
|
1423
|
+
xml: xml.slice(current.start, tag.index + tag[0].length),
|
|
1424
|
+
});
|
|
577
1425
|
current = undefined;
|
|
578
1426
|
}
|
|
579
1427
|
}
|
|
580
1428
|
return elements;
|
|
581
1429
|
}
|
|
582
1430
|
|
|
583
|
-
function junitCounts(cases)
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
1431
|
+
function junitCounts(cases)
|
|
1432
|
+
{
|
|
1433
|
+
const statuses = cases.map(
|
|
1434
|
+
test =>
|
|
1435
|
+
new Set(
|
|
1436
|
+
junitElements(test.xml, ['failure', 'error', 'skipped']).map(
|
|
1437
|
+
e => e.name,
|
|
1438
|
+
),
|
|
1439
|
+
),
|
|
1440
|
+
);
|
|
1441
|
+
return {
|
|
1442
|
+
tests: cases.length,
|
|
1443
|
+
...Object.fromEntries(
|
|
1444
|
+
['failure', 'error', 'skipped'].map(name => [
|
|
1445
|
+
name === 'skipped' ? name : `${name}s`,
|
|
1446
|
+
statuses.filter(status => status.has(name)).length,
|
|
1447
|
+
]),
|
|
1448
|
+
),
|
|
1449
|
+
};
|
|
587
1450
|
}
|
|
588
1451
|
|
|
589
1452
|
// One JUnit report from every target's: each target's suites, renamed with
|
|
590
1453
|
// the target, in one <testsuites>.
|
|
591
|
-
export function mergeJunit(reports)
|
|
1454
|
+
export function mergeJunit(reports)
|
|
1455
|
+
{
|
|
592
1456
|
const suites = [];
|
|
593
1457
|
const totals = {tests: 0, failures: 0, errors: 0, skipped: 0};
|
|
594
|
-
for (const {
|
|
595
|
-
|
|
596
|
-
const
|
|
1458
|
+
for (const {target, xml} of reports)
|
|
1459
|
+
{
|
|
1460
|
+
const elements = junitElements(xml, ['testsuite', 'testcase']);
|
|
1461
|
+
const found = elements.filter(e => e.name === 'testsuite');
|
|
597
1462
|
// Node's native JUnit reporter puts top-level tests directly in testsuites.
|
|
598
1463
|
// Keep those cases (and their native failure payloads) instead of silently
|
|
599
1464
|
// dropping them because they have no intervening testsuite element.
|
|
600
|
-
const loose = elements.filter(e => e.name ===
|
|
601
|
-
if (loose.length)
|
|
602
|
-
|
|
603
|
-
|
|
1465
|
+
const loose = elements.filter(e => e.name === 'testcase');
|
|
1466
|
+
if (loose.length)
|
|
1467
|
+
{
|
|
1468
|
+
const opening = `<testsuite name="native" ${Object.entries(
|
|
1469
|
+
junitCounts(loose),
|
|
1470
|
+
)
|
|
1471
|
+
.map(([key, value]) => `${key}="${value}"`)
|
|
1472
|
+
.join(' ')}>`;
|
|
1473
|
+
found.push({
|
|
1474
|
+
opening,
|
|
1475
|
+
xml:
|
|
1476
|
+
opening + '\n' + loose.map(e => e.xml).join('\n') + '\n</testsuite>',
|
|
1477
|
+
});
|
|
604
1478
|
}
|
|
605
|
-
for (const suite of found)
|
|
606
|
-
|
|
607
|
-
const
|
|
608
|
-
|
|
1479
|
+
for (const suite of found)
|
|
1480
|
+
{
|
|
1481
|
+
const attributes = [
|
|
1482
|
+
...suite.opening.matchAll(
|
|
1483
|
+
/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g,
|
|
1484
|
+
),
|
|
1485
|
+
];
|
|
1486
|
+
const values = Object.fromEntries(
|
|
1487
|
+
attributes.map(match => [match[1], match[2] ?? match[3]]),
|
|
1488
|
+
);
|
|
1489
|
+
const counts = junitCounts(junitElements(suite.xml, ['testcase']));
|
|
609
1490
|
for (const key of Object.keys(totals))
|
|
610
|
-
totals[key] += /^\d+$/.test(values[key] ??
|
|
611
|
-
|
|
1491
|
+
totals[key] += /^\d+$/.test(values[key] ?? '')
|
|
1492
|
+
? Number(values[key])
|
|
1493
|
+
: counts[key];
|
|
1494
|
+
const name = attributes.find(match => match[1] === 'name');
|
|
612
1495
|
const opening = name
|
|
613
|
-
? suite.opening.slice(0, name.index) +
|
|
614
|
-
|
|
1496
|
+
? suite.opening.slice(0, name.index) +
|
|
1497
|
+
` name="${escapeXml(target)}: ${values.name.replaceAll('"', '"')}"` +
|
|
1498
|
+
suite.opening.slice(name.index + name[0].length)
|
|
1499
|
+
: suite.opening.replace(
|
|
1500
|
+
/^<testsuite\b/,
|
|
1501
|
+
`<testsuite name="${escapeXml(target)}"`,
|
|
1502
|
+
);
|
|
615
1503
|
suites.push(opening + suite.xml.slice(suite.opening.length));
|
|
616
1504
|
}
|
|
617
1505
|
}
|
|
618
|
-
return
|
|
619
|
-
|
|
1506
|
+
return (
|
|
1507
|
+
`<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${totals.tests}" failures="${totals.failures}" errors="${totals.errors}" skipped="${totals.skipped}">\n` +
|
|
1508
|
+
suites.join('\n') +
|
|
1509
|
+
'\n</testsuites>\n'
|
|
1510
|
+
);
|
|
620
1511
|
}
|
|
621
1512
|
|
|
622
|
-
const escapeXml =
|
|
1513
|
+
const escapeXml = text =>
|
|
1514
|
+
text
|
|
1515
|
+
.replaceAll('&', '&')
|
|
1516
|
+
.replaceAll('<', '<')
|
|
1517
|
+
.replaceAll('>', '>')
|
|
1518
|
+
.replaceAll('"', '"');
|
|
623
1519
|
|
|
624
1520
|
// The harness statistics a run wrote (LAWSPEC_STATS): cover results, label
|
|
625
1521
|
// counts, flaky and known-discrepancy outcomes, benchmarks.
|
|
626
|
-
export async function harnessStatistics(directory)
|
|
1522
|
+
export async function harnessStatistics(directory)
|
|
1523
|
+
{
|
|
627
1524
|
const names = await readdir(directory).catch(() => []);
|
|
628
1525
|
const entries = [];
|
|
629
|
-
for (const name of names.filter(
|
|
630
|
-
try
|
|
1526
|
+
for (const name of names.filter(n => n.endsWith('.json')).sort())
|
|
1527
|
+
try
|
|
1528
|
+
{
|
|
1529
|
+
entries.push(
|
|
1530
|
+
JSON.parse(await readFile(path.join(directory, name), 'utf8')),
|
|
1531
|
+
);
|
|
1532
|
+
}
|
|
1533
|
+
catch
|
|
1534
|
+
{
|
|
1535
|
+
/* partial */
|
|
1536
|
+
}
|
|
631
1537
|
return entries;
|
|
632
1538
|
}
|
|
633
1539
|
|
|
634
1540
|
// The tests a run executed, as {name, classname}, from the runner's report or
|
|
635
1541
|
// output. Skipped tests are not counted.
|
|
636
|
-
export async function executedTests(report, output, root, since)
|
|
637
|
-
|
|
638
|
-
if(report?.kind==='
|
|
639
|
-
|
|
1542
|
+
export async function executedTests(report, output, root, since)
|
|
1543
|
+
{
|
|
1544
|
+
if (report?.kind === 'swift-events')
|
|
1545
|
+
return swiftExecutedTests(report, output, root, since);
|
|
1546
|
+
if (report?.kind === 'node-events')
|
|
1547
|
+
{
|
|
1548
|
+
const raw = await readFile(path.resolve(root, report.file), 'utf8').catch(
|
|
1549
|
+
() => '',
|
|
1550
|
+
);
|
|
640
1551
|
let rows;
|
|
641
|
-
try
|
|
642
|
-
|
|
643
|
-
rows
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
1552
|
+
try
|
|
1553
|
+
{
|
|
1554
|
+
rows = raw
|
|
1555
|
+
.trim()
|
|
1556
|
+
.split('\n')
|
|
1557
|
+
.map(line => JSON.parse(line));
|
|
1558
|
+
}
|
|
1559
|
+
catch
|
|
1560
|
+
{
|
|
1561
|
+
return [];
|
|
1562
|
+
}
|
|
1563
|
+
if (
|
|
1564
|
+
rows.length < 2 ||
|
|
1565
|
+
rows[0]?.event !== 'start' ||
|
|
1566
|
+
rows.at(-1)?.event !== 'end' ||
|
|
1567
|
+
rows.some(row => row?.version !== 1 || row?.run !== report.run) ||
|
|
1568
|
+
rows.at(-1).count !== rows.length - 2
|
|
1569
|
+
)
|
|
1570
|
+
return [];
|
|
1571
|
+
const tests = rows.slice(1, -1);
|
|
1572
|
+
if (
|
|
1573
|
+
tests.some(
|
|
1574
|
+
row =>
|
|
1575
|
+
row?.event !== 'test' ||
|
|
1576
|
+
typeof row.name !== 'string' ||
|
|
1577
|
+
typeof row.classname !== 'string' ||
|
|
1578
|
+
!['passed', 'failed', 'skipped'].includes(row.status) ||
|
|
1579
|
+
(row.time !== undefined &&
|
|
1580
|
+
(!Number.isFinite(row.time) || row.time < 0)) ||
|
|
1581
|
+
(row.failure !== undefined && typeof row.failure !== 'string'),
|
|
1582
|
+
)
|
|
1583
|
+
)
|
|
1584
|
+
return [];
|
|
650
1585
|
return tests;
|
|
651
1586
|
}
|
|
652
|
-
if (report?.kind ===
|
|
1587
|
+
if (report?.kind === 'beam-events')
|
|
1588
|
+
{
|
|
653
1589
|
// A native listener seals the invocation after its final event. Missing,
|
|
654
1590
|
// stale, malformed or interrupted reports are not execution evidence.
|
|
655
|
-
const raw = await readFile(path.resolve(root, report.file),
|
|
1591
|
+
const raw = await readFile(path.resolve(root, report.file), 'utf8').catch(
|
|
1592
|
+
() => '',
|
|
1593
|
+
);
|
|
656
1594
|
let rows;
|
|
657
|
-
try
|
|
658
|
-
|
|
659
|
-
rows
|
|
1595
|
+
try
|
|
1596
|
+
{
|
|
1597
|
+
rows = raw
|
|
1598
|
+
.trim()
|
|
1599
|
+
.split('\n')
|
|
1600
|
+
.map(line => JSON.parse(line));
|
|
1601
|
+
}
|
|
1602
|
+
catch
|
|
1603
|
+
{
|
|
1604
|
+
return [];
|
|
1605
|
+
}
|
|
1606
|
+
if (
|
|
1607
|
+
rows.length < 2 ||
|
|
1608
|
+
rows[0]?.event !== 'start' ||
|
|
1609
|
+
rows.at(-1)?.event !== 'end' ||
|
|
1610
|
+
rows.some(row => row?.run !== report.run)
|
|
1611
|
+
)
|
|
1612
|
+
return [];
|
|
660
1613
|
const tests = rows.slice(1, -1);
|
|
661
|
-
if (
|
|
662
|
-
|
|
663
|
-
|
|
1614
|
+
if (
|
|
1615
|
+
tests.some(
|
|
1616
|
+
row =>
|
|
1617
|
+
row.event !== 'test' ||
|
|
1618
|
+
!['passed', 'failed'].includes(row.status) ||
|
|
1619
|
+
typeof row.name !== 'string' ||
|
|
1620
|
+
typeof row.classname !== 'string' ||
|
|
1621
|
+
typeof row.identity !== 'string' ||
|
|
1622
|
+
!Number.isFinite(row.time) ||
|
|
1623
|
+
row.time < 0,
|
|
1624
|
+
)
|
|
1625
|
+
)
|
|
1626
|
+
return [];
|
|
664
1627
|
return tests;
|
|
665
1628
|
}
|
|
666
|
-
if (report?.kind ===
|
|
667
|
-
|
|
1629
|
+
if (report?.kind === 'junit')
|
|
1630
|
+
{
|
|
1631
|
+
const files =
|
|
1632
|
+
report.files ??
|
|
1633
|
+
(await reportFiles(path.join(root, report.directory), since));
|
|
668
1634
|
const tests = [];
|
|
669
|
-
for (const file of files)
|
|
670
|
-
|
|
671
|
-
const
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
const
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
1635
|
+
for (const file of files)
|
|
1636
|
+
{
|
|
1637
|
+
const location = path.resolve(root, file);
|
|
1638
|
+
const info = await stat(location).catch(() => null);
|
|
1639
|
+
if (!info || info.mtimeMs < since) continue;
|
|
1640
|
+
const xml = await readFile(location, 'utf8').catch(() => '');
|
|
1641
|
+
const attributes = opening =>
|
|
1642
|
+
Object.fromEntries(
|
|
1643
|
+
[
|
|
1644
|
+
...opening.matchAll(
|
|
1645
|
+
/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g,
|
|
1646
|
+
),
|
|
1647
|
+
].map(match => [match[1], unescapeXml(match[2] ?? match[3])]),
|
|
1648
|
+
);
|
|
1649
|
+
for (const element of junitElements(xml, ['testcase']))
|
|
1650
|
+
{
|
|
1651
|
+
const values = attributes(element.opening);
|
|
1652
|
+
const native = junitElements(element.xml, [
|
|
1653
|
+
'failure',
|
|
1654
|
+
'error',
|
|
1655
|
+
'skipped',
|
|
1656
|
+
]);
|
|
1657
|
+
const failure = native.filter(element => element.name !== 'skipped');
|
|
1658
|
+
const incomplete = junitElements(element.xml, ['property']).some(
|
|
1659
|
+
property =>
|
|
1660
|
+
{
|
|
1661
|
+
const value = attributes(property.opening);
|
|
1662
|
+
return (
|
|
1663
|
+
value.name === 'lawspec.status' && value.value === 'incomplete'
|
|
1664
|
+
);
|
|
1665
|
+
},
|
|
1666
|
+
);
|
|
1667
|
+
const status = incomplete
|
|
1668
|
+
? 'incomplete'
|
|
1669
|
+
: failure.length
|
|
1670
|
+
? 'failed'
|
|
1671
|
+
: native.length
|
|
1672
|
+
? 'skipped'
|
|
1673
|
+
: 'passed';
|
|
1674
|
+
const detail = element =>
|
|
1675
|
+
[
|
|
1676
|
+
attributes(element.opening).message,
|
|
1677
|
+
unescapeXml(
|
|
1678
|
+
element.xml
|
|
1679
|
+
.slice(element.opening.length)
|
|
1680
|
+
.replace(/<\/(failure|error|skipped)>\s*$/, '')
|
|
1681
|
+
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, text) => text),
|
|
1682
|
+
),
|
|
1683
|
+
]
|
|
1684
|
+
.filter(Boolean)
|
|
1685
|
+
.join('\n');
|
|
1686
|
+
const time =
|
|
1687
|
+
values.time === undefined ? undefined : Number(values.time);
|
|
1688
|
+
tests.push({
|
|
1689
|
+
name: (values.name ?? '').replace(/\(\)$/, ''),
|
|
1690
|
+
classname: values.classname ?? '',
|
|
1691
|
+
status,
|
|
1692
|
+
...(Number.isFinite(time) && time >= 0 ? {time} : {}),
|
|
1693
|
+
...(failure.length
|
|
1694
|
+
? {failure: failure.map(detail).join('\n')}
|
|
1695
|
+
: status === 'skipped'
|
|
1696
|
+
? {reason: native.map(detail).join('\n')}
|
|
1697
|
+
: {}),
|
|
683
1698
|
});
|
|
684
|
-
const status=incomplete?"incomplete":failure.length?"failed":native.length?"skipped":"passed";
|
|
685
|
-
const detail=element=>[attributes(element.opening).message,
|
|
686
|
-
unescapeXml(element.xml.slice(element.opening.length).replace(/<\/(failure|error|skipped)>\s*$/,"")
|
|
687
|
-
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g,(_,text)=>text))].filter(Boolean).join("\n");
|
|
688
|
-
const time=values.time===undefined?undefined:Number(values.time);
|
|
689
|
-
tests.push({name:(values.name??"").replace(/\(\)$/, ""),classname:values.classname??"",status,
|
|
690
|
-
...(Number.isFinite(time)&&time>=0?{time}:{}),
|
|
691
|
-
...(failure.length?{failure:failure.map(detail).join("\n")}:
|
|
692
|
-
status==="skipped"?{reason:native.map(detail).join("\n")}: {})});
|
|
693
1699
|
}
|
|
694
1700
|
}
|
|
695
1701
|
return tests;
|
|
696
1702
|
}
|
|
697
|
-
if (report?.kind ===
|
|
698
|
-
return output.split(
|
|
699
|
-
|
|
1703
|
+
if (report?.kind === 'go-json')
|
|
1704
|
+
return output.split('\n').flatMap(line =>
|
|
1705
|
+
{
|
|
1706
|
+
try
|
|
1707
|
+
{
|
|
700
1708
|
const event = JSON.parse(line);
|
|
701
|
-
return [
|
|
702
|
-
? [
|
|
703
|
-
|
|
1709
|
+
return ['pass', 'fail'].includes(event.Action) && event.Test
|
|
1710
|
+
? [
|
|
1711
|
+
{
|
|
1712
|
+
name: event.Test,
|
|
1713
|
+
classname: event.Package,
|
|
1714
|
+
status: event.Action === 'pass' ? 'passed' : 'failed',
|
|
1715
|
+
...(Number.isFinite(event.Elapsed) && event.Elapsed >= 0
|
|
1716
|
+
? {time: event.Elapsed}
|
|
1717
|
+
: {}),
|
|
1718
|
+
},
|
|
1719
|
+
]
|
|
1720
|
+
: [];
|
|
1721
|
+
}
|
|
1722
|
+
catch
|
|
1723
|
+
{
|
|
1724
|
+
return [];
|
|
1725
|
+
}
|
|
704
1726
|
});
|
|
705
|
-
if (report?.kind ===
|
|
706
|
-
return output.split(
|
|
1727
|
+
if (report?.kind === 'lines')
|
|
1728
|
+
return output.split('\n').flatMap(line =>
|
|
1729
|
+
{
|
|
707
1730
|
const match = line.match(report.pattern);
|
|
708
|
-
return match && match[2] !==
|
|
1731
|
+
return match && match[2] !== 'ignored'
|
|
1732
|
+
? [
|
|
1733
|
+
{
|
|
1734
|
+
name: match[1],
|
|
1735
|
+
classname: '',
|
|
1736
|
+
status: match[2] === 'ok' ? 'passed' : 'failed',
|
|
1737
|
+
},
|
|
1738
|
+
]
|
|
1739
|
+
: [];
|
|
709
1740
|
});
|
|
710
|
-
if (report?.kind ===
|
|
1741
|
+
if (report?.kind === 'hspec')
|
|
1742
|
+
{
|
|
711
1743
|
// hspec prints each module, then its examples indented beneath it.
|
|
712
1744
|
const tests = [];
|
|
713
|
-
let module =
|
|
714
|
-
for (const line of output.split(
|
|
715
|
-
|
|
716
|
-
|
|
1745
|
+
let module = '';
|
|
1746
|
+
for (const line of output.split('\n'))
|
|
1747
|
+
{
|
|
1748
|
+
if (
|
|
1749
|
+
/^\S/.test(line) &&
|
|
1750
|
+
!/^(Finished|Failures|Randomized|\d+ examples?)/.test(line)
|
|
1751
|
+
)
|
|
1752
|
+
module = line.trim();
|
|
1753
|
+
const example =
|
|
1754
|
+
line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/) ??
|
|
717
1755
|
line.match(/^\s+(model .+?|supervision)\s\[([✔✘])\]\s*$/);
|
|
718
|
-
if (example)
|
|
1756
|
+
if (example)
|
|
1757
|
+
tests.push({
|
|
1758
|
+
name: example[1],
|
|
1759
|
+
classname: module,
|
|
1760
|
+
status: example[2] === '✔' ? 'passed' : 'failed',
|
|
1761
|
+
});
|
|
719
1762
|
}
|
|
720
1763
|
return tests;
|
|
721
1764
|
}
|
|
@@ -724,104 +1767,211 @@ export async function executedTests(report, output, root, since) {
|
|
|
724
1767
|
|
|
725
1768
|
// Swift's XML seals the native aggregate; runtime events refine that completed
|
|
726
1769
|
// aggregate into individually attributable cases. Neither channel suffices alone.
|
|
727
|
-
async function swiftExecutedTests(report, output, root, since)
|
|
728
|
-
|
|
729
|
-
const
|
|
730
|
-
|
|
1770
|
+
async function swiftExecutedTests(report, output, root, since)
|
|
1771
|
+
{
|
|
1772
|
+
const native = await executedTests(
|
|
1773
|
+
{kind: 'junit', files: report.files},
|
|
1774
|
+
output,
|
|
1775
|
+
root,
|
|
1776
|
+
since,
|
|
1777
|
+
);
|
|
1778
|
+
const ordinary = native.filter(test => test.name !== 'ordered(_:)');
|
|
1779
|
+
const raw = await readFile(path.resolve(root, report.file), 'utf8').catch(
|
|
1780
|
+
() => '',
|
|
1781
|
+
);
|
|
731
1782
|
let rows;
|
|
732
|
-
try
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
1783
|
+
try
|
|
1784
|
+
{
|
|
1785
|
+
rows = raw
|
|
1786
|
+
.trim()
|
|
1787
|
+
.split('\n')
|
|
1788
|
+
.map(line => JSON.parse(line));
|
|
1789
|
+
}
|
|
1790
|
+
catch
|
|
1791
|
+
{
|
|
1792
|
+
return ordinary;
|
|
1793
|
+
}
|
|
1794
|
+
if (
|
|
1795
|
+
!rows.length ||
|
|
1796
|
+
rows.some(row => !row || row.version !== 1 || row.run !== report.run)
|
|
1797
|
+
)
|
|
1798
|
+
return ordinary;
|
|
1799
|
+
const key = (unit, name) => JSON.stringify([unit, name]);
|
|
1800
|
+
const expected = new Map(
|
|
1801
|
+
report.groups.map(group => [key(group.unit, group.law), group]),
|
|
1802
|
+
);
|
|
1803
|
+
const plans = new Map(),
|
|
1804
|
+
cases = new Map(),
|
|
1805
|
+
starts = new Set(),
|
|
1806
|
+
ends = new Map();
|
|
1807
|
+
for (const row of rows)
|
|
1808
|
+
{
|
|
1809
|
+
if (typeof row.unit !== 'string') return ordinary;
|
|
1810
|
+
if (row.event === 'plan')
|
|
1811
|
+
{
|
|
740
1812
|
if (!Array.isArray(row.cases) || !row.cases.length) return ordinary;
|
|
741
1813
|
const previous = plans.get(row.unit);
|
|
742
|
-
if (previous !== undefined)
|
|
743
|
-
|
|
1814
|
+
if (previous !== undefined)
|
|
1815
|
+
{
|
|
1816
|
+
if (JSON.stringify(previous) !== JSON.stringify(row.cases))
|
|
1817
|
+
return ordinary;
|
|
744
1818
|
continue;
|
|
745
1819
|
}
|
|
746
|
-
plans.set(row.unit,row.cases);
|
|
747
|
-
for (const item of row.cases)
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
1820
|
+
plans.set(row.unit, row.cases);
|
|
1821
|
+
for (const item of row.cases)
|
|
1822
|
+
{
|
|
1823
|
+
if (
|
|
1824
|
+
!item ||
|
|
1825
|
+
typeof item.id !== 'string' ||
|
|
1826
|
+
!item.id ||
|
|
1827
|
+
typeof item.name !== 'string' ||
|
|
1828
|
+
!item.name ||
|
|
1829
|
+
typeof item.law !== 'string' ||
|
|
1830
|
+
!expected.has(key(row.unit, item.law))
|
|
1831
|
+
)
|
|
1832
|
+
return ordinary;
|
|
1833
|
+
const id = key(row.unit, item.id);
|
|
751
1834
|
if (cases.has(id)) return ordinary;
|
|
752
|
-
cases.set(id,{...item,unit:row.unit});
|
|
1835
|
+
cases.set(id, {...item, unit: row.unit});
|
|
753
1836
|
}
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
1837
|
+
}
|
|
1838
|
+
else if (row.event === 'case-start' || row.event === 'case-end')
|
|
1839
|
+
{
|
|
1840
|
+
const id = key(row.unit, row.id),
|
|
1841
|
+
item = cases.get(id);
|
|
1842
|
+
if (!item || item.name !== row.name || item.law !== row.law)
|
|
1843
|
+
return ordinary;
|
|
1844
|
+
if (row.event === 'case-start')
|
|
1845
|
+
{
|
|
758
1846
|
if (starts.has(id)) return ordinary;
|
|
759
1847
|
starts.add(id);
|
|
760
|
-
} else {
|
|
761
|
-
if (!starts.has(id) || ends.has(id) || !["passed","failed"].includes(row.status) ||
|
|
762
|
-
!Number.isFinite(row.time) || row.time < 0 ||
|
|
763
|
-
(row.failure !== undefined && typeof row.failure !== "string")) return ordinary;
|
|
764
|
-
ends.set(id,row);
|
|
765
1848
|
}
|
|
766
|
-
|
|
1849
|
+
else
|
|
1850
|
+
{
|
|
1851
|
+
if (
|
|
1852
|
+
!starts.has(id) ||
|
|
1853
|
+
ends.has(id) ||
|
|
1854
|
+
!['passed', 'failed'].includes(row.status) ||
|
|
1855
|
+
!Number.isFinite(row.time) ||
|
|
1856
|
+
row.time < 0 ||
|
|
1857
|
+
(row.failure !== undefined && typeof row.failure !== 'string')
|
|
1858
|
+
)
|
|
1859
|
+
return ordinary;
|
|
1860
|
+
ends.set(id, row);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
else return ordinary;
|
|
767
1864
|
}
|
|
768
1865
|
if (cases.size !== starts.size || cases.size !== ends.size) return ordinary;
|
|
769
1866
|
const completed = [];
|
|
770
|
-
for (const group of expected.values())
|
|
771
|
-
|
|
1867
|
+
for (const group of expected.values())
|
|
1868
|
+
{
|
|
1869
|
+
const members = [...cases.values()].filter(
|
|
1870
|
+
item => item.unit === group.unit && item.law === group.law,
|
|
1871
|
+
);
|
|
772
1872
|
if (members.length < group.minimum) return ordinary;
|
|
773
|
-
const aggregates = native.filter(
|
|
774
|
-
test
|
|
775
|
-
|
|
1873
|
+
const aggregates = native.filter(
|
|
1874
|
+
test =>
|
|
1875
|
+
test.name === 'ordered(_:)' &&
|
|
1876
|
+
test.classname.split('.').pop() === group.suite,
|
|
1877
|
+
);
|
|
1878
|
+
if (
|
|
1879
|
+
aggregates.length !== 1 ||
|
|
1880
|
+
!['passed', 'failed'].includes(aggregates[0].status)
|
|
1881
|
+
)
|
|
1882
|
+
return ordinary;
|
|
776
1883
|
const aggregate = aggregates[0];
|
|
777
1884
|
// A framework failure outside the invocation body cannot credit that body's
|
|
778
1885
|
// earlier successes. Preserve them as incomplete observations instead.
|
|
779
|
-
const suiteFailed = [...cases.values()].some(
|
|
780
|
-
|
|
781
|
-
|
|
1886
|
+
const suiteFailed = [...cases.values()].some(item =>
|
|
1887
|
+
{
|
|
1888
|
+
const expectedGroup = expected.get(key(item.unit, item.law));
|
|
1889
|
+
return (
|
|
1890
|
+
expectedGroup.suite === group.suite &&
|
|
1891
|
+
ends.get(key(item.unit, item.id)).status === 'failed'
|
|
1892
|
+
);
|
|
782
1893
|
});
|
|
783
|
-
if (aggregate.status ===
|
|
784
|
-
for (const item of members)
|
|
785
|
-
|
|
786
|
-
const
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
1894
|
+
if (aggregate.status === 'passed' && suiteFailed) return ordinary;
|
|
1895
|
+
for (const item of members)
|
|
1896
|
+
{
|
|
1897
|
+
const result = ends.get(key(item.unit, item.id));
|
|
1898
|
+
const incomplete = aggregate.status === 'failed' && !suiteFailed;
|
|
1899
|
+
completed.push({
|
|
1900
|
+
name: item.id,
|
|
1901
|
+
classname: aggregate.classname,
|
|
1902
|
+
identity: `${item.unit}::${item.law}`,
|
|
1903
|
+
unit: item.unit,
|
|
1904
|
+
nativeName: item.law,
|
|
1905
|
+
status: incomplete ? 'incomplete' : result.status,
|
|
1906
|
+
time: result.time,
|
|
1907
|
+
...(incomplete
|
|
1908
|
+
? {
|
|
1909
|
+
failure:
|
|
1910
|
+
aggregate.failure ??
|
|
1911
|
+
'Swift aggregate failed outside its invocation bodies',
|
|
1912
|
+
}
|
|
1913
|
+
: result.failure === undefined
|
|
1914
|
+
? {}
|
|
1915
|
+
: {failure: result.failure}),
|
|
1916
|
+
});
|
|
792
1917
|
}
|
|
793
1918
|
}
|
|
794
|
-
return [...ordinary
|
|
1919
|
+
return [...ordinary, ...completed];
|
|
795
1920
|
}
|
|
796
1921
|
|
|
797
1922
|
// JUnit XML for a runner that writes none, from the tests it printed.
|
|
798
|
-
export function junitFromTests(tests)
|
|
799
|
-
|
|
800
|
-
const
|
|
801
|
-
const
|
|
1923
|
+
export function junitFromTests(tests)
|
|
1924
|
+
{
|
|
1925
|
+
const failures = tests.filter(t => t.status === 'failed').length;
|
|
1926
|
+
const errors = tests.filter(t => t.status === 'incomplete').length;
|
|
1927
|
+
const skipped = tests.filter(t => t.status === 'skipped').length;
|
|
802
1928
|
// A BEAM scheduler wraps native functions from several units. Its module
|
|
803
1929
|
// is not the declaring class: retain the unit from the validated identity.
|
|
804
|
-
const classname =
|
|
805
|
-
|
|
806
|
-
|
|
1930
|
+
const classname = test =>
|
|
1931
|
+
{
|
|
1932
|
+
const separator =
|
|
1933
|
+
typeof test.identity === 'string' ? test.identity.lastIndexOf('::') : -1;
|
|
1934
|
+
return separator > 0
|
|
1935
|
+
? test.identity.slice(0, separator)
|
|
1936
|
+
: (test.classname ?? '');
|
|
807
1937
|
};
|
|
808
|
-
return
|
|
809
|
-
tests.
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
t
|
|
813
|
-
|
|
1938
|
+
return (
|
|
1939
|
+
`<testsuite name="lawspec" tests="${tests.length}" failures="${failures}"${errors ? ` errors="${errors}"` : ''}${skipped ? ` skipped="${skipped}"` : ''}>\n` +
|
|
1940
|
+
tests
|
|
1941
|
+
.map(
|
|
1942
|
+
t =>
|
|
1943
|
+
` <testcase classname="${escapeXml(classname(t))}" name="${escapeXml(t.name)}"` +
|
|
1944
|
+
(Number.isFinite(t.time) ? ` time="${t.time}"` : '') +
|
|
1945
|
+
(t.status === 'failed'
|
|
1946
|
+
? `><failure message="${escapeXml(t.failure ?? 'failed')}"/></testcase>`
|
|
1947
|
+
: t.status === 'incomplete'
|
|
1948
|
+
? `><properties><property name="lawspec.status" value="incomplete"/></properties><error message="${escapeXml(t.failure ?? t.reason ?? 'incomplete execution')}"/></testcase>`
|
|
1949
|
+
: t.status === 'skipped'
|
|
1950
|
+
? `><skipped message="${escapeXml(t.reason ?? 'skipped')}"/></testcase>`
|
|
1951
|
+
: '/>'),
|
|
1952
|
+
)
|
|
1953
|
+
.join('\n') +
|
|
1954
|
+
'\n</testsuite>'
|
|
1955
|
+
);
|
|
814
1956
|
}
|
|
815
1957
|
|
|
816
|
-
async function reportFiles(directory, since)
|
|
1958
|
+
async function reportFiles(directory, since)
|
|
1959
|
+
{
|
|
817
1960
|
const names = await readdir(directory).catch(() => []);
|
|
818
1961
|
const files = [];
|
|
819
|
-
for (const name of names.filter(
|
|
1962
|
+
for (const name of names.filter(n => n.endsWith('.xml')))
|
|
1963
|
+
{
|
|
820
1964
|
const info = await stat(path.join(directory, name));
|
|
821
1965
|
if (info.mtimeMs >= since) files.push(path.join(directory, name));
|
|
822
1966
|
}
|
|
823
1967
|
return files;
|
|
824
1968
|
}
|
|
825
1969
|
|
|
826
|
-
const unescapeXml =
|
|
827
|
-
|
|
1970
|
+
const unescapeXml = text =>
|
|
1971
|
+
text
|
|
1972
|
+
.replaceAll('<', '<')
|
|
1973
|
+
.replaceAll('>', '>')
|
|
1974
|
+
.replaceAll('"', '"')
|
|
1975
|
+
.replaceAll(''', "'")
|
|
1976
|
+
.replaceAll(''', "'")
|
|
1977
|
+
.replaceAll('&', '&');
|