lawspec 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +109 -1
- package/README.md +34 -26
- package/UNICODE-LICENSE.txt +39 -0
- package/api.mjs +6 -23
- package/bin/lawspec.mjs +148 -65
- package/build.json +245 -177
- package/builtin-data-names.mjs +73 -0
- package/core.wasm +2 -2
- package/core_jsffi.js +5 -0
- package/examples/native-payments/clojure/lawspec.json +13 -2
- package/examples/native-payments/csharp/lawspec.json +13 -2
- package/examples/native-payments/elixir/lawspec.json +14 -3
- package/examples/native-payments/erlang/lawspec.json +14 -3
- package/examples/native-payments/fsharp/lawspec.json +14 -3
- package/examples/native-payments/gleam/lawspec.json +14 -3
- package/examples/native-payments/go/lawspec.json +14 -3
- package/examples/native-payments/groovy/lawspec.json +14 -3
- package/examples/native-payments/haskell/lawspec.json +14 -3
- package/examples/native-payments/java/lawspec.json +14 -3
- package/examples/native-payments/javascript/lawspec.json +14 -3
- package/examples/native-payments/kotlin/lawspec.json +14 -3
- package/examples/native-payments/python/lawspec.json +14 -3
- package/examples/native-payments/rust/lawspec.json +14 -3
- package/examples/native-payments/scala/lawspec.json +14 -3
- package/examples/native-payments/swift/lawspec.json +14 -3
- package/examples/native-payments/typescript/lawspec.json +14 -3
- package/examples/packages/orders/lawspec.json +25 -5
- package/examples/packages/orders/orders.lawspec +57 -53
- package/examples/packages/shop-domain/src/domain.lawspec +36 -33
- package/examples/packages/shop-tax-1/src/api.lawspec +1 -0
- package/examples/packages/shop-tax-1/src/rates.lawspec +5 -4
- package/examples/packages/shop-tax-2/src/api.lawspec +1 -0
- package/examples/packages/shop-tax-2/src/rates.lawspec +6 -4
- package/examples/rc1/authorization-snapshots.lawspec +113 -0
- package/examples/rc1/banner-filenames.lawspec +64 -0
- package/examples/rc1/bit-codec.lawspec +57 -0
- package/examples/rc1/bounded-witness-search.lawspec +151 -0
- package/examples/rc1/conflicting-requirements.lawspec +31 -0
- package/examples/rc1/constructive-witnesses.lawspec +120 -0
- package/examples/rc1/documentary-review.lawspec +43 -0
- package/examples/rc1/empty-domains.lawspec +47 -0
- package/examples/rc1/finite-integer-domains.lawspec +76 -0
- package/examples/rc1/law-metadata.lawspec +30 -0
- package/examples/rc1/native-payments.lawspec +108 -0
- package/examples/rc1/nesting/rrr.lawspec +34 -0
- package/examples/rc1/nesting/rru.lawspec +57 -0
- package/examples/rc1/nesting/rur.lawspec +58 -0
- package/examples/rc1/nesting/ruu.lawspec +78 -0
- package/examples/rc1/nesting/urr.lawspec +53 -0
- package/examples/rc1/nesting/uru.lawspec +73 -0
- package/examples/rc1/nesting/uur.lawspec +74 -0
- package/examples/rc1/nesting/uuu.lawspec +96 -0
- package/examples/rc1/order-workflow.lawspec +127 -0
- package/examples/rc1/parcel-measurements.lawspec +93 -0
- package/examples/rc1/picture-records.lawspec +52 -0
- package/examples/rc1/picture-tuples.lawspec +48 -0
- package/examples/rc1/record-payloads.lawspec +57 -0
- package/examples/rc1/recursive-records.lawspec +65 -0
- package/examples/rc1/refund-conformance.lawspec +45 -0
- package/examples/rc1/refund-policy.lawspec +89 -0
- package/examples/rc1/requirement-composition.lawspec +90 -0
- package/examples/rc1/structural-records.lawspec +54 -0
- package/examples/rc1-authoring.lawspec +57 -0
- package/examples/specs/abilities.lawspec +67 -65
- package/examples/specs/ability_imports.lawspec +58 -57
- package/examples/specs/actors.lawspec +23 -22
- package/examples/specs/algebra.lawspec +142 -141
- package/examples/specs/approvals.lawspec +16 -15
- package/examples/specs/async_bindings.lawspec +10 -9
- package/examples/specs/async_fetch.lawspec +12 -11
- package/examples/specs/atoi_codec.lawspec +16 -15
- package/examples/specs/boolean_flags.lawspec +10 -9
- package/examples/specs/builtins.lawspec +73 -72
- package/examples/specs/canonical_url.lawspec +14 -13
- package/examples/specs/collections.lawspec +153 -154
- package/examples/specs/composition.lawspec +70 -66
- package/examples/specs/concurrent_collections.lawspec +8 -7
- package/examples/specs/consistency.lawspec +4 -3
- package/examples/specs/contracts.lawspec +80 -79
- package/examples/specs/crypto.lawspec +26 -25
- package/examples/specs/ctc.lawspec +148 -0
- package/examples/specs/currying.lawspec +32 -31
- package/examples/specs/data_types.lawspec +50 -49
- package/examples/specs/distribution.lawspec +57 -56
- package/examples/specs/domain_modeling.lawspec +48 -47
- package/examples/specs/durations.lawspec +39 -38
- package/examples/specs/equivalent.lawspec +25 -24
- package/examples/specs/failures.lawspec +64 -63
- package/examples/specs/finite_data.lawspec +32 -31
- package/examples/specs/flow_stack.lawspec +64 -63
- package/examples/specs/gadt_expressions.lawspec +50 -49
- package/examples/specs/generation.lawspec +115 -114
- package/examples/specs/handler_bindings.lawspec +42 -40
- package/examples/specs/handles.lawspec +2 -1
- package/examples/specs/harness.lawspec +45 -43
- package/examples/specs/harness_testing.lawspec +18 -13
- package/examples/specs/indexed_arithmetic.lawspec +401 -126
- package/examples/specs/indexed_families.lawspec +182 -66
- package/examples/specs/keywords.lawspec +22 -21
- package/examples/specs/limits.lawspec +6 -5
- package/examples/specs/list_contracts.lawspec +121 -120
- package/examples/specs/list_refinements.lawspec +43 -42
- package/examples/specs/lists.lawspec +45 -44
- package/examples/specs/matchers.lawspec +79 -78
- package/examples/specs/matching.lawspec +43 -42
- package/examples/specs/mixed_inputs.lawspec +25 -24
- package/examples/specs/models.lawspec +44 -43
- package/examples/specs/parse_port.lawspec +28 -27
- package/examples/specs/payments.lawspec +55 -54
- package/examples/specs/railway.lawspec +59 -58
- package/examples/specs/records.lawspec +149 -148
- package/examples/specs/records_inner.lawspec +5 -4
- package/examples/specs/recursive_refinements.lawspec +30 -29
- package/examples/specs/refined_definitions.lawspec +24 -23
- package/examples/specs/refinements.lawspec +58 -57
- package/examples/specs/resilience.lawspec +159 -158
- package/examples/specs/resources.lawspec +61 -60
- package/examples/specs/scalar_adapters.lawspec +60 -59
- package/examples/specs/scalar_catalog.lawspec +165 -164
- package/examples/specs/scalars.lawspec +78 -77
- package/examples/specs/scheduling_order.lawspec +8 -7
- package/examples/specs/scheduling_overlap.lawspec +5 -4
- package/examples/specs/sequences.lawspec +66 -65
- package/examples/specs/sessions.lawspec +13 -12
- package/examples/specs/slug.lawspec +16 -15
- package/examples/specs/sum_refinements.lawspec +42 -41
- package/examples/specs/tables.lawspec +29 -28
- package/examples/specs/total_functions.lawspec +71 -70
- package/examples/specs/workflows.lawspec +101 -100
- package/examples-command.mjs +8 -4
- package/explain.mjs +114 -0
- package/expression-presentation.mjs +333 -0
- package/files.mjs +16 -0
- package/index.d.ts +406 -11
- package/launcher.mjs +9 -2
- package/lsp.mjs +235 -0
- package/native-evidence.mjs +187 -0
- package/native-examples.mjs +8 -1
- package/node-events-reporter.mjs +23 -0
- package/package.json +8 -1
- package/positional-fields.mjs +13 -0
- package/proof.d.ts +35 -0
- package/proof.mjs +174 -0
- package/starter-rc1.lawspec +26 -0
- package/starter.lawspec +16 -15
- package/templates.mjs +6 -4
- package/test-command.mjs +107 -35
- package/typed-presentation.mjs +165 -0
package/proof.mjs
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// Generated from templates/npm/proof.mjs by lawspec-dev generate. Do not edit.
|
|
2
|
+
// Host execution of compiler-produced tasks. The browser/WASM compiler stays
|
|
3
|
+
// pure; this Node host invokes Z3's SMT-LIB interface without a shell.
|
|
4
|
+
import {spawn} from 'node:child_process';
|
|
5
|
+
import {createHash} from 'node:crypto';
|
|
6
|
+
import {access,readFile,realpath} from 'node:fs/promises';
|
|
7
|
+
import {constants} from 'node:fs';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
|
|
10
|
+
const hash=value=>createHash('sha256').update(value).digest('hex');
|
|
11
|
+
const frame=value=>Buffer.byteLength(value,'utf8')+':'+value;
|
|
12
|
+
const fingerprint=values=>hash(values.map(frame).join(''));
|
|
13
|
+
const encoding='lawspec-smtlib-v1';
|
|
14
|
+
|
|
15
|
+
export function proofTaskFingerprint(task) {
|
|
16
|
+
return fingerprint([task.encoding,task.programFingerprint,String(task.machineBits),
|
|
17
|
+
task.owner,task.declaration,task.script]);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function optionsWithDefaults(options) {
|
|
21
|
+
const config={binary:options.binary??'z3',timeoutMs:options.timeoutMs??5000,
|
|
22
|
+
memoryMb:options.memoryMb??512,resourceLimit:options.resourceLimit??0,
|
|
23
|
+
seed:options.seed??0,maxOutputBytes:options.maxOutputBytes??1048576};
|
|
24
|
+
for(const [key,maximum] of [['timeoutMs',600000],['memoryMb',65536],
|
|
25
|
+
['resourceLimit',2147483647],['seed',2147483647],['maxOutputBytes',16777216]]) {
|
|
26
|
+
const minimum=['resourceLimit','seed'].includes(key)?0:1;
|
|
27
|
+
if(!Number.isSafeInteger(config[key])||config[key]<minimum||config[key]>maximum)
|
|
28
|
+
throw new TypeError(`Invalid Z3 ${key}`);
|
|
29
|
+
}
|
|
30
|
+
if(typeof config.binary!=='string'||!config.binary||config.binary.includes('\0'))
|
|
31
|
+
throw new TypeError('Invalid Z3 executable');
|
|
32
|
+
return config;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function executable(binary) {
|
|
36
|
+
const candidates=path.isAbsolute(binary)||binary.includes('/')||binary.includes('\\')
|
|
37
|
+
?[path.resolve(binary)]
|
|
38
|
+
:(process.env.PATH??'').split(path.delimiter).map(folder=>path.resolve(folder,binary));
|
|
39
|
+
for(const candidate of candidates) {
|
|
40
|
+
try {await access(candidate,constants.X_OK);return await realpath(candidate);}
|
|
41
|
+
catch(error) {if(!['ENOENT','EACCES','ENOTDIR'].includes(error.code))throw error;}
|
|
42
|
+
}
|
|
43
|
+
throw Object.assign(new Error(`Z3 executable is unavailable: ${binary}`),{code:'ENOENT'});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Only the first response line is a protocol token. Models and unknown reasons
|
|
47
|
+
// remain raw prover output; this host does not implement another SMT parser.
|
|
48
|
+
function execute(binary,args,initial,config,interactive=false) {
|
|
49
|
+
return new Promise(resolve=>{
|
|
50
|
+
const started=performance.now();
|
|
51
|
+
const child=spawn(binary,args,{stdio:['pipe','pipe','pipe'],shell:false});
|
|
52
|
+
child.stdout.setEncoding('utf8');child.stderr.setEncoding('utf8');
|
|
53
|
+
let stdout='',stderr='',input=initial,bytes=0,outcome=null,error=null;
|
|
54
|
+
let timeout=false,outputLimit=false,answered=false,closed=false;
|
|
55
|
+
const timer=setTimeout(()=>{timeout=true;child.kill('SIGKILL');},config.timeoutMs);
|
|
56
|
+
const append=(which,chunk)=>{
|
|
57
|
+
bytes+=Buffer.byteLength(chunk,'utf8');
|
|
58
|
+
if(bytes>config.maxOutputBytes){outputLimit=true;child.kill('SIGKILL');return;}
|
|
59
|
+
if(which==='stdout')stdout+=chunk;else stderr+=chunk;
|
|
60
|
+
if(interactive&&!answered&&stdout.includes('\n')) {
|
|
61
|
+
answered=true;
|
|
62
|
+
const token=stdout.slice(0,stdout.indexOf('\n')).trim();
|
|
63
|
+
if(['sat','unsat','unknown'].includes(token))outcome=token;
|
|
64
|
+
else error='unexpected Z3 protocol response';
|
|
65
|
+
const finish=token==='sat'?'(get-model)\n(exit)\n':
|
|
66
|
+
token==='unknown'?'(get-info :reason-unknown)\n(exit)\n':'(exit)\n';
|
|
67
|
+
input+=finish;
|
|
68
|
+
child.stdin.end(finish);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
child.stdout.on('data',chunk=>append('stdout',chunk));
|
|
72
|
+
child.stderr.on('data',chunk=>append('stderr',chunk));
|
|
73
|
+
child.stdin.on('error',problem=>{if(problem.code!=='EPIPE')error=problem.message;});
|
|
74
|
+
child.on('error',problem=>{error=problem.message;});
|
|
75
|
+
child.on('close',(exitCode,signal)=>{
|
|
76
|
+
if(closed)return;
|
|
77
|
+
closed=true;clearTimeout(timer);
|
|
78
|
+
resolve({stdout,stderr,input,outcome,error,timeout,outputLimit,exitCode,signal,
|
|
79
|
+
elapsedMs:performance.now()-started});
|
|
80
|
+
});
|
|
81
|
+
if(interactive)child.stdin.write(initial);else child.stdin.end(initial);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function prover(config) {
|
|
86
|
+
const binary=await executable(config.binary);
|
|
87
|
+
const binaryFingerprint=hash(await readFile(binary));
|
|
88
|
+
const versionRun=await execute(binary,['-version'],'',config);
|
|
89
|
+
const version=versionRun.stdout.trim();
|
|
90
|
+
if(versionRun.exitCode!==0||versionRun.error||versionRun.timeout||versionRun.outputLimit||
|
|
91
|
+
!/^Z3 version [0-9]+\.[0-9]+/.test(version))
|
|
92
|
+
throw Object.assign(new Error('The selected executable did not return a successful Z3 version response'),
|
|
93
|
+
{outcome:versionRun.timeout?'timeout':'failed'});
|
|
94
|
+
const args=['-in','-smt2',`-t:${config.timeoutMs}`,`-memory:${config.memoryMb}`];
|
|
95
|
+
const identity=fingerprint([binaryFingerprint,version,JSON.stringify(args),JSON.stringify(config)]);
|
|
96
|
+
return {binary,binaryFingerprint,version,args,fingerprint:identity};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function validTask(task) {
|
|
100
|
+
return task.encoding===encoding&&[32,64].includes(task.machineBits)&&
|
|
101
|
+
['owner','declaration','programFingerprint','script','taskFingerprint'].every(key=>typeof task[key]==='string')&&
|
|
102
|
+
/^[0-9a-f]{64}$/.test(task.programFingerprint)&&
|
|
103
|
+
proofTaskFingerprint(task)===task.taskFingerprint;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function solve(task,selected,config) {
|
|
107
|
+
const common={owner:task.owner,declaration:task.declaration,at:task.at,
|
|
108
|
+
programFingerprint:task.programFingerprint,taskFingerprint:task.taskFingerprint,
|
|
109
|
+
encoding:task.encoding,machineBits:task.machineBits,configuration:config};
|
|
110
|
+
if(task.status==='unsupported')return {...common,status:'unsupported',outcome:'unsupported',reason:task.reason};
|
|
111
|
+
if(task.status!=='ready'||!validTask(task))
|
|
112
|
+
return {...common,status:'incomplete',outcome:'failed',reason:'Compiler proof task binding is invalid'};
|
|
113
|
+
if(selected.error)return {...common,status:'incomplete',outcome:selected.outcome,reason:selected.error};
|
|
114
|
+
const input=`(set-option :print-success false)\n(set-option :produce-models true)\n`+
|
|
115
|
+
`(set-option :random-seed ${config.seed})\n(set-option :rlimit ${config.resourceLimit})\n`+
|
|
116
|
+
task.script+'(check-sat)\n';
|
|
117
|
+
const run=await execute(selected.binary,selected.args,input,config,true);
|
|
118
|
+
let unchanged=false;
|
|
119
|
+
try {unchanged=hash(await readFile(selected.binary))===selected.binaryFingerprint;} catch {}
|
|
120
|
+
const metadata={...common,prover:{name:'z3',version:selected.version,
|
|
121
|
+
binary:selected.binary,binaryFingerprint:selected.binaryFingerprint,
|
|
122
|
+
fingerprint:selected.fingerprint,arguments:selected.args},configuration:config,
|
|
123
|
+
runFingerprint:fingerprint([task.taskFingerprint,selected.fingerprint,run.input]),
|
|
124
|
+
elapsedMs:run.elapsedMs,exitCode:run.exitCode,signal:run.signal,
|
|
125
|
+
stdout:run.stdout,stderr:run.stderr};
|
|
126
|
+
let outcome=run.outcome,reason;
|
|
127
|
+
if(!unchanged){outcome='failed';reason='Z3 executable changed during execution';}
|
|
128
|
+
else if(run.timeout){outcome='timeout';reason='Z3 exceeded the host wall-clock limit';}
|
|
129
|
+
else if(run.outputLimit){outcome='failed';reason='Z3 exceeded the output limit';}
|
|
130
|
+
else if(run.exitCode!==0&&/out of memory|out_of_memory/i.test(run.stdout+'\n'+run.stderr)) {
|
|
131
|
+
outcome='out-of-memory';reason='Z3 reported memory exhaustion';
|
|
132
|
+
} else if(run.error||run.exitCode!==0||!outcome||run.stdout.includes('(error ')) {
|
|
133
|
+
outcome='failed';reason=run.error??'Z3 did not complete the task protocol successfully';
|
|
134
|
+
} else if(outcome==='unsat')reason='Z3 established this law, including required definedness; trusts the typed translation and selected solver, without a kernel-checked certificate';
|
|
135
|
+
else if(outcome==='sat')reason='Z3 returned a satisfying model for the negated proof task; it has not been validated as a LawSpec counterexample';
|
|
136
|
+
else {
|
|
137
|
+
reason='Z3 could not decide this task: '+run.stdout.slice(run.stdout.indexOf('\n')+1).trim();
|
|
138
|
+
if(/:reason-unknown\s+"timeout"/.test(run.stdout))outcome='timeout';
|
|
139
|
+
else if(/:reason-unknown\s+"max\. resource limit exceeded"/.test(run.stdout))outcome='resource-limit';
|
|
140
|
+
}
|
|
141
|
+
return {...metadata,status:outcome==='unsat'?'proved':'incomplete',outcome,reason,
|
|
142
|
+
...(run.outcome==='sat'?{model:run.stdout.slice(run.stdout.indexOf('\n')+1).trim(),modelValidated:false}:{})};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Tasks always come freshly from the compiler for the supplied source request.
|
|
146
|
+
// An imported JSON solver report cannot upgrade a law's evidence through here.
|
|
147
|
+
export async function proveWithZ3(compiler,request,options={}) {
|
|
148
|
+
const config=optionsWithDefaults(options);
|
|
149
|
+
const result=await compiler.proofTasks(request);
|
|
150
|
+
if(result.diagnostics?.length)return {...result,solverEvidence:[]};
|
|
151
|
+
const tasks=result.proofTasks??[];
|
|
152
|
+
let selected={};
|
|
153
|
+
if(tasks.some(task=>task.status==='ready')) {
|
|
154
|
+
try {selected=await prover(config);}
|
|
155
|
+
catch(error){selected={error:error.message,outcome:error.code==='ENOENT'?'unavailable':error.outcome??'failed'};}
|
|
156
|
+
}
|
|
157
|
+
const solverEvidence=[];
|
|
158
|
+
for(const task of tasks)solverEvidence.push(await solve(task,selected,config));
|
|
159
|
+
const diagnostics=[...(result.diagnostics??[])];
|
|
160
|
+
const evidence=(result.evidence??[]).map(row=>{
|
|
161
|
+
const found=solverEvidence.find(item=>item.owner===row.owner&&item.declaration===row.declaration);
|
|
162
|
+
if(row.stage!=='law'||!found)return row;
|
|
163
|
+
if(found.status!=='proved') {
|
|
164
|
+
if(['proved','exhaustively-checked','known-failing','skipped','flaky'].includes(row.status))return row;
|
|
165
|
+
return {...row,status:found.status==='unsupported'?'unsupported':'incomplete',reason:found.reason};
|
|
166
|
+
}
|
|
167
|
+
if(row.status==='known-failing') {
|
|
168
|
+
diagnostics.push({code:'harness',message:`Z3 proved ${row.declaration}; its harness cannot mark it known failing`,at:found.at});
|
|
169
|
+
return row;
|
|
170
|
+
}
|
|
171
|
+
return {...row,status:'proved',reason:found.reason+`; task ${found.taskFingerprint}`};
|
|
172
|
+
});
|
|
173
|
+
return {...result,diagnostics,evidence,solverEvidence};
|
|
174
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Generated from templates/npm/starter-rc1.lawspec by lawspec-dev generate. Do not edit.
|
|
2
|
+
unit example.codec
|
|
3
|
+
|
|
4
|
+
itoa of type Int32 -> Text
|
|
5
|
+
atoi of type Text -> Int32
|
|
6
|
+
|
|
7
|
+
law codec is
|
|
8
|
+
description:
|
|
9
|
+
"Converting an integer to decimal text and reading it back preserves it."
|
|
10
|
+
end description
|
|
11
|
+
|
|
12
|
+
definition:
|
|
13
|
+
for all (x of type Int32) =>
|
|
14
|
+
atoi (itoa x) must be x
|
|
15
|
+
end for all
|
|
16
|
+
end definition
|
|
17
|
+
|
|
18
|
+
expect itoa x = encoded
|
|
19
|
+
expect atoi encoded = x
|
|
20
|
+
in table (x, encoded):
|
|
21
|
+
row -42, "-42"
|
|
22
|
+
row 0, "0"
|
|
23
|
+
row 42, "42"
|
|
24
|
+
end table
|
|
25
|
+
end law
|
|
26
|
+
end unit
|
package/starter.lawspec
CHANGED
|
@@ -3,33 +3,34 @@ unit example.atoi_codec
|
|
|
3
3
|
itoa :: Int32 -> Text
|
|
4
4
|
atoi :: Text -> Int32
|
|
5
5
|
|
|
6
|
-
law
|
|
7
|
-
definition
|
|
8
|
-
|
|
9
|
-
end
|
|
6
|
+
law "|itoa and then atoi yields a| is
|
|
7
|
+
definition :
|
|
8
|
+
apply law "|round trip identity is preserved| itoa atoi
|
|
9
|
+
end definition
|
|
10
10
|
|
|
11
|
-
description
|
|
11
|
+
description :
|
|
12
12
|
"converting an Int32 to Text with {itoa} and then back with {atoi} yields the original Int32"
|
|
13
|
-
end
|
|
13
|
+
end description
|
|
14
14
|
|
|
15
|
-
rationale
|
|
15
|
+
rationale :
|
|
16
16
|
"representing an Int32 as Text must not change its value"
|
|
17
|
-
end
|
|
17
|
+
end rationale
|
|
18
18
|
|
|
19
|
-
example
|
|
19
|
+
example "|negative integers use a minus sign and round-trip unchanged| :
|
|
20
20
|
x = -42
|
|
21
21
|
expect itoa x = "-42"
|
|
22
22
|
expect atoi (itoa x) = -42
|
|
23
|
-
end
|
|
23
|
+
end example
|
|
24
24
|
|
|
25
|
-
example
|
|
25
|
+
example "|zero renders as 0 and round-trips unchanged| :
|
|
26
26
|
x = 0
|
|
27
27
|
expect itoa x = "0"
|
|
28
28
|
expect atoi (itoa x) = 0
|
|
29
|
-
end
|
|
29
|
+
end example
|
|
30
30
|
|
|
31
|
-
references
|
|
31
|
+
references :
|
|
32
32
|
"left inverse"
|
|
33
33
|
"round-trip property"
|
|
34
|
-
end
|
|
35
|
-
end
|
|
34
|
+
end references
|
|
35
|
+
end law
|
|
36
|
+
end unit
|
package/templates.mjs
CHANGED
|
@@ -178,22 +178,24 @@ const scaffolds = {
|
|
|
178
178
|
"haskell": {
|
|
179
179
|
"readable": {
|
|
180
180
|
"stack.yaml": "snapshot: lts-24.58\npackages: [.]\n",
|
|
181
|
-
"package.yaml": "name: lawspec-example\nversion: 0.1.0\ndependencies: [base, text, bytestring, containers, network, directory, time]\nlibrary:\n source-dirs: src\ntests:\n laws:\n main: Spec.hs\n source-dirs: test\n dependencies: [lawspec-example, hspec, hedgehog, hspec-hedgehog, containers, mtl]\n ghc-options: [-threaded, -rtsopts, \"-with-rtsopts=-N\"]\n build-tools: [hspec-discover]\n",
|
|
181
|
+
"package.yaml": "name: lawspec-example\nversion: 0.1.0\ndependencies: [base, text, bytestring, containers, network, directory, time]\nlibrary:\n source-dirs: src\ntests:\n laws:\n main: Spec.hs\n source-dirs: test\n dependencies: [lawspec-example, hspec, hedgehog, hspec-hedgehog, containers, mtl]\n ghc-options: [-O0, -threaded, -rtsopts, \"-with-rtsopts=-N\"]\n build-tools: [hspec-discover]\n",
|
|
182
182
|
"test/Spec.hs": "{-# OPTIONS_GHC -F -pgmF hspec-discover #-}\n"
|
|
183
183
|
},
|
|
184
184
|
"compact": {
|
|
185
185
|
"stack.yaml": "snapshot: lts-24.58\npackages: [.]\n",
|
|
186
|
-
"package.yaml": "name: lawspec-example\nversion: 0.1.0\ndependencies: [base, text, bytestring, containers, network, directory, time]\nlibrary:\n source-dirs: src\ntests:\n laws:\n main: Spec.hs\n source-dirs: test\n dependencies: [lawspec-example, hspec, hedgehog, hspec-hedgehog, containers, mtl]\n ghc-options: [-threaded, -rtsopts, \"-with-rtsopts=-N\"]\n build-tools: [hspec-discover]\n",
|
|
186
|
+
"package.yaml": "name: lawspec-example\nversion: 0.1.0\ndependencies: [base, text, bytestring, containers, network, directory, time]\nlibrary:\n source-dirs: src\ntests:\n laws:\n main: Spec.hs\n source-dirs: test\n dependencies: [lawspec-example, hspec, hedgehog, hspec-hedgehog, containers, mtl]\n ghc-options: [-O0, -threaded, -rtsopts, \"-with-rtsopts=-N\"]\n build-tools: [hspec-discover]\n",
|
|
187
187
|
"test/Spec.hs": "{-# OPTIONS_GHC -F -pgmF hspec-discover #-}\n"
|
|
188
188
|
}
|
|
189
189
|
},
|
|
190
190
|
"kotlin": {
|
|
191
191
|
"readable": {
|
|
192
192
|
"settings.gradle.kts": "rootProject.name = \"lawspec-example\"\n",
|
|
193
|
+
"gradle.properties": "kotlin.daemon.jvmargs=-Xmx2g\n",
|
|
193
194
|
"build.gradle.kts": "plugins {\n kotlin(\"jvm\") version \"2.3.21\"\n}\n\nrepositories {\n mavenCentral()\n}\n\nkotlin {\n jvmToolchain(25)\n}\n\ndependencies {\n implementation(\"org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0\")\n testImplementation(\"io.kotest:kotest-runner-junit5:5.9.1\")\n testImplementation(\"io.kotest:kotest-assertions-core:5.9.1\")\n testImplementation(\"io.kotest:kotest-property:5.9.1\")\n}\n\ntasks.test {\n useJUnitPlatform()\n}\n"
|
|
194
195
|
},
|
|
195
196
|
"compact": {
|
|
196
197
|
"settings.gradle.kts": "rootProject.name = \"lawspec-example\"\n",
|
|
198
|
+
"gradle.properties": "kotlin.daemon.jvmargs=-Xmx2g\n",
|
|
197
199
|
"build.gradle.kts": "plugins { kotlin(\"jvm\") version \"2.3.21\" }\nrepositories { mavenCentral() }\nkotlin { jvmToolchain(25) }\ndependencies { implementation(\"org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0\"); testImplementation(\"io.kotest:kotest-runner-junit5:5.9.1\"); testImplementation(\"io.kotest:kotest-assertions-core:5.9.1\"); testImplementation(\"io.kotest:kotest-property:5.9.1\") }\ntasks.test { useJUnitPlatform() }\n"
|
|
198
200
|
}
|
|
199
201
|
},
|
|
@@ -280,11 +282,11 @@ const scaffolds = {
|
|
|
280
282
|
"groovy": {
|
|
281
283
|
"readable": {
|
|
282
284
|
"settings.gradle": "rootProject.name = 'lawspec-example'\n",
|
|
283
|
-
"build.gradle": "plugins {\n id 'groovy'\n}\n\nrepositories {\n mavenCentral()\n}\n\njava {\n toolchain { languageVersion = JavaLanguageVersion.of(25) }\n}\n\nsourceSets {\n main { java { srcDirs = [] }; groovy { srcDirs = ['src/main/groovy', 'src/main/java'] } }\n test { java { srcDirs = [] }; groovy { srcDirs = ['src/test/groovy', 'src/test/java'] } }\n}\n\ndependencies {\n implementation 'org.apache.groovy:groovy:5.0.8'\n testImplementation platform('org.spockframework:spock-bom:2.5-groovy-5.0')\n testImplementation 'org.spockframework:spock-core'\n testImplementation 'org.jetbrains:jetCheck:0.3.0'\n testImplementation 'org.junit.jupiter:junit-jupiter-api:5.14.0'\n testRuntimeOnly 'org.junit.platform:junit-platform-launcher'\n}\n\ntest {\n useJUnitPlatform()\n}\n"
|
|
285
|
+
"build.gradle": "plugins {\n id 'groovy'\n}\n\nrepositories {\n mavenCentral()\n}\n\njava {\n toolchain { languageVersion = JavaLanguageVersion.of(25) }\n}\n\nsourceSets {\n main { java { srcDirs = [] }; groovy { srcDirs = ['src/main/groovy', 'src/main/java'] } }\n test { java { srcDirs = [] }; groovy { srcDirs = ['src/test/groovy', 'src/test/java'] } }\n}\n\ntasks.withType(org.gradle.api.tasks.compile.GroovyCompile).configureEach {\n groovyOptions.forkOptions.memoryMaximumSize = '2g'\n}\n\ndependencies {\n implementation 'org.apache.groovy:groovy:5.0.8'\n testImplementation platform('org.spockframework:spock-bom:2.5-groovy-5.0')\n testImplementation 'org.spockframework:spock-core'\n testImplementation 'org.jetbrains:jetCheck:0.3.0'\n testImplementation 'org.junit.jupiter:junit-jupiter-api:5.14.0'\n testRuntimeOnly 'org.junit.platform:junit-platform-launcher'\n}\n\ntest {\n useJUnitPlatform()\n}\n"
|
|
284
286
|
},
|
|
285
287
|
"compact": {
|
|
286
288
|
"settings.gradle": "rootProject.name = 'lawspec-example'\n",
|
|
287
|
-
"build.gradle": "plugins { id 'groovy' }\nrepositories { mavenCentral() }\njava { toolchain { languageVersion = JavaLanguageVersion.of(25) } }\nsourceSets { main { java { srcDirs = [] }; groovy { srcDirs = ['src/main/groovy', 'src/main/java'] } }; test { java { srcDirs = [] }; groovy { srcDirs = ['src/test/groovy', 'src/test/java'] } } }\ndependencies { implementation 'org.apache.groovy:groovy:5.0.8'; testImplementation platform('org.spockframework:spock-bom:2.5-groovy-5.0'); testImplementation 'org.spockframework:spock-core'; testImplementation 'org.jetbrains:jetCheck:0.3.0'; testImplementation 'org.junit.jupiter:junit-jupiter-api:5.14.0'; testRuntimeOnly 'org.junit.platform:junit-platform-launcher' }\ntest { useJUnitPlatform() }\n"
|
|
289
|
+
"build.gradle": "plugins { id 'groovy' }\nrepositories { mavenCentral() }\njava { toolchain { languageVersion = JavaLanguageVersion.of(25) } }\nsourceSets { main { java { srcDirs = [] }; groovy { srcDirs = ['src/main/groovy', 'src/main/java'] } }; test { java { srcDirs = [] }; groovy { srcDirs = ['src/test/groovy', 'src/test/java'] } } }\ntasks.withType(org.gradle.api.tasks.compile.GroovyCompile).configureEach { groovyOptions.forkOptions.memoryMaximumSize = '2g' }\ndependencies { implementation 'org.apache.groovy:groovy:5.0.8'; testImplementation platform('org.spockframework:spock-bom:2.5-groovy-5.0'); testImplementation 'org.spockframework:spock-core'; testImplementation 'org.jetbrains:jetCheck:0.3.0'; testImplementation 'org.junit.jupiter:junit-jupiter-api:5.14.0'; testRuntimeOnly 'org.junit.platform:junit-platform-launcher' }\ntest { useJUnitPlatform() }\n"
|
|
288
290
|
}
|
|
289
291
|
},
|
|
290
292
|
"clojure": {
|
package/test-command.mjs
CHANGED
|
@@ -99,6 +99,12 @@ const groupBy = (entries, key) => {
|
|
|
99
99
|
return [...groups.entries()];
|
|
100
100
|
};
|
|
101
101
|
|
|
102
|
+
// Hpack discovers generated modules before native result keys are computed.
|
|
103
|
+
// Stack owns this metadata transformation; no compilation or tests run here.
|
|
104
|
+
export function configurationInvocations(target) {
|
|
105
|
+
return target.language==='haskell'?[{command:'stack',args:['--no-terminal','config','build-files']}]:[];
|
|
106
|
+
}
|
|
107
|
+
|
|
102
108
|
// Rebar and Mix run their configured compile hooks. Gleam has no corresponding
|
|
103
109
|
// hook, so build its generated crypto bridge before starting a native test VM.
|
|
104
110
|
export function preparationInvocations(target, files) {
|
|
@@ -197,7 +203,14 @@ export function benchmarkInvocations(target, entries, { offline = false } = {})
|
|
|
197
203
|
// LawSpec.TestNames), then a kind after a separator no name contains, so one
|
|
198
204
|
// law's name never selects another's. `scratch` is a folder for reports;
|
|
199
205
|
// `coverage` asks each runner to measure coverage too (see coverageSetup).
|
|
200
|
-
export function invocations(
|
|
206
|
+
export function invocations(...arguments_) {
|
|
207
|
+
return nativeInvocations(...arguments_).map(run => {
|
|
208
|
+
const matches=run.matches??run.ran;
|
|
209
|
+
return matches?{...run,matches,ran:test=>test.status==='passed'?matches(test):[]}:run;
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function nativeInvocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false,
|
|
201
214
|
root = process.cwd(), coverageConfig } = {}) {
|
|
202
215
|
if (!entries.length) return [];
|
|
203
216
|
const language = target.language;
|
|
@@ -206,12 +219,14 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
206
219
|
const directory = testDir || defaultDir;
|
|
207
220
|
return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
|
|
208
221
|
};
|
|
209
|
-
const
|
|
222
|
+
const model = (entry) => entry.kind === "model";
|
|
223
|
+
const named = (entry) => new RegExp(`^${regex(entry.name)}${model(entry) ? "$" : "_"}`);
|
|
210
224
|
const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
|
|
211
225
|
if (["erlang", "elixir", "gleam"].includes(language)) {
|
|
212
226
|
const run = randomUUID();
|
|
213
227
|
const file = path.join(scratch, `beam-${run}.jsonl`);
|
|
214
228
|
const identity = (entry) => `${entry.unit}::${entry.name}`;
|
|
229
|
+
const matching = test => entries.filter(entry => test.identity === identity(entry));
|
|
215
230
|
const env = { LAWSPEC_BEAM_REPORT: file, LAWSPEC_BEAM_RUN: run };
|
|
216
231
|
const measured = coverage ? {run, file: path.resolve(coverage, "runs", `${run}.coverdata`),
|
|
217
232
|
...(language === "elixir" ? {beamDirectory: coverageConfig?.compilePath} : {}),
|
|
@@ -242,34 +257,52 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
242
257
|
}
|
|
243
258
|
}
|
|
244
259
|
return [{ laws: entries, command, args, env, ...(measured ? {coverage: measured} : {}), report: {kind: "beam-events", file, run},
|
|
245
|
-
ran:
|
|
260
|
+
matches: matching, ran: test => test.status === "passed" ? matching(test) : [] }];
|
|
246
261
|
}
|
|
247
262
|
if (language === "python")
|
|
248
|
-
return groupBy(entries, (e) => e.file).
|
|
263
|
+
return groupBy(entries, (e) => e.file).flatMap(([file, selected], n) => {
|
|
264
|
+
const laws=selected.filter(entry=>!model(entry));
|
|
265
|
+
const models=selected.filter(model);
|
|
266
|
+
const invocations=[];
|
|
249
267
|
const report = path.join(scratch, `pytest-${n}.xml`);
|
|
250
268
|
// parallel: pytest-xdist runs the tests on several workers, when it
|
|
251
269
|
// is installed (lawspec test checks); otherwise one after another.
|
|
252
|
-
const workers = xdist &&
|
|
270
|
+
const workers = xdist && selected.some((e) => e.parallel) ? ["-p", "xdist", "-n", "auto"] : [];
|
|
253
271
|
const pytest = ["-m", "pytest", "-q", ...workers, file, "-k", laws.map((e) => `${e.name}__`).join(" or "), `--junitxml=${report}`];
|
|
254
|
-
|
|
272
|
+
if(laws.length)invocations.push({ laws, command: target.python || "python3",
|
|
255
273
|
args: coverage ? ["-m", "coverage", "run", "--append", `--data-file=${path.join(coverage, ".coverage")}`, ...pytest.slice(1)] : pytest,
|
|
256
274
|
report: { kind: "junit", files: [report] },
|
|
257
|
-
ran: (test) => laws.filter((e) => test.name.startsWith(`${e.name}__`)) };
|
|
275
|
+
ran: (test) => laws.filter((e) => test.name.startsWith(`${e.name}__`)) });
|
|
276
|
+
if(models.length){
|
|
277
|
+
const modelReport=path.join(scratch,`pytest-model-${n}.xml`);
|
|
278
|
+
const arguments_=["-m","pytest","-q",...workers,
|
|
279
|
+
...new Set(models.map(entry=>`${file}::${entry.name}`)),`--junitxml=${modelReport}`];
|
|
280
|
+
invocations.push({laws:models,command:target.python||"python3",
|
|
281
|
+
args:coverage?["-m","coverage","run","--append",`--data-file=${path.join(coverage,".coverage")}`,...arguments_.slice(1)]:arguments_,
|
|
282
|
+
report:{kind:"junit",files:[modelReport]},
|
|
283
|
+
ran:test=>models.filter(entry=>test.name===entry.name)});
|
|
284
|
+
}
|
|
285
|
+
return invocations;
|
|
258
286
|
});
|
|
259
287
|
if (language === "javascript" || language === "typescript") {
|
|
260
|
-
const pattern = (e) =>
|
|
288
|
+
const pattern = (e) => model(e)?`^${regex(e.label)}$`:
|
|
289
|
+
`^${regex(e.label)}( example: .*| property| known failing| skipped| replay| search)?$`;
|
|
261
290
|
const files = [...new Set(entries.map((e) => language === "typescript"
|
|
262
291
|
? `dist/${without(e.file, ".ts")}.js` : e.file))];
|
|
263
|
-
const
|
|
292
|
+
const file = path.join(scratch, "node-events.jsonl");
|
|
293
|
+
const runId=randomUUID();
|
|
294
|
+
const matching = test => entries.filter(entry => new RegExp(pattern(entry)).test(test.name));
|
|
264
295
|
// parallel: test files run at the same time too.
|
|
265
296
|
const node = ["--test", ...(entries.some((e) => e.parallel) ? ["--test-concurrency=" + Math.max(2, (globalThis.navigator?.hardwareConcurrency ?? 2))] : []),
|
|
266
297
|
`--test-name-pattern=(${entries.map(pattern).join("|")})`,
|
|
267
298
|
"--test-reporter=spec", "--test-reporter-destination=stdout",
|
|
268
|
-
|
|
299
|
+
`--test-reporter=${new URL('./node-events-reporter.mjs',import.meta.url).href}`,
|
|
300
|
+
`--test-reporter-destination=${file}`, ...files];
|
|
269
301
|
const run = { laws: entries, command: coverage ? "npx" : process.execPath,
|
|
270
302
|
args: coverage ? ["--no-install", "c8", "--reporter=text", `--reports-dir=${coverage}`, process.execPath, ...node] : node,
|
|
271
|
-
|
|
272
|
-
|
|
303
|
+
// A CLI launched from node:test must start an independent native run.
|
|
304
|
+
env:{LAWSPEC_NODE_RUN:runId,NODE_TEST_CONTEXT:undefined},report: { kind: "node-events", file, run:runId },
|
|
305
|
+
matches: matching, ran: test => test.status === 'passed' ? matching(test) : [] };
|
|
273
306
|
return language === "typescript"
|
|
274
307
|
? [{ laws: [], command: "npm", args: ["exec", "--", "tsc", "-p", "tsconfig.json"] }, run]
|
|
275
308
|
: [run];
|
|
@@ -278,7 +311,9 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
278
311
|
return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, laws]) => ({
|
|
279
312
|
laws, command: "go",
|
|
280
313
|
args: ["test", "-json", "-count=1", ...(coverage ? ["-cover", `-coverprofile=${path.join(coverage, `go-${directory.replaceAll("/", "_") || "root"}.out`)}`] : []),
|
|
281
|
-
`./${directory}`, "-run",
|
|
314
|
+
`./${directory}`, "-run", laws.some(model)?
|
|
315
|
+
`^(${[...new Set(laws.map(e=>regex(e.name)+(model(e)?"$":"_")))].join("|")})`:
|
|
316
|
+
`^(${laws.map((e) => regex(e.name)).join("|")})_`],
|
|
282
317
|
report: { kind: "go-json" },
|
|
283
318
|
ran: (test) => laws.filter((e) => named(e).test(test.name)) }));
|
|
284
319
|
if (language === "java") {
|
|
@@ -286,9 +321,9 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
286
321
|
return [{ laws: entries, command: target.maven || "mvn",
|
|
287
322
|
args: [...(offline ? ["-o"] : []), "-B", ...(coverage ? ["org.jacoco:jacoco-maven-plugin:prepare-agent"] : []), "test",
|
|
288
323
|
...(coverage ? ["org.jacoco:jacoco-maven-plugin:report"] : []),
|
|
289
|
-
`-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.map((e) =>
|
|
324
|
+
`-Dtest=${byClass.map(([name, laws]) => `${name}#${[...new Set(laws.map((e) => model(e)?e.name:`${e.name}_*`))].join("+")}`).join(",")}`],
|
|
290
325
|
report: { kind: "junit", directory: "target/surefire-reports" },
|
|
291
|
-
ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && named(e).test(test.name)) }];
|
|
326
|
+
ran: (test) => entries.filter((e) => test.classname === (e.scope??className(e.file, "src/test/java", ".java")) && named(e).test(test.name)) }];
|
|
292
327
|
}
|
|
293
328
|
// Kotest names tests by string, which Gradle's method filters cannot
|
|
294
329
|
// select. Kotest's own filter selects them within each class: one pattern,
|
|
@@ -296,13 +331,15 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
296
331
|
// patterns must all match, so alternatives go in one group).
|
|
297
332
|
if (language === "kotlin")
|
|
298
333
|
return groupBy(entries, (e) => e.file).map(([file, laws]) => {
|
|
299
|
-
const filter =
|
|
334
|
+
const filter = laws.some(model)?
|
|
335
|
+
`(${[...new Set(laws.map(e=>regex(e.name)+(model(e)?"":"_.*")))].join("|")})`:
|
|
336
|
+
`(${laws.map((e) => `${e.name}_`).join("|")})*`;
|
|
300
337
|
const name = className(file, "src/test/kotlin", ".kt");
|
|
301
338
|
return { laws, command: target.gradle || "gradle",
|
|
302
339
|
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["koverXmlReport"] : []), "--rerun", "--tests", name],
|
|
303
340
|
env: { "kotest.filter.tests": filter, kotest_filter_tests: filter },
|
|
304
341
|
report: { kind: "junit", directory: "build/test-results/test" },
|
|
305
|
-
ran: (test) => laws.filter((e) => test.classname === name && named(e).test(test.name)) };
|
|
342
|
+
ran: (test) => laws.filter((e) => test.classname === (e.scope??name) && named(e).test(test.name)) };
|
|
306
343
|
});
|
|
307
344
|
// ScalaTest selects tests by substring (-z): each law's tests are named
|
|
308
345
|
// after it, then a kind after a separator no name contains. Its JUnit
|
|
@@ -317,19 +354,22 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
317
354
|
const measured = coverage ? ["-O", `-coverage-out:${path.resolve(coverage, "data")}`,
|
|
318
355
|
"-O", `-coverage-exclude-files:.*/${testRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/.*`] : [];
|
|
319
356
|
return { laws, command: target.scala || "scala",
|
|
320
|
-
args: ["test", ".", ...(offline ? ["--offline"] : []), ...measured, "--test-only", name, "--",
|
|
321
|
-
...laws.
|
|
357
|
+
args: ["test", ".", ...(offline ? ["--power","--offline"] : []), ...measured, "--test-only", name, "--",
|
|
358
|
+
...[...new Map(laws.map(e=>[(model(e)?"exact:":"prefix:")+e.name,e])).values()]
|
|
359
|
+
.flatMap(e=>[model(e)?"-t":"-z",model(e)?e.name:`${e.name}_`]), "-u", directory],
|
|
322
360
|
report: { kind: "junit", directory: path.relative(root, directory) || "." },
|
|
323
|
-
ran: (test) => laws.filter((e) => test.classname === name && named(e).test(test.name)) };
|
|
361
|
+
ran: (test) => laws.filter((e) => test.classname === (e.scope??name) && named(e).test(test.name)) };
|
|
324
362
|
});
|
|
325
363
|
// A Clojure test is the law's words, a double dash and the kind of test
|
|
326
364
|
// (LawSpec.TestNames); the runner selects tests by namespace and prefix.
|
|
327
365
|
if (language === "clojure") {
|
|
328
|
-
const ns = (e) => without(relativeTo(e.file, "test"), ".clj").replaceAll("/", ".").replaceAll("_", "-");
|
|
366
|
+
const ns = (e) => e.scope??without(relativeTo(e.file, "test"), ".clj").replaceAll("/", ".").replaceAll("_", "-");
|
|
329
367
|
return [{ laws: entries, command: target.clojure || "clojure",
|
|
330
|
-
args: ["-M:test", ...entries.
|
|
368
|
+
args: ["-M:test", ...[...new Map(entries.map(e=>[(model(e)?"exact:":"prefix:")+ns(e)+"/"+e.name,e])).values()]
|
|
369
|
+
.flatMap(e=>[model(e)?"--select-exact":"--select",`${ns(e)}/${e.name}${model(e)?"":"--"}`])],
|
|
331
370
|
report: { kind: "junit", directory: "target/test-results" },
|
|
332
|
-
ran: (test) => entries.filter((e) => test.classname === ns(e) &&
|
|
371
|
+
ran: (test) => entries.filter((e) => test.classname === ns(e) &&
|
|
372
|
+
(model(e)?test.name===e.name:test.name.startsWith(`${e.name}--`))) }];
|
|
333
373
|
}
|
|
334
374
|
// A Spock feature is the law's words, a colon and the kind of test
|
|
335
375
|
// (LawSpec.TestNames), so "name: *" selects one law's features only.
|
|
@@ -337,15 +377,16 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
337
377
|
const name = (e) => className(e.file, "src/test/groovy", ".groovy");
|
|
338
378
|
return [{ laws: entries, command: target.gradle || "gradle",
|
|
339
379
|
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["jacocoTestReport"] : []), "--rerun",
|
|
340
|
-
...entries.
|
|
380
|
+
...[...new Set(entries.map(e=>`${e.scope??name(e)}.${e.name}${model(e)?"":": *"}`))].flatMap(name=>["--tests",name])],
|
|
341
381
|
report: { kind: "junit", directory: "build/test-results/test" },
|
|
342
|
-
ran: (test) => entries.filter((e) => test.classname ===
|
|
382
|
+
ran: (test) => entries.filter((e) => test.classname === (e.scope??name(e)) &&
|
|
383
|
+
(model(e)?test.name===e.name:test.name.startsWith(`${e.name}: `))) }];
|
|
343
384
|
}
|
|
344
385
|
if (language === "rust")
|
|
345
386
|
return groupBy(entries, (e) => e.file).map(([file, laws]) => ({
|
|
346
387
|
laws, command: "cargo",
|
|
347
388
|
args: [...(coverage ? ["llvm-cov", "--no-report"] : []), "test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
|
|
348
|
-
...laws.map((e) => e.name)],
|
|
389
|
+
...new Set(laws.map((e) => e.name))],
|
|
349
390
|
report: { kind: "lines", pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/ },
|
|
350
391
|
ran: (test) => laws.filter((e) => test.name === e.name) }));
|
|
351
392
|
// xUnit v3 on Microsoft.Testing.Platform selects test methods with its
|
|
@@ -381,11 +422,13 @@ export function invocations(target, entries, { offline = false, scratch = ".", c
|
|
|
381
422
|
}
|
|
382
423
|
if (language === "haskell") {
|
|
383
424
|
const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
|
|
384
|
-
const matches = entries.map((e) =>
|
|
425
|
+
const matches = [...new Set(entries.map((e) => model(e)?
|
|
426
|
+
`/${e.scope??module(e)}/${e.unit} models/${e.name}/`:`${module(e)}/${e.name}_`))];
|
|
385
427
|
return [{ laws: entries, command: "stack",
|
|
386
|
-
args: ["--no-terminal", "test", ...(coverage ? ["--coverage"] : []), "--test-arguments",
|
|
428
|
+
args: ["--no-terminal", "test", ...(coverage ? ["--coverage"] : []), "--test-arguments",
|
|
429
|
+
matches.map((m) => `--match ${m.includes(" ")?JSON.stringify(m):m}`).join(" ")],
|
|
387
430
|
report: { kind: "hspec" },
|
|
388
|
-
ran: (test) => entries.filter((e) => test.classname === module(e) && named(e).test(test.name)) }];
|
|
431
|
+
ran: (test) => entries.filter((e) => test.classname === (e.scope??module(e)) && named(e).test(test.name)) }];
|
|
389
432
|
}
|
|
390
433
|
throw new Error(`lawspec test does not support ${language}`);
|
|
391
434
|
}
|
|
@@ -502,6 +545,20 @@ export async function harnessStatistics(directory) {
|
|
|
502
545
|
// The tests a run executed, as {name, classname}, from the runner's report or
|
|
503
546
|
// output. Skipped tests are not counted.
|
|
504
547
|
export async function executedTests(report, output, root, since) {
|
|
548
|
+
if(report?.kind==='node-events') {
|
|
549
|
+
const raw=await readFile(path.resolve(root,report.file),'utf8').catch(()=>'');
|
|
550
|
+
let rows;
|
|
551
|
+
try {rows=raw.trim().split('\n').map(line=>JSON.parse(line));}catch{return [];}
|
|
552
|
+
if(rows.length<2||rows[0]?.event!=='start'||rows.at(-1)?.event!=='end'||
|
|
553
|
+
rows.some(row=>row?.version!==1||row?.run!==report.run)||
|
|
554
|
+
rows.at(-1).count!==rows.length-2) return [];
|
|
555
|
+
const tests=rows.slice(1,-1);
|
|
556
|
+
if(tests.some(row=>row?.event!=='test'||typeof row.name!=='string'||typeof row.classname!=='string'||
|
|
557
|
+
!['passed','failed','skipped'].includes(row.status)||
|
|
558
|
+
(row.time!==undefined&&(!Number.isFinite(row.time)||row.time<0))||
|
|
559
|
+
(row.failure!==undefined&&typeof row.failure!=='string')))return [];
|
|
560
|
+
return tests;
|
|
561
|
+
}
|
|
505
562
|
if (report?.kind === "beam-events") {
|
|
506
563
|
// A native listener seals the invocation after its final event. Missing,
|
|
507
564
|
// stale, malformed or interrupted reports are not execution evidence.
|
|
@@ -520,11 +577,25 @@ export async function executedTests(report, output, root, since) {
|
|
|
520
577
|
const files = report.files ?? await reportFiles(path.join(root, report.directory), since);
|
|
521
578
|
const tests = [];
|
|
522
579
|
for (const file of files) {
|
|
523
|
-
const
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
580
|
+
const location=path.resolve(root,file);
|
|
581
|
+
const info=await stat(location).catch(()=>null);
|
|
582
|
+
if(!info||info.mtimeMs<since)continue;
|
|
583
|
+
const xml=await readFile(location,"utf8").catch(()=>"");
|
|
584
|
+
const attributes=opening=>Object.fromEntries([...opening.matchAll(/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)]
|
|
585
|
+
.map(match=>[match[1],unescapeXml(match[2]??match[3])]));
|
|
586
|
+
for(const element of junitElements(xml,["testcase"])){
|
|
587
|
+
const values=attributes(element.opening);
|
|
588
|
+
const native=junitElements(element.xml,["failure","error","skipped"]);
|
|
589
|
+
const failure=native.filter(element=>element.name!=="skipped");
|
|
590
|
+
const status=failure.length?"failed":native.length?"skipped":"passed";
|
|
591
|
+
const detail=element=>[attributes(element.opening).message,
|
|
592
|
+
unescapeXml(element.xml.slice(element.opening.length).replace(/<\/(failure|error|skipped)>\s*$/,"")
|
|
593
|
+
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g,(_,text)=>text))].filter(Boolean).join("\n");
|
|
594
|
+
const time=values.time===undefined?undefined:Number(values.time);
|
|
595
|
+
tests.push({name:(values.name??"").replace(/\(\)$/, ""),classname:values.classname??"",status,
|
|
596
|
+
...(Number.isFinite(time)&&time>=0?{time}:{}),
|
|
597
|
+
...(failure.length?{failure:failure.map(detail).join("\n")}:
|
|
598
|
+
status==="skipped"?{reason:native.map(detail).join("\n")}: {})});
|
|
528
599
|
}
|
|
529
600
|
}
|
|
530
601
|
return tests;
|
|
@@ -548,7 +619,8 @@ export async function executedTests(report, output, root, since) {
|
|
|
548
619
|
let module = "";
|
|
549
620
|
for (const line of output.split("\n")) {
|
|
550
621
|
if (/^\S/.test(line) && !/^(Finished|Failures|Randomized|\d+ examples?)/.test(line)) module = line.trim();
|
|
551
|
-
const example = line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/)
|
|
622
|
+
const example = line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/) ??
|
|
623
|
+
line.match(/^\s+(model .+?|supervision)\s\[([✔✘])\]\s*$/);
|
|
552
624
|
if (example) tests.push({ name: example[1], classname: module, status: example[2] === "✔" ? "passed" : "failed" });
|
|
553
625
|
}
|
|
554
626
|
return tests;
|