lawspec 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +361 -2
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +306 -38
- package/build.json +492 -106
- package/compatibility.json +17 -0
- package/core.wasm +0 -0
- package/core_jsffi.js +2 -2
- package/doctor.mjs +7 -1
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/examples/specs/abilities.lawspec +103 -0
- package/examples/specs/ability_imports.lawspec +107 -0
- package/examples/specs/builtins.lawspec +121 -0
- package/examples/specs/crypto.lawspec +51 -0
- package/examples/specs/distribution.lawspec +29 -0
- package/examples/specs/failures.lawspec +96 -0
- package/examples/specs/handler_bindings.lawspec +65 -0
- package/examples/specs/harness.lawspec +75 -0
- package/examples/specs/harness_testing.lawspec +45 -0
- package/examples/specs/matchers.lawspec +112 -0
- package/examples/specs/resources.lawspec +116 -0
- package/examples/specs/scheduling_order.lawspec +18 -0
- package/examples/specs/scheduling_overlap.lawspec +14 -0
- package/examples/specs/tables.lawspec +64 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +12 -1
- package/native-examples.mjs +7 -2
- package/package.json +1 -1
- package/templates.mjs +105 -16
- package/test-command.mjs +316 -35
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,363 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.22.0] - 2026-10-10
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Erlang, Elixir and Gleam targets share a BEAM runtime and generate native
|
|
14
|
+
data types, adapters and checked definitions. Tests use PropEr with EUnit,
|
|
15
|
+
StreamData with ExUnit, and qcheck with gleeunit, including each framework's
|
|
16
|
+
own generation and shrinking.
|
|
17
|
+
- BEAM ability handlers, typed failures, asynchronous tasks, workflow policies
|
|
18
|
+
and built-in abilities, with native type, function, handler and generator
|
|
19
|
+
bindings. Native payment examples demonstrate adopting and removing
|
|
20
|
+
generated tests in an existing application.
|
|
21
|
+
- BEAM stateful models, collection models, opaque handles, consistency
|
|
22
|
+
models, scenarios, typed sessions and mailboxes, and actors with OTP
|
|
23
|
+
supervision. Stateful checks have individual identities for CLI selection,
|
|
24
|
+
reports, caching and replay.
|
|
25
|
+
- Secure BEAM distribution uses the shared wire format and cryptographic
|
|
26
|
+
seed keys. Projects importing `lawspec.crypto` or `lawspec.network` build a
|
|
27
|
+
small C/OpenSSL NIF before compilation. The bridge requires OTP 29,
|
|
28
|
+
OpenSSL 3.5 or later matching OTP's OpenSSL major version, and a Unix C
|
|
29
|
+
toolchain on macOS, Linux or WSL. Projects without those imports have no
|
|
30
|
+
native crypto build step.
|
|
31
|
+
- BEAM setup guides, toolchain checks, offline dependencies, JUnit reporting,
|
|
32
|
+
coverage, and failure replay from the exact shrunk inputs.
|
|
33
|
+
- BEAM harness strategies, generated-case coverage and labels, targeted
|
|
34
|
+
search, repeats and flaky retries, skips and known failures, random law
|
|
35
|
+
order, parallel execution and benchmarks in the native test frameworks.
|
|
36
|
+
- BEAM matcher, table and description examples, with recorded values read
|
|
37
|
+
from the shared project folder and updated by `lawspec test --update-recorded`.
|
|
38
|
+
- BEAM per-case and shared resources use dedicated owner processes for
|
|
39
|
+
acquisition, reset and release. Harness timeouts cancel test workers and
|
|
40
|
+
join their asynchronous children; cleanup has a separate bounded allowance.
|
|
41
|
+
Cleanup failures fail the run, including under retries and known failures.
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
44
|
+
|
|
45
|
+
- Recorded values use one typed format across targets, including exact
|
|
46
|
+
numbers, floats, bytes, raw text, presence and nested data. Identity labels
|
|
47
|
+
are local to each recording. Existing extended scalar recordings may need
|
|
48
|
+
updating with `lawspec test --update-recorded`.
|
|
49
|
+
- Generated TypeScript, Python, Go and Kotlin product types retain their type
|
|
50
|
+
parameters when their sole constructor fixes a parameter's type.
|
|
51
|
+
- Generated Haskell constructors and codecs apply fixed type parameters to
|
|
52
|
+
their fields, including nested data.
|
|
53
|
+
- Erlang EUnit output retains deeper failure diagnostics, including native
|
|
54
|
+
generator identities used by acceptance checks.
|
|
55
|
+
- CI logs include exit statuses, and acceptance logs retain completed stages
|
|
56
|
+
when a runner exits unexpectedly.
|
|
57
|
+
- Failure databases and reports keep separate project roots apart, including
|
|
58
|
+
projects using the same target. Replaying or repairing one project's
|
|
59
|
+
failure cannot clear another project's counterexample.
|
|
60
|
+
- CLI JUnit output retains cases from flat Node reports and nested native
|
|
61
|
+
suites.
|
|
62
|
+
- BEAM JUnit output includes intentionally skipped laws and their reasons;
|
|
63
|
+
skips remain separate from evidence that a test executed. Scheduled cases
|
|
64
|
+
keep their declaring unit, so identical test names in different units
|
|
65
|
+
remain distinct.
|
|
66
|
+
- BEAM generators follow scalar and constructor-field refinements, including
|
|
67
|
+
named symbol identities, minimum lengths and bounds depending on earlier
|
|
68
|
+
fields. Native shrinking preserves these constraints.
|
|
69
|
+
- Generated BEAM sets and maps keep their portable order and distinct keys;
|
|
70
|
+
native values with unsorted or duplicate entries fail validation.
|
|
71
|
+
|
|
72
|
+
## [0.21.0] - 2026-10-07
|
|
73
|
+
|
|
74
|
+
### Added
|
|
75
|
+
|
|
76
|
+
- Existing features are abilities. `async f ::` is `f :: ... uses Async`
|
|
77
|
+
(both spellings mean the same declaration); workflow policies are handler
|
|
78
|
+
transformers over `Async`, `Clock` and `Fail`; a protocol is `Session P`,
|
|
79
|
+
a mailbox `Mailbox T`, an actor a `Process` with a `State` handler, a
|
|
80
|
+
supervisor a `Fail` handler that restarts, a model a stateful spec
|
|
81
|
+
handler checked through `abstract`, a scenario a program under a
|
|
82
|
+
`Scheduler` handler, and distribution the `Network` ability whose
|
|
83
|
+
transports are handlers. The syntax is unchanged; `lawspec check --json`
|
|
84
|
+
lists each unit's `abilityRows`, and the new page "Existing features as
|
|
85
|
+
abilities" shows each mapping.
|
|
86
|
+
- The network is secure by default on every target: each node has an
|
|
87
|
+
ML-DSA-65 identity, the first frame between two nodes waits for a signed
|
|
88
|
+
ML-KEM-768 handshake, and frames cross sealed with AES-256-GCM. The record
|
|
89
|
+
format is the same on every target, and each checks the same handshake
|
|
90
|
+
vector. `nativeBindings.network` in `lawspec.json` binds a node identity
|
|
91
|
+
and trusted peers (written to `lawspec-network.conf`). An in-memory
|
|
92
|
+
transport without the handshake exists for tests only; no setting turns
|
|
93
|
+
security off. A program that makes nodes imports `lawspec.network`, which
|
|
94
|
+
writes the secure handler's module beside the runtime; programs without
|
|
95
|
+
it, or without `lawspec.crypto`, no longer depend on any crypto library,
|
|
96
|
+
and `lawspec init` scaffolds projects without them. Scenarios' network
|
|
97
|
+
runs use the in-memory transport made for tests. (Rust and Haskell call
|
|
98
|
+
`lawspec_network::install()` / `LawSpecNetwork.install` before making
|
|
99
|
+
nodes; the generated tests do.)
|
|
100
|
+
- Temporal propositions: `eventually within 2 s, P`, `always within 500 ms,
|
|
101
|
+
P` and `never within d, P`, over the `Clock`; under `using virtual clock`
|
|
102
|
+
they are deterministic and checked at compile time.
|
|
103
|
+
- Performance budgets: `expect f x takes at most 5 ms`, or `e takes at most
|
|
104
|
+
d` as a law's claim, timed on the real clock, with the new evidence status
|
|
105
|
+
`measured`.
|
|
106
|
+
- Typed mailboxes have `receive_within` / `receiveWithin`: the next message,
|
|
107
|
+
or nothing when none arrives in time.
|
|
108
|
+
- The `Async` default handler also starts tasks, waits for them and runs
|
|
109
|
+
several side by side; workflows' `all` groups run through it.
|
|
110
|
+
- Built-in abilities. `import lawspec.time`, `lawspec.randomness`,
|
|
111
|
+
`lawspec.crypto`, `lawspec.host`, `lawspec.logging` or `lawspec.concurrent`
|
|
112
|
+
adds the unit and its abilities: `Clock`; `Random` and `SecureRandom`;
|
|
113
|
+
`Hash`, `KeyExchange`, `Signature` and `Aead`; `FileSystem`, `Environment`
|
|
114
|
+
and `Ports`; `Log` and `Trace`; `Async`. Each has laws, and a default
|
|
115
|
+
handler on every target, generated into the unit's module and reported in
|
|
116
|
+
evidence with the new status `default-handler` (`DEFAULT HANDLER`).
|
|
117
|
+
`handlers` in `lawspec.json` binds another, and the ability's laws then
|
|
118
|
+
check it.
|
|
119
|
+
- `Instant`, microseconds since 1970: `t + d`, `t - d`, `b - a` (a
|
|
120
|
+
`Duration`) and comparisons, saturating rather than failing. `Clock` has
|
|
121
|
+
`now` and `sleep`, and its laws say time does not go back and sleeping
|
|
122
|
+
lets the time pass. `using virtual clock` runs a law under a clock that
|
|
123
|
+
moves only when told to; `advance d` lets `d` pass.
|
|
124
|
+
- `Random` is reproducible: the same seed gives the same draws on all eight
|
|
125
|
+
targets. `using seeded random n` starts it at `n`; the default handler
|
|
126
|
+
starts at the run's seed. `SecureRandom` is the operating system's
|
|
127
|
+
generator. They are separate abilities: seeded random where SecureRandom
|
|
128
|
+
is needed is a compile error, SecureRandom has no spec handlers, and the
|
|
129
|
+
native interfaces differ, so the typed targets reject the mix too.
|
|
130
|
+
- Cryptography is post-quantum by default: ML-KEM-768 (FIPS 203) key
|
|
131
|
+
exchange, ML-DSA-65 (FIPS 204) signatures, SHA3-256 and SHAKE256 (FIPS
|
|
132
|
+
202), and AES-256-GCM authenticated encryption, with SLH-DSA-SHAKE-128f
|
|
133
|
+
(FIPS 205) as an alternative Signature handler. Keys, ciphertexts and
|
|
134
|
+
signatures are opaque values whose bytes are the standards' encodings.
|
|
135
|
+
Their laws: decapsulating gives the encapsulated secret, a signature
|
|
136
|
+
verifies exactly for its message and its signer's key, and unsealing fails
|
|
137
|
+
for another key or other associated data.
|
|
138
|
+
- A program that imports `lawspec.crypto` gets a test of the default
|
|
139
|
+
handlers against NIST's ACVP and CAVP vectors, on every target.
|
|
140
|
+
- New acceptance suites `builtins` and `crypto`, on all eight targets, with
|
|
141
|
+
mutants: a clock that goes back, a seeded Random used as SecureRandom (a
|
|
142
|
+
compile error in the typed targets), a log written twice, a signature
|
|
143
|
+
handler that verifies a tampered message, and a key exchange that returns
|
|
144
|
+
another secret.
|
|
145
|
+
- Abilities. `ability Gateway is authorize :: Card -> Payment ... laws ... end`
|
|
146
|
+
names a dependency's operations and the laws every handler of it keeps.
|
|
147
|
+
`uses Gateway, Clock` after a signature says what it uses; a native adapter
|
|
148
|
+
gets one handler per ability before its arguments. A checked definition may
|
|
149
|
+
use abilities, and its ability row is inferred; a definition that lists its
|
|
150
|
+
abilities must list at least those.
|
|
151
|
+
- Handlers. `handler fakeGateway for Gateway is authorize c is ... end ... end`
|
|
152
|
+
is a spec handler; each clause is a checked definition. `with state s :: S
|
|
153
|
+
start e` keeps a state, updated with `~s := e;`. Each ability's native
|
|
154
|
+
production handler is written by hand (`GatewayHandler`), or bound under
|
|
155
|
+
`handlers` in `lawspec.json`.
|
|
156
|
+
- A law holds for every lawful handler. `using h` names a handler only when
|
|
157
|
+
it is part of the claim; otherwise the law runs under each lawful handler in
|
|
158
|
+
turn, one law per choice. `using recording Gateway` records calls, which a
|
|
159
|
+
law counts with `calls of capture` and `calls of capture with (cents)`.
|
|
160
|
+
- Each ability law is one obligation per handler. For a spec handler without
|
|
161
|
+
state the compiler proves it, or checks every case of a finite domain, so a
|
|
162
|
+
handler that breaks it is a compile error; native handlers are
|
|
163
|
+
property-tested.
|
|
164
|
+
- `fails with E` is short for `uses Fail E`. `raise e` aborts with a failure,
|
|
165
|
+
and `prelude.attempt e` in a law gives `Right` of the value or `Left` of the
|
|
166
|
+
failure.
|
|
167
|
+
- Core has `Perform`, `Handle` and `Calls`. Every target passes handlers in
|
|
168
|
+
the context it gives every definition (evidence passing), and generates a
|
|
169
|
+
native interface per ability: a Python `Protocol`, a TypeScript interface, a
|
|
170
|
+
Go interface, a Java or Kotlin interface, a Rust trait, a Haskell record of
|
|
171
|
+
`IO` operations; with spec handler and recording classes beside it.
|
|
172
|
+
- Abilities cross units. `import` brings a unit's abilities and handlers;
|
|
173
|
+
imported definitions may use abilities, and the declaring unit keeps the
|
|
174
|
+
native interface, production handler and recording.
|
|
175
|
+
- Effects in order: `a; b` and `let x = e in body` in definitions and laws.
|
|
176
|
+
An operation that gives `Unit` is called for its effect.
|
|
177
|
+
- `handle e with h end` installs a spec handler around part of a definition,
|
|
178
|
+
which then does not use its ability. A handler's clauses may use other
|
|
179
|
+
abilities; a law under that handler gets handlers for them too.
|
|
180
|
+
- A parameterized ability may be used at several types in one unit (`Store
|
|
181
|
+
Int32`, `Store Text`), each with its own interface (`StoreInt32`).
|
|
182
|
+
- An operation's type may be refined; every handler owes its result's
|
|
183
|
+
refinement as a law.
|
|
184
|
+
- Native adapters fail with the runtime's `Fail` on every target (`raise
|
|
185
|
+
ls.Fail(v)`, `throw new ls.Fail(v)`, `panic(LawSpecFail{Value: v})`,
|
|
186
|
+
`LawSpecRuntime.Fail`, `ls::fail`, `LS.Fail`), and `failures` in
|
|
187
|
+
`lawspec.json` maps the application's own exceptions to failure
|
|
188
|
+
constructors.
|
|
189
|
+
- The compiler runs spec handlers with state over a finite domain, so a law
|
|
190
|
+
they break is a compile error.
|
|
191
|
+
- `lawspec explain` shows a law's handlers and the ability rows of what it
|
|
192
|
+
calls.
|
|
193
|
+
- A bound production handler speaks the bound native types: through the
|
|
194
|
+
handler schema in Python and JavaScript, and an `<Ability>Bound` wrapper in
|
|
195
|
+
the typed targets. A bound adapter gets its handlers too.
|
|
196
|
+
- New acceptance suite `abilities`, with mutants: an adapter that calls its
|
|
197
|
+
dependency twice, native handlers that break an ability law or a refined
|
|
198
|
+
result, and an adapter that fails with the wrong failure. It now covers two
|
|
199
|
+
units. New suite `handlerbindings`: bound handlers, bound types and mapped
|
|
200
|
+
exceptions on all eight targets.
|
|
201
|
+
- Matchers: typed predicates that read like sentences. `xs has same items
|
|
202
|
+
as ys`, `xs contains x` (an item, or text inside text), `xs contains all of
|
|
203
|
+
ys`, `xs is subset of ys`, `x is within 0.001 of y`, `t starts with "a"`,
|
|
204
|
+
`t ends with "z"`, `t matches regex "[a-z]+"` and `v matches Shipped _ _`.
|
|
205
|
+
The list matchers are checked definitions of a built-in unit,
|
|
206
|
+
`lawspec.matchers`.
|
|
207
|
+
- `Regex` and `regex "..."`. Every regex literal is checked at compile time
|
|
208
|
+
against a portable dialect, the part of RE2 and ECMAScript that means the
|
|
209
|
+
same in both, and matches a whole text. Each runtime has its own matcher, so
|
|
210
|
+
no target needs a regex library.
|
|
211
|
+
- Typed failures in laws: `expect charge o fails with Declined _`, with
|
|
212
|
+
`message contains "..."` for a failure's message field. `expect e` takes
|
|
213
|
+
any `Bool`.
|
|
214
|
+
- Tables: `table (a, b, expected) is row 1, 2, 3 ... expect f a b = expected
|
|
215
|
+
end`. Each row is its own example.
|
|
216
|
+
- Examples fenced with `example` in a law's description are examples of the
|
|
217
|
+
law, named after the description.
|
|
218
|
+
- Recorded values: `expect f x = recorded "name"` compares with
|
|
219
|
+
`recorded/<unit>/<name>`, by the portable rendering every target shares. A
|
|
220
|
+
missing or changed recording fails; `lawspec test --update-recorded`
|
|
221
|
+
records again.
|
|
222
|
+
- Resources: `resource Database is acquire ... release ... reset ... end`,
|
|
223
|
+
taken by a law with `law ... for db :: Database is`. Each case acquires
|
|
224
|
+
them first and releases them after, even when it fails. Built in:
|
|
225
|
+
`TemporaryDirectory`, `TemporaryFile`, `FreePort` and `SavedEnvironment`.
|
|
226
|
+
- A failing comparison names where a list or data value first differs, with
|
|
227
|
+
both values in the portable rendering.
|
|
228
|
+
- New acceptance suites `matchers`, `failures`, `tables` and `resources`, with
|
|
229
|
+
mutants: a reordered list that fails `=` but passes `has same items as`, a
|
|
230
|
+
resource never released, a failure of the wrong constructor, and a stale
|
|
231
|
+
recording.
|
|
232
|
+
- Harness units: `harness shop.testing for shop is ... end`, in its own file
|
|
233
|
+
or after a unit's members, says how a unit's laws are tested and can never
|
|
234
|
+
change what they mean. It may not declare laws, definitions, abilities,
|
|
235
|
+
handlers or types, refers only to its unit's laws, handlers and resources,
|
|
236
|
+
and its expressions call checked definitions only.
|
|
237
|
+
- Strategies: `strategy orders :: Order is frequency 9 small, 1 bulk end`,
|
|
238
|
+
with `any`, `one of`, `such that p at most n discards` and `bind x :: T
|
|
239
|
+
from g in g'`, used with `for law \`x\` use orders for o`. A strategy's
|
|
240
|
+
type must be the input's, and every value it draws is checked against the
|
|
241
|
+
input's refinements.
|
|
242
|
+
- `test with fakeGateway, native` narrows the lawful handlers a law runs
|
|
243
|
+
against; a variant it leaves out stays an obligation, reported skipped.
|
|
244
|
+
- Adequacy: `cover 10% "label" when p` (an unmet cover fails the run),
|
|
245
|
+
`classify p as "label"`, `label e` and `target maximize e`, with
|
|
246
|
+
statistics printed and kept by `lawspec test`.
|
|
247
|
+
- Run metadata per unit, group (`for laws`) or law: `tags`, `skip`,
|
|
248
|
+
`known failing` (a known-failing law that passes fails the run), `timeout`,
|
|
249
|
+
`repeat`, `retry flaky`, `order random` and `parallel`.
|
|
250
|
+
- `share R per group | unit | run`, only for a resource that declares
|
|
251
|
+
`reset`; `benchmark \`name\` is e end`, measured and never asserted.
|
|
252
|
+
- All eight targets, through each one's property-testing library.
|
|
253
|
+
- Evidence has three harness statuses, `known-failing`, `flaky` and `skipped`.
|
|
254
|
+
`lawspec evidence` shows a `HARNESS` section after the obligations: each
|
|
255
|
+
law's harness and its last run's adequacy.
|
|
256
|
+
- `lawspec test --tag a --exclude-tag b` selects laws by harness tags;
|
|
257
|
+
`--report junit=path` writes one JUnit report merged across targets;
|
|
258
|
+
`--coverage` measures coverage with coverage.py, c8, `go test -cover`,
|
|
259
|
+
JaCoCo, Kover, cargo-llvm-cov or hpc, and says how to install a missing one.
|
|
260
|
+
A failing law is kept with its seed in `.lawspec/failures` and replayed
|
|
261
|
+
first; Hypothesis and proptest keep their counterexamples there too.
|
|
262
|
+
- New acceptance suite `harness`, with mutants, among them spec mutants (an
|
|
263
|
+
acceptance suite may now edit its specs): an unmet cover, a strategy drawing
|
|
264
|
+
outside the input's refinement, a harness declaring a law, sharing a
|
|
265
|
+
resource without `reset`, and a known-failing law that passes.
|
|
266
|
+
- A shared resource is shared at run time: `share R per group | unit | run`
|
|
267
|
+
acquires R once per scope, resets it before every later case, lets one case
|
|
268
|
+
hold it at a time, and releases it when the test process ends, on all eight
|
|
269
|
+
targets. The `resources` suite shares a pool that may be opened only once
|
|
270
|
+
per process, with mutants for a pool never drained and for a harness that
|
|
271
|
+
does not share it.
|
|
272
|
+
- `expect ... fails with` checks native adapters' failures too: synchronous
|
|
273
|
+
and async adapters raise the runtime's `Fail` on every target, and a
|
|
274
|
+
constructor mapped from an application's exception under `failures` in
|
|
275
|
+
`lawspec.json` matches with its message.
|
|
276
|
+
- A benchmark may call what uses abilities; it runs under their production
|
|
277
|
+
handlers. `lawspec test --benchmarks` runs the harness's benchmarks after the
|
|
278
|
+
laws.
|
|
279
|
+
- A strategy's type may be an inline refinement, `strategy small :: (o ::
|
|
280
|
+
Order where itemsOf o <= 3) is ... end`, whose values are kept like `such
|
|
281
|
+
that`. Harness expressions and strategies may name a unit's imports through
|
|
282
|
+
their aliases.
|
|
283
|
+
- `target maximize` steers on every target: where the property library has
|
|
284
|
+
no targeted search, LawSpec climbs after the property, moving the
|
|
285
|
+
best-scoring case's integers while the score rises.
|
|
286
|
+
- LawSpec owns a unit's scheduling through a small harness driver in each
|
|
287
|
+
runtime; the test framework hosts and reports. `order random` is seeded by
|
|
288
|
+
the run's seed, printed, and replayable on all eight targets; `parallel`
|
|
289
|
+
runs a unit's tests at the same time on all eight (Python on a thread pool
|
|
290
|
+
when pytest-xdist is absent, Java through a generated
|
|
291
|
+
`junit-platform.properties`, Haskell on the threaded runtime), and each
|
|
292
|
+
runtime records the parallelism it achieved, which `lawspec test` prints.
|
|
293
|
+
The test manifest carries whether a law's unit is parallel, and its
|
|
294
|
+
benchmarks.
|
|
295
|
+
- `resource T is concurrent`: a resource cases cannot interfere through. A
|
|
296
|
+
harness may share a resource under `parallel` only if it is concurrent.
|
|
297
|
+
- New acceptance suite `scheduling`: on every target, the same seed gives
|
|
298
|
+
the same order, other seeds other orders, and a `parallel` unit's laws
|
|
299
|
+
overlap.
|
|
300
|
+
|
|
301
|
+
### Changed
|
|
302
|
+
|
|
303
|
+
- Workflow time is the `Clock` ability's: a law that installs a clock (such
|
|
304
|
+
as `using virtual clock`) runs its workflows on it. On a virtual clock,
|
|
305
|
+
timeouts and hedges count virtual time, so generated workflow tests now
|
|
306
|
+
run with them on, deterministically; before, they were off because they
|
|
307
|
+
used real time.
|
|
308
|
+
- Channel-end `take` tokens are 32 bytes from the operating system's secure
|
|
309
|
+
generator on every target (Haskell's were clock-based).
|
|
310
|
+
- Remote definitions are named by a SHA3-256 content hash that names its
|
|
311
|
+
algorithm: `sha3-256:` then 64 hexadecimal digits.
|
|
312
|
+
- A generated project that imports `lawspec.crypto` or `lawspec.network`
|
|
313
|
+
depends on the default handlers' libraries (in Haskell and Rust,
|
|
314
|
+
`lawspec.randomness` does too, for its secure generator):
|
|
315
|
+
`cryptography` (Python), `@noble/post-quantum` (JavaScript, TypeScript),
|
|
316
|
+
`circl` and Go 1.25 (Go), Bouncy Castle (Java, Kotlin), `crypton`, `mlkem`
|
|
317
|
+
and `mldsa` (Haskell), and RustCrypto's `ml-kem`, `ml-dsa`, `slh-dsa`,
|
|
318
|
+
`sha3`, `shake` and `aes-gcm` (Rust).
|
|
319
|
+
- In Go, a program's ability that shares a name with another unit's (such
|
|
320
|
+
as a built-in `Log`) is named after its unit (`ExampleShopLog`), since
|
|
321
|
+
every Go package holds every ability.
|
|
322
|
+
- A Haskell project depends on `directory`. A Go test file imports rapid only
|
|
323
|
+
when a law draws its inputs with it.
|
|
324
|
+
- Generated tests are named after law labels on every target
|
|
325
|
+
(`test_charges_once__property`, `TestChargesOnce_Property`,
|
|
326
|
+
`lawChargesOnce_property`, `law_charges_once`), unique within a unit. The
|
|
327
|
+
test manifest carries each law's name, tags, skip and known failing.
|
|
328
|
+
- On Java and Kotlin, an ability or handler whose class would be named like
|
|
329
|
+
the class holding its unit's abilities is a compile error that says what to
|
|
330
|
+
rename.
|
|
331
|
+
- The failure database keeps a failing case's inputs in LawSpec's wire
|
|
332
|
+
encoding (`.lawspec/failures/<target>/inputs`), on every target, and the
|
|
333
|
+
law's tests replay them before any new case.
|
|
334
|
+
- Built-in resources acquire and release through `lawspec.host`'s
|
|
335
|
+
abilities, under their production handlers: `FileSystem` gains
|
|
336
|
+
`temporaryDirectory` and `temporaryFile`, `Environment` gains
|
|
337
|
+
`environmentSnapshot` and `restoreEnvironment`, and `FreePort` uses
|
|
338
|
+
`freePort`. A program that names a built-in resource gets `lawspec.host`.
|
|
339
|
+
|
|
340
|
+
### Fixed
|
|
341
|
+
|
|
342
|
+
- Java ability interfaces no longer end a wrapped parameter list with a
|
|
343
|
+
comma; a Rust handler parameter that is a keyword is a raw identifier; a
|
|
344
|
+
Rust ability operation's result is checked against its type.
|
|
345
|
+
- A law that hands a resource it takes to a definition that releases it,
|
|
346
|
+
through any chain of checked definitions or spec handler clauses, is now
|
|
347
|
+
an error, like a law that calls the release itself. The check is a
|
|
348
|
+
conservative data-flow analysis over the whole program.
|
|
349
|
+
- `any` in a strategy of a refined type aims at the refinement, as a law
|
|
350
|
+
input's generator does: the refinement's constant integer bounds narrow
|
|
351
|
+
the range it draws from, instead of drawing the whole type and discarding
|
|
352
|
+
what the refinement rejects.
|
|
353
|
+
- A size-indexed collection input (`SizedStack 3 Int8`, `SizedQueue n a`)
|
|
354
|
+
is generated from its size, as a user family's index directs generation,
|
|
355
|
+
instead of drawing a collection of any size and filtering for the size.
|
|
356
|
+
The family's measure is recognised under the name an importing unit calls
|
|
357
|
+
it by. Python's Hypothesis no longer fails the `collections` suite now and
|
|
358
|
+
then with the `filter_too_much` health check.
|
|
359
|
+
- Generating a program with many data types no longer takes time quadratic
|
|
360
|
+
in their number: each target's native data names count clashes once, and
|
|
361
|
+
each codec and native type maker builds its type registry and names once
|
|
362
|
+
per unit rather than once per adapter argument. Generating every bundled
|
|
363
|
+
example is about three times faster for Haskell and twice for Go and
|
|
364
|
+
Kotlin, with byte-identical output.
|
|
365
|
+
|
|
9
366
|
## [0.20.1] - 2026-10-06
|
|
10
367
|
|
|
11
368
|
### Added
|
|
@@ -15,7 +372,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
15
372
|
requirements the tests verify), `canonical_decisions.yaml` (the design
|
|
16
373
|
decisions, each cited as `ref:KEY`) and `canonical_exemptions.yaml` sit at
|
|
17
374
|
the root, and `canonical_vetting/` holds every comment, page, decision and
|
|
18
|
-
reference as pending
|
|
375
|
+
reference as pending with owner review deferred until after the 0.21 refactor.
|
|
19
376
|
- `make canon` and `lawspec-dev canon` run canon over the repository, and the
|
|
20
377
|
complete check runs it as its `canon` step. Set `CANON_HOME` to a canon
|
|
21
378
|
checkout built with `stack build`; without one the step is skipped with a
|
|
@@ -923,7 +1280,9 @@ A maintenance release. The language, generated code and API are unchanged from
|
|
|
923
1280
|
machine-sized bindings reject an architecture mismatch. Rust adapters take
|
|
924
1281
|
owned values.
|
|
925
1282
|
|
|
926
|
-
[Unreleased]: https://github.com/brain-fuel/lawspec/compare/v0.
|
|
1283
|
+
[Unreleased]: https://github.com/brain-fuel/lawspec/compare/v0.22.0...HEAD
|
|
1284
|
+
[0.22.0]: https://github.com/brain-fuel/lawspec/compare/v0.21.0...v0.22.0
|
|
1285
|
+
[0.21.0]: https://github.com/brain-fuel/lawspec/compare/v0.20.1...v0.21.0
|
|
927
1286
|
[0.20.1]: https://github.com/brain-fuel/lawspec/compare/v0.20.0...v0.20.1
|
|
928
1287
|
[0.20.0]: https://github.com/brain-fuel/lawspec/compare/v0.19.1...v0.20.0
|
|
929
1288
|
[0.19.1]: https://github.com/brain-fuel/lawspec/compare/v0.19.0...v0.19.1
|
package/README.md
CHANGED
|
@@ -30,7 +30,7 @@ Node 22+ and the selected target's build tools are required. No Haskell
|
|
|
30
30
|
toolchain is needed. The reference platforms are macOS and Linux.
|
|
31
31
|
|
|
32
32
|
```sh
|
|
33
|
-
npm install --save-dev lawspec@0.
|
|
33
|
+
npm install --save-dev lawspec@0.22.0
|
|
34
34
|
npx lawspec --version
|
|
35
35
|
```
|
|
36
36
|
|
|
@@ -40,8 +40,8 @@ LawSpec can create `package.json` and its test script:
|
|
|
40
40
|
```sh
|
|
41
41
|
mkdir lawspec-example
|
|
42
42
|
cd lawspec-example
|
|
43
|
-
npm exec --package=lawspec@0.
|
|
44
|
-
npm install --save-dev lawspec@0.
|
|
43
|
+
npm exec --package=lawspec@0.22.0 -- lawspec init --target javascript
|
|
44
|
+
npm install --save-dev lawspec@0.22.0
|
|
45
45
|
npx lawspec check
|
|
46
46
|
npx lawspec explain 'example.atoi_codec::itoa and then atoi yields a'
|
|
47
47
|
npx lawspec doctor
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// Generated from templates/npm/beam-coverage.mjs by lawspec-dev generate. Do not edit.
|
|
2
|
+
import {execFile} from "node:child_process";
|
|
3
|
+
import {createHash} from "node:crypto";
|
|
4
|
+
import {mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile} from "node:fs/promises";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import {promisify} from "node:util";
|
|
8
|
+
|
|
9
|
+
const exec = promisify(execFile);
|
|
10
|
+
export function beamCoverageRequirement(target, report) {
|
|
11
|
+
if (target.language === "gleam" && Number(report?.versions?.gleam?.split(".")[1] ?? 0) < 19)
|
|
12
|
+
return {tool: "Gleam 1.19 for source coverage", install: "upgrade Gleam to 1.19 so native counters refer to the original source lines"};
|
|
13
|
+
if (target.language === "elixir" && report?.coverage?.tool !== "Mix.Tasks.Test.Coverage")
|
|
14
|
+
return {tool: "Mix.Tasks.Test.Coverage", install: "set test_coverage[:tool] to Mix.Tasks.Test.Coverage in mix.exs"};
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const collector = String.raw`#!/usr/bin/env escript
|
|
19
|
+
-mode(compile).
|
|
20
|
+
main([Input, Output]) ->
|
|
21
|
+
try
|
|
22
|
+
{ok, Bytes} = file:read_file(Input),
|
|
23
|
+
#{<<"runs">> := Runs, <<"directory">> := Directory} = json:decode(Bytes),
|
|
24
|
+
{ok, _} = cover:start(),
|
|
25
|
+
[ok = cover:import(text(maps:get(<<"file">>, Run))) || Run <- Runs],
|
|
26
|
+
Beams = lists:foldl(fun(Run, Acc) -> maps:merge(Acc, maps:get(<<"beams">>, Run, #{})) end, #{}, Runs),
|
|
27
|
+
Modules = [analyse(Module, text(Directory), Beams) || Module <- lists:sort(cover:imported_modules())],
|
|
28
|
+
true = Modules =/= [],
|
|
29
|
+
ok = file:write_file(Output, json:encode(Modules)),
|
|
30
|
+
cover:stop()
|
|
31
|
+
catch Class:Reason:Stack ->
|
|
32
|
+
io:format(standard_error, "Cannot collect BEAM coverage: ~tp:~tp~n~tp~n", [Class, Reason, Stack]), halt(1)
|
|
33
|
+
end.
|
|
34
|
+
|
|
35
|
+
analyse(Module, Directory, Beams) ->
|
|
36
|
+
{ok, Calls} = cover:analyse(Module, calls, line),
|
|
37
|
+
Lines = lists:foldl(fun
|
|
38
|
+
({{_, Line}, Hits}, Acc) when Line > 0 -> maps:update_with(Line, fun(N) -> N + Hits end, Hits, Acc);
|
|
39
|
+
(_, Acc) -> Acc
|
|
40
|
+
end, #{}, Calls),
|
|
41
|
+
Name = atom_to_binary(Module),
|
|
42
|
+
Rows = [#{line => Line, hits => Hits} || {Line, Hits} <- lists:sort(maps:to_list(Lines))],
|
|
43
|
+
case maps:find(Name, Beams) of
|
|
44
|
+
{ok, Beam} -> #{name => Name, lines => Rows, source => source(Module, text(Beam))};
|
|
45
|
+
error ->
|
|
46
|
+
File = binary_to_list(binary:encode_hex(crypto:hash(sha256, Name), lowercase)) ++ ".html",
|
|
47
|
+
{ok, _} = cover:analyse_to_file(Module, [html, {outfile, filename:join(Directory, File)}]),
|
|
48
|
+
#{name => Name, lines => Rows, report => unicode:characters_to_binary(File)}
|
|
49
|
+
end.
|
|
50
|
+
|
|
51
|
+
source(Module, Beam) ->
|
|
52
|
+
{ok, {Module, [{compile_info, Info}]}} = beam_lib:chunks(Beam, [compile_info]),
|
|
53
|
+
Source = case proplists:get_value(source, Info) of
|
|
54
|
+
undefined ->
|
|
55
|
+
{ok, {Module, [{abstract_code, {raw_abstract_v1, Forms}}]}} = beam_lib:chunks(Beam, [abstract_code]),
|
|
56
|
+
[File | _] = [F || {attribute, _, file, {F, _}} <- Forms], File;
|
|
57
|
+
File -> File
|
|
58
|
+
end,
|
|
59
|
+
unicode:characters_to_binary(Source).
|
|
60
|
+
text(Value) -> unicode:characters_to_list(Value).
|
|
61
|
+
`;
|
|
62
|
+
|
|
63
|
+
const escape = (value) => String(value).replace(/[&<>"']/g, (c) =>
|
|
64
|
+
({"&": "&", "<": "<", ">": ">", '"': """, "'": "'"}[c]));
|
|
65
|
+
const totals = (lines) => ({covered: lines.filter(({hits}) => hits > 0).length, total: lines.length});
|
|
66
|
+
const percentage = ({covered, total}) => total ? `${(100 * covered / total).toFixed(1)}%` : "no executable lines";
|
|
67
|
+
const page = (title, body) => `<!doctype html><html lang="en"><meta charset="utf-8"><title>${escape(title)}</title>
|
|
68
|
+
<style>body{font:16px system-ui;margin:2rem;color:#222}table{border-collapse:collapse;width:100%}td,th{padding:.3rem .7rem;text-align:left;border-bottom:1px solid #ddd}pre{margin:0;tab-size:4}.hit{background:#e4f5e9}.miss{background:#ffe6e6}.line,.count{width:4rem;text-align:right;color:#555}a{color:#174da8}</style>
|
|
69
|
+
<h1>${escape(title)}</h1>${body}</html>\n`;
|
|
70
|
+
|
|
71
|
+
// Only exports explicitly returned by this run's invocations are imported.
|
|
72
|
+
// A missing export is an error even if the native runner exited successfully.
|
|
73
|
+
export async function collectBeamCoverage({root, directory, runs}) {
|
|
74
|
+
if (!runs.length) return null;
|
|
75
|
+
root = path.resolve(root);
|
|
76
|
+
directory = path.resolve(directory);
|
|
77
|
+
const temporary = await mkdtemp(path.join(os.tmpdir(), "lawspec-beam-coverage-"));
|
|
78
|
+
try {
|
|
79
|
+
const inputs = [];
|
|
80
|
+
const seen = new Set();
|
|
81
|
+
let rebarDirectories;
|
|
82
|
+
for (const run of runs) {
|
|
83
|
+
if (!run.run || seen.has(run.run) || path.basename(run.file) !== `${run.run}.coverdata`)
|
|
84
|
+
throw new Error("Invalid or duplicate BEAM coverage invocation");
|
|
85
|
+
seen.add(run.run);
|
|
86
|
+
const info = await stat(run.file).catch(() => null);
|
|
87
|
+
if (!info?.isFile() || !info.size) throw new Error(`Native runner did not export coverage for ${run.run}: ${run.file}`);
|
|
88
|
+
let beams;
|
|
89
|
+
if (run.tool === "rebar3" && !rebarDirectories) {
|
|
90
|
+
// Before the first compile Rebar only knows dependency ebin paths.
|
|
91
|
+
// Ask after the test run so fresh and relocated applications are found.
|
|
92
|
+
const result = await exec("rebar3", ["as", "test", "path", "--ebin", "--separator", "\n"], {cwd: root});
|
|
93
|
+
rebarDirectories = result.stdout.trim().split("\n").filter(Boolean);
|
|
94
|
+
if (!rebarDirectories.length || !rebarDirectories.every((item) => path.isAbsolute(item)))
|
|
95
|
+
throw new Error("Cannot resolve Rebar's application beam directories after coverage");
|
|
96
|
+
}
|
|
97
|
+
const directories = run.tool === "rebar3" ? rebarDirectories
|
|
98
|
+
: run.beamDirectories ?? (run.beamDirectory ? [run.beamDirectory] : []);
|
|
99
|
+
for (const directory of directories) {
|
|
100
|
+
const entries = (await readdir(directory)).filter((name) => name.endsWith(".beam"));
|
|
101
|
+
beams = {...beams, ...Object.fromEntries(entries.map((name) => [name.slice(0, -5), path.resolve(directory, name)]))};
|
|
102
|
+
}
|
|
103
|
+
if (run.metadata) {
|
|
104
|
+
const metadata = JSON.parse(await readFile(run.metadata, "utf8"));
|
|
105
|
+
if (metadata.run !== run.run || metadata.file !== run.file || !metadata.modules?.length)
|
|
106
|
+
throw new Error(`Gleam coverage metadata does not match invocation ${run.run}`);
|
|
107
|
+
beams = Object.fromEntries(metadata.modules.map(({name, beam}) => [name, beam]));
|
|
108
|
+
}
|
|
109
|
+
inputs.push({file: path.resolve(run.file), ...(beams ? {beams} : {})});
|
|
110
|
+
}
|
|
111
|
+
// Assemble all pages off to the side. A failed import or source mapping
|
|
112
|
+
// cannot leave an apparently complete index behind.
|
|
113
|
+
const pages = path.join(temporary, "modules");
|
|
114
|
+
await mkdir(pages);
|
|
115
|
+
const input = path.join(temporary, "input.json"), output = path.join(temporary, "output.json");
|
|
116
|
+
const script = path.join(temporary, "collect.escript");
|
|
117
|
+
await writeFile(script, collector);
|
|
118
|
+
await writeFile(input, JSON.stringify({runs: inputs, directory: pages}));
|
|
119
|
+
await exec("escript", [script, input, output], {cwd: root, maxBuffer: 8 * 1024 * 1024});
|
|
120
|
+
const modules = JSON.parse(await readFile(output, "utf8"));
|
|
121
|
+
for (const module of modules) {
|
|
122
|
+
module.summary = totals(module.lines);
|
|
123
|
+
if (!module.source) continue;
|
|
124
|
+
let source = path.resolve(root, module.source);
|
|
125
|
+
const text = await readFile(source, "utf8");
|
|
126
|
+
// Gleam copies foreign Erlang sources into its build tree. Prefer the
|
|
127
|
+
// original when it is byte-for-byte the source that was compiled.
|
|
128
|
+
const marker = `${path.sep}_gleam_artefacts${path.sep}`;
|
|
129
|
+
if (source.includes(marker)) {
|
|
130
|
+
const original = path.join(root, "src", source.split(marker).at(-1));
|
|
131
|
+
if (await readFile(original, "utf8").catch(() => null) === text) source = original;
|
|
132
|
+
}
|
|
133
|
+
module.source = path.relative(root, source);
|
|
134
|
+
const sourceLines = text.split(/\r?\n/);
|
|
135
|
+
if (module.lines.some(({line}) => line > sourceLines.length))
|
|
136
|
+
throw new Error(`Coverage lines do not match source ${module.source}`);
|
|
137
|
+
const hits = new Map(module.lines.map(({line, hits}) => [line, hits]));
|
|
138
|
+
module.report = createHash("sha256").update(module.name).digest("hex") + ".html";
|
|
139
|
+
const rows = sourceLines.map((line, index) => {
|
|
140
|
+
const count = hits.get(index + 1);
|
|
141
|
+
return `<tr class="${count === undefined ? "" : count ? "hit" : "miss"}" id="L${index + 1}"><td class="line">${index + 1}</td><td class="count">${count ?? ""}</td><td><pre>${escape(line)}</pre></td></tr>`;
|
|
142
|
+
}).join("\n");
|
|
143
|
+
await writeFile(path.join(pages, module.report), page(module.name,
|
|
144
|
+
`<p>${escape(module.source)} — ${percentage(module.summary)}</p><table>${rows}</table>`));
|
|
145
|
+
}
|
|
146
|
+
const summary = modules.reduce((sum, module) => ({covered: sum.covered + module.summary.covered,
|
|
147
|
+
total: sum.total + module.summary.total}), {covered: 0, total: 0});
|
|
148
|
+
const result = {version: 1, root, runs: runs.map(({run}) => run), summary, modules};
|
|
149
|
+
const rows = modules.map((module) => `<tr><td><a href="modules/${module.report}">${escape(module.name)}</a></td><td>${module.summary.covered}/${module.summary.total}</td><td>${percentage(module.summary)}</td></tr>`).join("\n");
|
|
150
|
+
await mkdir(path.join(directory, "modules"), {recursive: true});
|
|
151
|
+
for (const module of modules)
|
|
152
|
+
await writeFile(path.join(directory, "modules", module.report), await readFile(path.join(pages, module.report)));
|
|
153
|
+
await writeFile(path.join(directory, "coverage.json"), JSON.stringify(result, null, 2) + "\n");
|
|
154
|
+
await writeFile(path.join(directory, "index.html"), page("BEAM line coverage",
|
|
155
|
+
`<p>${summary.covered}/${summary.total} executable lines covered (${percentage(summary)}); ${runs.length} native run(s).</p><table><thead><tr><th>Module</th><th>Lines</th><th>Coverage</th></tr></thead><tbody>${rows}</tbody></table>`));
|
|
156
|
+
return result;
|
|
157
|
+
} finally {
|
|
158
|
+
await rm(temporary, {recursive: true, force: true});
|
|
159
|
+
}
|
|
160
|
+
}
|