@systemfsoftware/xstate-test 0.0.0-stage → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE +21 -0
- package/README.md +1821 -2
- package/dist/adapter-Dd4Go0U3.d.ts +58 -0
- package/dist/effect-schema.d.ts +28 -0
- package/dist/effect-schema.mjs +40 -0
- package/dist/finalizerError-DOptAIDs.mjs +8 -0
- package/dist/index-CSbyUxJr.d.ts +2062 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.mjs +178 -0
- package/dist/playwright.d.ts +204 -0
- package/dist/playwright.mjs +287 -0
- package/dist/propertyTest-5bO0RsUu.mjs +4660 -0
- package/dist/propertyTest-BHnBV8ax.d.ts +147 -0
- package/dist/schema.d.ts +59 -0
- package/dist/schema.mjs +822 -0
- package/dist/vitest.d.ts +74 -0
- package/dist/vitest.mjs +139 -0
- package/package.json +88 -3
package/README.md
CHANGED
|
@@ -1,3 +1,1822 @@
|
|
|
1
|
-
#
|
|
1
|
+
# `@xstate/test`
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> `@systemfsoftware/xstate-test` is forked from [statelyai/xstate](https://github.com/statelyai/xstate) (`packages/xstate-test`, branch `next` at `2146ae26ebfc7e6a624b3a1f237f9e6ddc30b9f5`), MIT. The text below is upstream's.
|
|
4
|
+
|
|
5
|
+
Model-based and property-based testing for XState, built on
|
|
6
|
+
[fast-check](https://fast-check.dev).
|
|
7
|
+
|
|
8
|
+
Describe the behavior you expect as a state machine. `@xstate/test` generates
|
|
9
|
+
event sequences from that machine, sends each sequence to the machine and to
|
|
10
|
+
your implementation, and fails when the two disagree or when a property you
|
|
11
|
+
declared stops holding.
|
|
12
|
+
|
|
13
|
+
Two functions generate the sequences:
|
|
14
|
+
|
|
15
|
+
- `testPaths()` walks the machine's state graph and runs every path it finds.
|
|
16
|
+
- `propertyTest()` generates random sequences and shrinks a failing one to a
|
|
17
|
+
minimal counterexample.
|
|
18
|
+
|
|
19
|
+
Both take the same options, return the same coverage object, and throw the
|
|
20
|
+
same `ModelTestFailure`.
|
|
21
|
+
[`examples/property-testing-cart`](../../examples/property-testing-cart) is the
|
|
22
|
+
complete worked example.
|
|
23
|
+
|
|
24
|
+
- [Installation](#installation)
|
|
25
|
+
- [Quick start](#quick-start)
|
|
26
|
+
- [Concepts](#concepts)
|
|
27
|
+
- [How-to guides](#how-to-guides)
|
|
28
|
+
- [Reference](#reference)
|
|
29
|
+
- [Migrating from `@xstate/test` 0.x and 1.0 beta](#migrating-from-xstatetest-0x-and-10-beta)
|
|
30
|
+
- [Comparison](#comparison)
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pnpm add -D @xstate/test fast-check xstate
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`xstate` and `fast-check` are required peer dependencies. The other peer
|
|
39
|
+
dependencies are optional:
|
|
40
|
+
|
|
41
|
+
| Package | Needed for |
|
|
42
|
+
| ------------------------------------------ | ------------------------------------------------------------------------------------ |
|
|
43
|
+
| `zod` (v3.25+ or v4) | Deriving event generators from Zod event schemas. |
|
|
44
|
+
| `effect` (v4) | Deriving event generators from Effect Schemas, through `@xstate/test/effect-schema`. |
|
|
45
|
+
| `@playwright/test` or `playwright` (1.40+) | `@xstate/test/playwright`. |
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
This section tests a hand-written cart against a cart machine, first with
|
|
50
|
+
`propertyTest()` and then with `testPaths()`.
|
|
51
|
+
|
|
52
|
+
The machine is the model. It says what the cart must do:
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
// cart.machine.ts
|
|
56
|
+
import { createMachine, types } from 'xstate'
|
|
57
|
+
|
|
58
|
+
export const cartMachine = createMachine({
|
|
59
|
+
id: 'cart',
|
|
60
|
+
schemas: {
|
|
61
|
+
context: types<{ items: Record<string, number> }>(),
|
|
62
|
+
events: {
|
|
63
|
+
ADD: types<{ sku: string }>(),
|
|
64
|
+
REMOVE: types<{ sku: string }>(),
|
|
65
|
+
CHECKOUT: types<{}>(),
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
context: { items: {} },
|
|
69
|
+
initial: 'shopping',
|
|
70
|
+
states: {
|
|
71
|
+
shopping: {
|
|
72
|
+
on: {
|
|
73
|
+
ADD: ({ context, event }) => ({
|
|
74
|
+
context: {
|
|
75
|
+
items: {
|
|
76
|
+
...context.items,
|
|
77
|
+
[event.sku]: (context.items[event.sku] ?? 0) + 1,
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
}),
|
|
81
|
+
REMOVE: ({ context, event }) => {
|
|
82
|
+
const { [event.sku]: _removed, ...items } = context.items
|
|
83
|
+
return { context: { items } }
|
|
84
|
+
},
|
|
85
|
+
CHECKOUT: ({ context }) =>
|
|
86
|
+
Object.keys(context.items).length
|
|
87
|
+
? { target: 'checkedOut' }
|
|
88
|
+
: undefined,
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
checkedOut: { type: 'final' },
|
|
92
|
+
},
|
|
93
|
+
})
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The implementation is the code under test:
|
|
97
|
+
|
|
98
|
+
```ts
|
|
99
|
+
// cart.ts
|
|
100
|
+
export function createCart() {
|
|
101
|
+
const items: Record<string, number> = {}
|
|
102
|
+
return {
|
|
103
|
+
add: (sku: string) => {
|
|
104
|
+
items[sku] = (items[sku] ?? 0) + 1
|
|
105
|
+
},
|
|
106
|
+
remove: (sku: string) => {
|
|
107
|
+
delete items[sku]
|
|
108
|
+
},
|
|
109
|
+
items: () => ({ ...items }),
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Describe the events and the SUT
|
|
115
|
+
|
|
116
|
+
`events` says how to generate each event's payload. `sut` says how to send an
|
|
117
|
+
event to the cart and how to compare the cart with the machine:
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
// cart.test.ts
|
|
121
|
+
import { formatTestCoverage, propertyTest, type TestSut } from '@xstate/test'
|
|
122
|
+
import * as fc from 'fast-check'
|
|
123
|
+
import type { EventFrom, SnapshotFrom } from 'xstate'
|
|
124
|
+
import { createCart } from './cart'
|
|
125
|
+
import { cartMachine } from './cart.machine'
|
|
126
|
+
|
|
127
|
+
const sku = fc.constantFrom('apple', 'pear')
|
|
128
|
+
const events = {
|
|
129
|
+
ADD: fc.record({ sku }),
|
|
130
|
+
REMOVE: fc.record({ sku }),
|
|
131
|
+
CHECKOUT: fc.constant({}),
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const cartSut: TestSut<
|
|
135
|
+
SnapshotFrom<typeof cartMachine>,
|
|
136
|
+
EventFrom<typeof cartMachine>
|
|
137
|
+
> = {
|
|
138
|
+
create: () => {
|
|
139
|
+
const cart = createCart()
|
|
140
|
+
return {
|
|
141
|
+
send: (event) => {
|
|
142
|
+
if (event.type === 'ADD') {
|
|
143
|
+
cart.add(event.sku)
|
|
144
|
+
}
|
|
145
|
+
if (event.type === 'REMOVE') {
|
|
146
|
+
cart.remove(event.sku)
|
|
147
|
+
}
|
|
148
|
+
},
|
|
149
|
+
read: () => cart.items(),
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
projectModel: (snapshot) => snapshot.context.items,
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Generate random sequences with `propertyTest()`
|
|
157
|
+
|
|
158
|
+
```ts
|
|
159
|
+
test('the cart matches the model', async () => {
|
|
160
|
+
const { coverage } = await propertyTest(cartMachine, {
|
|
161
|
+
seed: 1,
|
|
162
|
+
numRuns: 100,
|
|
163
|
+
events,
|
|
164
|
+
sut: cartSut,
|
|
165
|
+
})
|
|
166
|
+
console.log(formatTestCoverage(coverage))
|
|
167
|
+
})
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`propertyTest()` generates 100 sequences of `ADD`, `REMOVE`, and `CHECKOUT`.
|
|
171
|
+
For every step, it applies the event to the machine, sends it to the cart
|
|
172
|
+
through `send`, and compares `read()` with `projectModel(snapshot)`. The
|
|
173
|
+
printed coverage starts with:
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
Test coverage
|
|
177
|
+
|
|
178
|
+
states: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
179
|
+
stateNodes: 3/3 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
180
|
+
configurations: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
181
|
+
statuses: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
182
|
+
eventTypes: 4/4 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
183
|
+
transitions: 3/3 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
184
|
+
...
|
|
185
|
+
event cases:
|
|
186
|
+
- ADD / default: 154 generated, 115 applicable, 115 executed, 39 ignored
|
|
187
|
+
- CHECKOUT / default: 170 generated, 126 applicable, 126 executed, 44 ignored
|
|
188
|
+
- REMOVE / default: 147 generated, 112 applicable, 112 executed, 35 ignored
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
An event case is `ignored` when it is not applicable to the current step. Here,
|
|
192
|
+
that is every event generated after `CHECKOUT` reaches the final `checkedOut`
|
|
193
|
+
state and the actor stops. A case is also inapplicable when its `when` returns
|
|
194
|
+
`false`, its `resolve` or `pick()` returns nothing, or `swarm` leaves it out of
|
|
195
|
+
the run. Whether the machine has a transition for the event does not matter:
|
|
196
|
+
`CHECKOUT` on an empty cart is sent to the machine and the cart, the machine
|
|
197
|
+
stays in `shopping`, and the case counts as executed. See [Events](#events) to
|
|
198
|
+
skip such events.
|
|
199
|
+
|
|
200
|
+
### Walk the state graph with `testPaths()`
|
|
201
|
+
|
|
202
|
+
`testPaths()` takes the same `events` and the same `sut`:
|
|
203
|
+
|
|
204
|
+
```ts
|
|
205
|
+
import { testPaths } from '@xstate/test'
|
|
206
|
+
|
|
207
|
+
test('every simple path matches the model', async () => {
|
|
208
|
+
const { coverage, results } = await testPaths(cartMachine, {
|
|
209
|
+
pathGenerator: 'simple',
|
|
210
|
+
events,
|
|
211
|
+
sut: cartSut,
|
|
212
|
+
// Quantities grow without bound; stop expanding at 2.
|
|
213
|
+
stopWhen: (snapshot) => Object.values(snapshot.context.items).some((qty) => qty >= 2),
|
|
214
|
+
})
|
|
215
|
+
console.log(`${results.length} paths`)
|
|
216
|
+
console.log(formatTestCoverage(coverage))
|
|
217
|
+
})
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Each `fc` arbitrary is sampled into three concrete payloads before traversal
|
|
221
|
+
(`samples` defaults to `3`), and each path runs once. The output starts with:
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
15 paths
|
|
225
|
+
Test coverage
|
|
226
|
+
|
|
227
|
+
states: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
228
|
+
stateNodes: 3/3 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
229
|
+
configurations: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
230
|
+
statuses: 2/2 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
231
|
+
eventTypes: 4/4 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
232
|
+
transitions: 3/3 covered (100.0%), 0 uncovered, 0 unreachable, 0 unknown
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
`pathGenerator: 'simple'` is needed here. `REMOVE` of the only item returns
|
|
236
|
+
the cart to a state that is already reachable, so the default `'shortest'`
|
|
237
|
+
generator never takes it.
|
|
238
|
+
|
|
239
|
+
When the cart and the machine disagree, both functions throw a
|
|
240
|
+
`ModelTestFailure` that shows the failing sequence step by step. See
|
|
241
|
+
[What a failure looks like](../../examples/property-testing-cart#what-a-failure-looks-like)
|
|
242
|
+
in the cart example.
|
|
243
|
+
|
|
244
|
+
## Concepts
|
|
245
|
+
|
|
246
|
+
### Model and system under test
|
|
247
|
+
|
|
248
|
+
The **model** is an XState machine that describes the expected behavior. The
|
|
249
|
+
**system under test** (SUT) is the code being tested: a class, a store, a
|
|
250
|
+
service, or a web page. A run starts both, sends them the same events, and
|
|
251
|
+
checks them after every **stable step**, which is the point at which an event
|
|
252
|
+
or command has been fully processed.
|
|
253
|
+
|
|
254
|
+
The model runs in memory. By default it is stepped through XState's pure
|
|
255
|
+
`transition()` function, so no actions run and no actors start.
|
|
256
|
+
|
|
257
|
+
### Events
|
|
258
|
+
|
|
259
|
+
`events` says which events a run may send, and how their payloads are
|
|
260
|
+
produced. Each key is an event type, and each value is one of:
|
|
261
|
+
|
|
262
|
+
- a generator, such as `fc.record({ sku })`. The generator produces the payload
|
|
263
|
+
without `type`; the key supplies `type`. The generator becomes one **event
|
|
264
|
+
case** named `default`.
|
|
265
|
+
- a descriptor object, `{ case, generate, when, resolve, weight }`.
|
|
266
|
+
- an array of generators and descriptors, one event case each.
|
|
267
|
+
|
|
268
|
+
```ts
|
|
269
|
+
events: {
|
|
270
|
+
ADD: ;
|
|
271
|
+
;[
|
|
272
|
+
{ case: 'apple', generate: fc.constant({ sku: 'apple' }) },
|
|
273
|
+
{
|
|
274
|
+
case: 'known-sku',
|
|
275
|
+
generate: fc.constantFrom('apple', 'pear'),
|
|
276
|
+
resolve: ({ generated }) => ({ sku: generated as string }),
|
|
277
|
+
when: ({ snapshot }) => Object.keys(snapshot.context.items).length < 3,
|
|
278
|
+
},
|
|
279
|
+
]
|
|
280
|
+
}
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
| Descriptor field | Purpose |
|
|
284
|
+
| ---------------- | ----------------------------------------------------------------------------------------------------------------------------- |
|
|
285
|
+
| `generate` | Produces the payload, or with `resolve`, a value `resolve` turns into a payload. |
|
|
286
|
+
| `case` | Names the event case in coverage and in `sut.send()`. Defaults to `'default'`. |
|
|
287
|
+
| `when` | Returns `false` to make the case inapplicable for the current snapshot and event. |
|
|
288
|
+
| `resolve` | Turns the generated value into a payload using the current model snapshot. Returning `undefined` makes the case inapplicable. |
|
|
289
|
+
| `weight` | Relative generation frequency under `propertyTest()`. Defaults to `1`. |
|
|
290
|
+
|
|
291
|
+
The runner sends an applicable event even when the current state has no
|
|
292
|
+
transition for it. The machine does not change, the event reaches the SUT, and
|
|
293
|
+
the step checks that the SUT ignores it too. To send only events the machine
|
|
294
|
+
handles in the current state, use `snapshot.can(event)`:
|
|
295
|
+
|
|
296
|
+
```ts
|
|
297
|
+
events: {
|
|
298
|
+
CHECKOUT: {
|
|
299
|
+
generate: fc.constant({}),
|
|
300
|
+
when: ({ snapshot, event }) => snapshot.can(event)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
`CHECKOUT` on an empty cart is then `ignored` rather than executed.
|
|
306
|
+
|
|
307
|
+
`resolve` is for payloads that must refer to the current state, such as an item
|
|
308
|
+
that is already in the cart. fast-check shrinks the generated value, and the
|
|
309
|
+
resolved event is what gets recorded, so replay fixtures do not depend on the
|
|
310
|
+
resolver.
|
|
311
|
+
|
|
312
|
+
`pick(select, toPayload?)` builds such a descriptor. `select` lists the
|
|
313
|
+
candidates in the current snapshot, and `toPayload` turns the picked one into
|
|
314
|
+
the payload; without it, the candidate is the payload. The case is
|
|
315
|
+
inapplicable when `select` returns an empty array. The index into the
|
|
316
|
+
candidates is generated with `fc.nat()`, so a failing run shrinks towards the
|
|
317
|
+
first candidate. The snapshot type is inferred from the machine:
|
|
318
|
+
|
|
319
|
+
```ts
|
|
320
|
+
import { pick } from '@xstate/test'
|
|
321
|
+
|
|
322
|
+
events: {
|
|
323
|
+
REMOVE: pick(
|
|
324
|
+
(snapshot) => Object.keys(snapshot.context.items),
|
|
325
|
+
(sku) => ({ sku }),
|
|
326
|
+
)
|
|
327
|
+
}
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
`propertyTest()` sends only the event types that `events` configures, plus the
|
|
331
|
+
types it derives from schemas (see
|
|
332
|
+
[Derive event generators from schemas](#derive-event-generators-from-schemas)).
|
|
333
|
+
`testPaths()` also offers every event type the machine handles that `events`
|
|
334
|
+
does not configure, as a bare `{ type }` event.
|
|
335
|
+
|
|
336
|
+
Generated payloads must be plain objects. A generator that produces a
|
|
337
|
+
primitive, an array, or `null` fails the run with an error naming the event
|
|
338
|
+
case.
|
|
339
|
+
|
|
340
|
+
### The `sut` option
|
|
341
|
+
|
|
342
|
+
`sut` describes the system under test. Both entry points accept the same
|
|
343
|
+
shape:
|
|
344
|
+
|
|
345
|
+
```ts
|
|
346
|
+
sut: {
|
|
347
|
+
create: () => {
|
|
348
|
+
const cart = createCart();
|
|
349
|
+
return {
|
|
350
|
+
send: (event) => { /* perform the event */ },
|
|
351
|
+
read: () => cart.items(),
|
|
352
|
+
dispose: () => { /* tear down */ }
|
|
353
|
+
};
|
|
354
|
+
},
|
|
355
|
+
projectModel: (snapshot) => snapshot.context.items
|
|
356
|
+
}
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
`create()` runs once per run: once per generated sequence, per shrink attempt,
|
|
360
|
+
or per path. The session it returns is disposed at the end of the run, whether
|
|
361
|
+
the run passed or failed. After every stable step, the runner compares
|
|
362
|
+
`projectSut(read())` with `projectModel(snapshot)` using `equivalent`, which
|
|
363
|
+
defaults to structural, key-order-insensitive deep equality. Omit `read` and `projectModel` to check the SUT only
|
|
364
|
+
through `states` assertions.
|
|
365
|
+
|
|
366
|
+
The [session members](#sut) are listed in the reference.
|
|
367
|
+
|
|
368
|
+
### Oracles
|
|
369
|
+
|
|
370
|
+
An **oracle** decides whether a step is correct. Every oracle runs on every
|
|
371
|
+
stable step:
|
|
372
|
+
|
|
373
|
+
| Oracle | Option | Fails when |
|
|
374
|
+
| ------------------------ | --------------------------------------- | ------------------------------------------------------------------------------- |
|
|
375
|
+
| Invariant | `invariant` | The function throws. |
|
|
376
|
+
| Temporal property | `temporal` | An `always`, `never`, `eventually`, `until`, or `respond` property is violated. |
|
|
377
|
+
| SUT comparison | `sut.projectModel` with `session.read` | The SUT's observation differs from the model's projection. |
|
|
378
|
+
| State assertions | `states`, `session.states`, `meta.test` | An assertion for a matching state throws. |
|
|
379
|
+
| SUT check | `session.check` | The function throws, such as the page oracles of `@xstate/test/playwright`. |
|
|
380
|
+
| Reference implementation | `reference` | A second implementation of the same logic disagrees with the model. |
|
|
381
|
+
|
|
382
|
+
Two oracles are checked once, when the campaign ends, and fail it with a
|
|
383
|
+
`TestCampaignError`:
|
|
384
|
+
|
|
385
|
+
| Oracle | Option | Fails when |
|
|
386
|
+
| -------------------- | ----------- | ----------------------------------------- |
|
|
387
|
+
| `sometimes` property | `temporal` | The predicate held on no step of any run. |
|
|
388
|
+
| Reachability | `reachable` | No run entered a listed state. |
|
|
389
|
+
|
|
390
|
+
`states` is keyed by state value (`'shopping'`, `'form.email'`), by state node
|
|
391
|
+
id (`'#cart.shopping'`), or `'*'`, which runs when no other key matches. A
|
|
392
|
+
`states` map on the session replaces the top-level one. Functions in a state
|
|
393
|
+
node's `meta.test` run too, and receive the session and the snapshot.
|
|
394
|
+
|
|
395
|
+
### Modes
|
|
396
|
+
|
|
397
|
+
`mode` decides how the model runs:
|
|
398
|
+
|
|
399
|
+
- `'pure'` (the default) steps the machine through `transition()`. No action
|
|
400
|
+
runs and no invoked actor starts. Invoke results and `after` delays are
|
|
401
|
+
reached only by sending the internal events for them.
|
|
402
|
+
- `'executed'` runs the machine as a real actor on a `SimulatedClock`. Invoked
|
|
403
|
+
and spawned actors start, `onDone` and `onError` fire, and `after`
|
|
404
|
+
transitions fire when the clock is advanced.
|
|
405
|
+
|
|
406
|
+
See [Steer invoked services](#steer-invoked-services) and
|
|
407
|
+
[Test delayed transitions](#test-delayed-transitions).
|
|
408
|
+
|
|
409
|
+
### `testPaths()` or `propertyTest()`
|
|
410
|
+
|
|
411
|
+
| | `testPaths()` | `propertyTest()` |
|
|
412
|
+
| ---------------------------- | ------------------------------------------------- | ------------------------------------ |
|
|
413
|
+
| Sequences | Paths through the reachable state graph | Random sequences |
|
|
414
|
+
| Payloads | Sampled once before traversal, `samples` per case | Generated per run |
|
|
415
|
+
| On failure | Throws on the first failing path | Throws a shrunk counterexample |
|
|
416
|
+
| `onDone`, `onError`, `after` | Traversed as internal events | Reached with `mode: 'executed'` |
|
|
417
|
+
| Budget | Every path | `numRuns`, or `until` with `maxRuns` |
|
|
418
|
+
| Offline suites | No | `generateTestSuite()` |
|
|
419
|
+
|
|
420
|
+
Use `testPaths()` when the reachable graph is finite and you want every path
|
|
421
|
+
through it. Use `propertyTest()` when payloads, ordering, or timing matter, or
|
|
422
|
+
when the graph is too large to enumerate.
|
|
423
|
+
|
|
424
|
+
### Coverage
|
|
425
|
+
|
|
426
|
+
Both entry points resolve with `{ coverage }`. Coverage counts what the runs
|
|
427
|
+
exercised across these dimensions: `states`, `stateNodes`, `configurations`,
|
|
428
|
+
`statuses`, `eventTypes`, `transitions`, `transitionPairs`, `requirements`,
|
|
429
|
+
and `frontiers`. Each dimension sorts its ids into `covered`,
|
|
430
|
+
`uncovered`, `unreachable`, and `unknown`. Transition hits come from the
|
|
431
|
+
microsteps XState took, not from the states that were visited.
|
|
432
|
+
|
|
433
|
+
Coverage is relative to the events you supplied and the bounds you set. Full
|
|
434
|
+
transition coverage means every declared transition was taken at least once,
|
|
435
|
+
not that every behavior was tested. See [Coverage](#coverage-1) in the
|
|
436
|
+
reference.
|
|
437
|
+
|
|
438
|
+
## How-to guides
|
|
439
|
+
|
|
440
|
+
### Derive event generators from schemas
|
|
441
|
+
|
|
442
|
+
When the machine declares its event payloads as Zod schemas, `propertyTest()`
|
|
443
|
+
and `testPaths()` derive the generators from them:
|
|
444
|
+
|
|
445
|
+
```ts
|
|
446
|
+
import { setup } from 'xstate'
|
|
447
|
+
import * as z from 'zod'
|
|
448
|
+
|
|
449
|
+
const counterMachine = setup({
|
|
450
|
+
schemas: {
|
|
451
|
+
events: {
|
|
452
|
+
INC: z.object({ by: z.number().int().min(1).max(5) }),
|
|
453
|
+
RESET: z.object({}),
|
|
454
|
+
},
|
|
455
|
+
},
|
|
456
|
+
}).createMachine({
|
|
457
|
+
context: { count: 0 },
|
|
458
|
+
on: {
|
|
459
|
+
INC: ({ context, event }) => ({
|
|
460
|
+
context: { count: context.count + event.by },
|
|
461
|
+
}),
|
|
462
|
+
RESET: () => ({ context: { count: 0 } }),
|
|
463
|
+
},
|
|
464
|
+
})
|
|
465
|
+
|
|
466
|
+
await propertyTest(counterMachine, {
|
|
467
|
+
invariant: ({ snapshot }) => {
|
|
468
|
+
expect(snapshot.context.count).toBeGreaterThanOrEqual(0)
|
|
469
|
+
},
|
|
470
|
+
})
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
Declared constraints are honored: this `INC` generator only produces integers
|
|
474
|
+
from 1 to 5. Every event type with a runtime schema that `events` does not
|
|
475
|
+
configure is derived. Entries in `events` override derived ones. Pass
|
|
476
|
+
`deriveEvents: false` to derive nothing.
|
|
477
|
+
|
|
478
|
+
To inspect or edit the derived map, call `eventsFromSchemas()` yourself, and
|
|
479
|
+
merge overrides with `mergeEventGenerators()`:
|
|
480
|
+
|
|
481
|
+
```ts
|
|
482
|
+
import { eventsFromSchemas, mergeEventGenerators } from '@xstate/test'
|
|
483
|
+
|
|
484
|
+
await propertyTest(counterMachine, {
|
|
485
|
+
deriveEvents: false,
|
|
486
|
+
events: mergeEventGenerators(eventsFromSchemas(counterMachine), {
|
|
487
|
+
INC: fc.record({ by: fc.constant(1) }),
|
|
488
|
+
}),
|
|
489
|
+
})
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
For Effect Schemas, import `eventsFromSchemas` from
|
|
493
|
+
`@xstate/test/effect-schema`. `fromEffectSchema(schema)` and
|
|
494
|
+
`fromEffectSchemas(map)` from the same entrypoint convert schemas directly:
|
|
495
|
+
|
|
496
|
+
```ts
|
|
497
|
+
import { fromEffectSchemas } from '@xstate/test/effect-schema'
|
|
498
|
+
import * as Schema from 'effect/Schema'
|
|
499
|
+
|
|
500
|
+
await propertyTest(counterMachine, {
|
|
501
|
+
deriveEvents: false,
|
|
502
|
+
events: fromEffectSchemas({
|
|
503
|
+
INC: Schema.Struct({
|
|
504
|
+
by: Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0))),
|
|
505
|
+
}),
|
|
506
|
+
RESET: Schema.Struct({}),
|
|
507
|
+
}),
|
|
508
|
+
})
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
Schemas that expose no structure cannot be derived from. That includes other
|
|
512
|
+
Standard Schema libraries and type-only `types<...>()` declarations. Implicit
|
|
513
|
+
derivation skips `types<...>()` declarations. For other schemas, configure the
|
|
514
|
+
event in `events`, or pass a `fallback` converter to `eventsFromSchemas()`.
|
|
515
|
+
The supported Zod kinds are listed under
|
|
516
|
+
[`eventsFromSchemas()`](#eventsfromschemas) in the reference.
|
|
517
|
+
|
|
518
|
+
### Steer invoked services
|
|
519
|
+
|
|
520
|
+
Run the machine with `mode: 'executed'` to reach `onDone` and `onError`
|
|
521
|
+
transitions. Replace real services with `actors` or `outcomes`, so a run
|
|
522
|
+
never reaches the network.
|
|
523
|
+
|
|
524
|
+
`actors` substitutes fixed logic through `machine.provide({ actors })`:
|
|
525
|
+
|
|
526
|
+
```ts
|
|
527
|
+
await propertyTest(orderMachine, {
|
|
528
|
+
mode: 'executed',
|
|
529
|
+
actors: {
|
|
530
|
+
chargeCard: createAsyncLogic({ run: async () => ({ id: 'ch_1' }) }),
|
|
531
|
+
},
|
|
532
|
+
events: { SUBMIT: fc.constant({}) },
|
|
533
|
+
})
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
`outcomes` replaces a named invoke source with a stub. The stub stays pending
|
|
537
|
+
until the run generates an `outcome` command for it, so fast-check chooses the
|
|
538
|
+
result and when it arrives, and shrinks both:
|
|
539
|
+
|
|
540
|
+
```ts
|
|
541
|
+
await propertyTest(orderMachine, {
|
|
542
|
+
mode: 'executed',
|
|
543
|
+
outcomes: {
|
|
544
|
+
chargeCard: fc.oneof(
|
|
545
|
+
fc.record({ ok: fc.constant(true as const), output: fc.record({ id: fc.string() }) }),
|
|
546
|
+
fc.record({ ok: fc.constant(false as const), error: fc.constant('declined') }),
|
|
547
|
+
),
|
|
548
|
+
},
|
|
549
|
+
events: { SUBMIT: fc.constant({}) },
|
|
550
|
+
})
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
An outcome is `{ ok: true, output }` or `{ ok: false, error }`. A stub that
|
|
554
|
+
never receives an outcome stays pending, and the run ends with the machine in
|
|
555
|
+
the invoking state. `propertyTest()` rejects `actors` and `outcomes` in pure
|
|
556
|
+
mode.
|
|
557
|
+
|
|
558
|
+
`testPaths()` already traverses `onDone` and `onError`: the state graph
|
|
559
|
+
contains `xstate.done.actor` and `xstate.error.actor` transitions. It samples
|
|
560
|
+
`outcomes` the same way it samples `events`:
|
|
561
|
+
|
|
562
|
+
- In pure mode, it sends the internal event with the sampled `output` or
|
|
563
|
+
`error` as its payload.
|
|
564
|
+
- In executed mode, it stubs every invoke source a path resolves and resolves
|
|
565
|
+
the stub with the sampled outcome for the branch the path took. A branch
|
|
566
|
+
with no matching outcome gets a synthesized one: `{ ok: true, output: undefined }`
|
|
567
|
+
or `{ ok: false, error: new Error('generated failure') }`.
|
|
568
|
+
|
|
569
|
+
A source named in `outcomes` needs no implementation in either mode.
|
|
570
|
+
|
|
571
|
+
```ts
|
|
572
|
+
const { coverage } = await testPaths(orderMachine, {
|
|
573
|
+
mode: 'executed',
|
|
574
|
+
outcomes: {
|
|
575
|
+
chargeCard: fc.constant({ ok: true as const, output: { id: 'ch_1' } }),
|
|
576
|
+
},
|
|
577
|
+
})
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
This run reaches `declined` too: no `ok: false` outcome is declared, so the
|
|
581
|
+
error branch resolves with a synthesized failure.
|
|
582
|
+
|
|
583
|
+
Executed mode is deterministic for everything that goes through the actor
|
|
584
|
+
system and the simulated clock. Real network calls, timers created outside the
|
|
585
|
+
actor clock, `Date.now()`, and `Math.random()` are not intercepted. A step
|
|
586
|
+
settles once the actor system has produced no new events for two consecutive
|
|
587
|
+
macrotask turns. An invoked actor whose promise is still pending at that point
|
|
588
|
+
is listed in the step's `pendingActors`, and
|
|
589
|
+
`coverage.exploration.pendingActorSteps` counts such steps: its result may
|
|
590
|
+
land in a later step.
|
|
591
|
+
|
|
592
|
+
### Test delayed transitions
|
|
593
|
+
|
|
594
|
+
In executed mode, time moves only through `advance` commands. With
|
|
595
|
+
`propertyTest()`, configure a generator for them:
|
|
596
|
+
|
|
597
|
+
```ts
|
|
598
|
+
await propertyTest(orderMachine, {
|
|
599
|
+
mode: 'executed',
|
|
600
|
+
outcomes: { chargeCard: fc.constant({ ok: true as const, output: { id: 'ch_1' } }) },
|
|
601
|
+
events: { SUBMIT: fc.constant({}) },
|
|
602
|
+
commands: { advance: fc.integer({ min: 1_000, max: 10_000 }) },
|
|
603
|
+
})
|
|
604
|
+
```
|
|
605
|
+
|
|
606
|
+
`testPaths()` needs no configuration. Traversal offers only the `after`
|
|
607
|
+
transition that is due first, counting the time already spent in enclosing
|
|
608
|
+
states. In executed mode, each `xstate.after` step advances the clock to that
|
|
609
|
+
timer's due time, read from the actor's scheduler, so a delay computed at
|
|
610
|
+
runtime works too.
|
|
611
|
+
|
|
612
|
+
In pure mode, the model has no clock. A SUT that owns its clock can implement
|
|
613
|
+
`session.advance(ms)` and return the events that fired; the runner applies
|
|
614
|
+
them to the model before the next comparison.
|
|
615
|
+
|
|
616
|
+
### Test a web page with Playwright
|
|
617
|
+
|
|
618
|
+
`createPlaywrightSut(page, config)` from `@xstate/test/playwright` returns a
|
|
619
|
+
`sut` that drives a Playwright page:
|
|
620
|
+
|
|
621
|
+
```ts
|
|
622
|
+
import { test } from '@playwright/test'
|
|
623
|
+
import { propertyTest } from '@xstate/test'
|
|
624
|
+
import { createPlaywrightSut } from '@xstate/test/playwright'
|
|
625
|
+
import * as fc from 'fast-check'
|
|
626
|
+
import { formMachine } from './form.machine'
|
|
627
|
+
|
|
628
|
+
test('the form matches its model', async ({ page }) => {
|
|
629
|
+
await propertyTest(formMachine, {
|
|
630
|
+
numRuns: 25,
|
|
631
|
+
maxCommands: 8,
|
|
632
|
+
events: {
|
|
633
|
+
FILL: fc.record({ value: fc.constantFrom('', 'Ada', 'ada@example.com') }),
|
|
634
|
+
NEXT: fc.constant({}),
|
|
635
|
+
BACK: fc.constant({}),
|
|
636
|
+
},
|
|
637
|
+
sut: createPlaywrightSut(page, {
|
|
638
|
+
reset: async (page) => {
|
|
639
|
+
await page.goto('/')
|
|
640
|
+
},
|
|
641
|
+
events: {
|
|
642
|
+
FILL: (page, event) => page.fill('#field', event.value),
|
|
643
|
+
NEXT: (page) => page.click('#next'),
|
|
644
|
+
BACK: (page) => page.click('#back'),
|
|
645
|
+
},
|
|
646
|
+
read: async (page) => ({
|
|
647
|
+
step: await page.locator('#step').textContent(),
|
|
648
|
+
error: await page.locator('#error').textContent(),
|
|
649
|
+
}),
|
|
650
|
+
projectModel: (snapshot) => ({
|
|
651
|
+
step: String(snapshot.value),
|
|
652
|
+
error: snapshot.context.error,
|
|
653
|
+
}),
|
|
654
|
+
}),
|
|
655
|
+
})
|
|
656
|
+
})
|
|
657
|
+
```
|
|
658
|
+
|
|
659
|
+
Put navigation in `reset`. It runs at the start of every run, and without it a
|
|
660
|
+
run starts from the page the previous run left behind.
|
|
661
|
+
|
|
662
|
+
These defaults apply to every step:
|
|
663
|
+
|
|
664
|
+
- `settle` waits for `page.waitForLoadState('load')`, then for pending
|
|
665
|
+
microtasks, before each comparison. Pass `settle` to wait for something
|
|
666
|
+
specific, such as `(page) => expect(page.locator('#step')).toBeVisible()`.
|
|
667
|
+
Playwright discourages waiting for `'networkidle'`.
|
|
668
|
+
- `advance` calls `page.clock.runFor(ms)`. Call `page.clock.install()` before
|
|
669
|
+
navigating for it to have an effect.
|
|
670
|
+
- The page oracles fail the step when the page threw an uncaught exception,
|
|
671
|
+
logged a console error, left a promise rejection unhandled, or received a
|
|
672
|
+
response with a status of 400 or above.
|
|
673
|
+
|
|
674
|
+
The page oracles are checked after every stable step. Choose them with
|
|
675
|
+
`oracles`. In the object form, oracles you leave out are off:
|
|
676
|
+
|
|
677
|
+
```ts
|
|
678
|
+
sut: createPlaywrightSut(page, {
|
|
679
|
+
events,
|
|
680
|
+
read,
|
|
681
|
+
projectModel,
|
|
682
|
+
// Fail on console warnings too, and only on server errors.
|
|
683
|
+
oracles: { pageError: true, console: 'warn', http: 500, unhandledRejection: true },
|
|
684
|
+
})
|
|
685
|
+
```
|
|
686
|
+
|
|
687
|
+
`oracles: false` turns them all off. Responses to requests that a `mocks`
|
|
688
|
+
route handled are not checked, so a mocked `500` does not fail the step. A
|
|
689
|
+
failing oracle fails the run with `SUT check failed after N steps`, and its
|
|
690
|
+
cause is a `PlaywrightOracleError` whose `messages` list what the page
|
|
691
|
+
reported.
|
|
692
|
+
|
|
693
|
+
To attach failure artifacts to the Playwright report, pass the `testInfo`
|
|
694
|
+
fixture:
|
|
695
|
+
|
|
696
|
+
```ts
|
|
697
|
+
test('the form matches its model', async ({ page }, testInfo) => {
|
|
698
|
+
await propertyTest(formMachine, {
|
|
699
|
+
events,
|
|
700
|
+
sut: createPlaywrightSut(page, {
|
|
701
|
+
events: formActions,
|
|
702
|
+
read,
|
|
703
|
+
projectModel,
|
|
704
|
+
testInfo,
|
|
705
|
+
step: (name, body) => test.step(name, body),
|
|
706
|
+
}),
|
|
707
|
+
})
|
|
708
|
+
})
|
|
709
|
+
```
|
|
710
|
+
|
|
711
|
+
With `testInfo`, every run records a Playwright trace. When the campaign
|
|
712
|
+
fails, the failing run's trace, a `fixture.json` with the `TestFixture`, and
|
|
713
|
+
a screenshot of the page are attached. The failing run is the shrunk
|
|
714
|
+
counterexample. `trace: 'on'` also attaches the last run's trace when the
|
|
715
|
+
campaign passes, and `screenshots: 'every-step'` attaches one screenshot per
|
|
716
|
+
stable step of the failing run. `step` wraps each event action, so the report
|
|
717
|
+
lists one step per event.
|
|
718
|
+
|
|
719
|
+
To stub network calls per event case, use `mocks`. The key is
|
|
720
|
+
`"<type>.<case>"` or `"<case>"`:
|
|
721
|
+
|
|
722
|
+
```ts
|
|
723
|
+
events: {
|
|
724
|
+
SUBMIT: [
|
|
725
|
+
{ case: 'ok', generate: fc.constant({}) },
|
|
726
|
+
{ case: 'error', generate: fc.constant({}) }
|
|
727
|
+
]
|
|
728
|
+
},
|
|
729
|
+
sut: createPlaywrightSut(page, {
|
|
730
|
+
events: { SUBMIT: (page) => page.click('#submit') },
|
|
731
|
+
mocks: {
|
|
732
|
+
'SUBMIT.ok': (page) =>
|
|
733
|
+
page.route('**/api/submit', (route) => route.fulfill({ status: 200 })),
|
|
734
|
+
'SUBMIT.error': (page) =>
|
|
735
|
+
page.route('**/api/submit', (route) => route.fulfill({ status: 500 }))
|
|
736
|
+
},
|
|
737
|
+
read,
|
|
738
|
+
projectModel
|
|
739
|
+
})
|
|
740
|
+
```
|
|
741
|
+
|
|
742
|
+
A mock runs before the event action, only when the resolved case differs from
|
|
743
|
+
the last one applied. Routes a mock installs are removed before another case's
|
|
744
|
+
mock is applied and when the run ends.
|
|
745
|
+
[`examples/property-testing-playwright`](../../examples/property-testing-playwright)
|
|
746
|
+
is a runnable version. All options are listed under
|
|
747
|
+
[`createPlaywrightSut()`](#createplaywrightsut) in the reference.
|
|
748
|
+
|
|
749
|
+
### Run model tests with Vitest
|
|
750
|
+
|
|
751
|
+
`@xstate/test/vitest` registers a model test in one call. `it.model` runs
|
|
752
|
+
`propertyTest()`, and `it.paths` runs `testPaths()`:
|
|
753
|
+
|
|
754
|
+
```ts
|
|
755
|
+
import { it } from '@xstate/test/vitest'
|
|
756
|
+
|
|
757
|
+
it.model('the cart matches the model', cartMachine, {
|
|
758
|
+
events,
|
|
759
|
+
sut: cartSut,
|
|
760
|
+
})
|
|
761
|
+
|
|
762
|
+
it.paths('every path matches the model', cartMachine, {
|
|
763
|
+
pathGenerator: 'simple',
|
|
764
|
+
events,
|
|
765
|
+
sut: cartSut,
|
|
766
|
+
stopWhen: (snapshot) => Object.values(snapshot.context.items).some((qty) => qty >= 2),
|
|
767
|
+
})
|
|
768
|
+
```
|
|
769
|
+
|
|
770
|
+
Each test:
|
|
771
|
+
|
|
772
|
+
- saves failures, keyed by the test file and the test's full name, and
|
|
773
|
+
replays them first on the next run. See
|
|
774
|
+
[Save failures and replay them first](#save-failures-and-replay-them-first).
|
|
775
|
+
Pass `failures: false` to turn this off, or `failures: { dir, replay }` to
|
|
776
|
+
change the directory or the replay mode.
|
|
777
|
+
- sets the test timeout to `until.timeMs` plus 5 seconds when `until` has a
|
|
778
|
+
`timeMs`. A fourth argument sets the timeout explicitly.
|
|
779
|
+
- prints `formatTestCoverage(coverage)` when the campaign fails.
|
|
780
|
+
- stores `testCoverageToJSON(coverage)` as `task.meta.xstateTestCoverage`,
|
|
781
|
+
where custom reporters can read it.
|
|
782
|
+
|
|
783
|
+
`it.model.fails` passes only when the campaign fails, and, with `message`,
|
|
784
|
+
only when the failure message matches:
|
|
785
|
+
|
|
786
|
+
```ts
|
|
787
|
+
it.model.fails('finds the remove bug', cartMachine, buggyOptions, {
|
|
788
|
+
message: /Property observation diverged/,
|
|
789
|
+
})
|
|
790
|
+
```
|
|
791
|
+
|
|
792
|
+
`it` and `test` from `@xstate/test/vitest` use Vitest's global `it` and
|
|
793
|
+
`test`, so they need `test.globals: true`. Without globals, wrap Vitest's own:
|
|
794
|
+
|
|
795
|
+
```ts
|
|
796
|
+
import { withModelTests } from '@xstate/test/vitest'
|
|
797
|
+
import { it as vitestIt } from 'vitest'
|
|
798
|
+
|
|
799
|
+
const it = withModelTests(vitestIt)
|
|
800
|
+
```
|
|
801
|
+
|
|
802
|
+
`vitest` is an optional peer dependency. `@xstate/test/vitest` imports only
|
|
803
|
+
its types.
|
|
804
|
+
|
|
805
|
+
### Check liveness and reachability
|
|
806
|
+
|
|
807
|
+
`always` and `never` check every step. Three more checks cover behavior that
|
|
808
|
+
must happen:
|
|
809
|
+
|
|
810
|
+
- `respond` requires every step on which `trigger` holds to be followed,
|
|
811
|
+
within `within` steps, by a step on which `response` holds. The response may
|
|
812
|
+
hold on the trigger step itself.
|
|
813
|
+
- `sometimes` requires the predicate to hold on at least one step of at least
|
|
814
|
+
one run. A run in which it never holds does not fail; the campaign fails
|
|
815
|
+
when no run satisfied it.
|
|
816
|
+
- `reachable` lists states that at least one run must enter, as state values,
|
|
817
|
+
state node ids, or tags.
|
|
818
|
+
|
|
819
|
+
This cart machine invokes `pay` from a `paying` state, which moves to `done`
|
|
820
|
+
on success and back to `shopping` on failure, as in the
|
|
821
|
+
[cart example](../../examples/property-testing-cart):
|
|
822
|
+
|
|
823
|
+
```ts
|
|
824
|
+
const whileShopping = ({ snapshot }) => snapshot.matches('shopping')
|
|
825
|
+
|
|
826
|
+
await propertyTest(cartMachine, {
|
|
827
|
+
events: {
|
|
828
|
+
ADD: { generate: fc.record({ sku }), when: whileShopping },
|
|
829
|
+
CHECKOUT: { generate: fc.constant({}), when: whileShopping },
|
|
830
|
+
},
|
|
831
|
+
mode: 'executed',
|
|
832
|
+
outcomes: {
|
|
833
|
+
pay: fc.oneof(
|
|
834
|
+
fc.constant({ ok: true as const, output: {} }),
|
|
835
|
+
fc.constant({ ok: false as const, error: 'declined' }),
|
|
836
|
+
),
|
|
837
|
+
},
|
|
838
|
+
temporal: [
|
|
839
|
+
{
|
|
840
|
+
type: 'respond',
|
|
841
|
+
id: 'payment-settles',
|
|
842
|
+
within: 1,
|
|
843
|
+
trigger: ({ snapshot }) => snapshot.matches('paying'),
|
|
844
|
+
response: ({ snapshot }) => !snapshot.matches('paying'),
|
|
845
|
+
},
|
|
846
|
+
{
|
|
847
|
+
type: 'sometimes',
|
|
848
|
+
id: 'declined',
|
|
849
|
+
predicate: ({ snapshot }) => snapshot.context.lastError !== null,
|
|
850
|
+
},
|
|
851
|
+
],
|
|
852
|
+
reachable: ['#cart.done'],
|
|
853
|
+
})
|
|
854
|
+
```
|
|
855
|
+
|
|
856
|
+
`payment-settles` holds because `when` offers the cart's events only while
|
|
857
|
+
shopping: after `CHECKOUT`, the next step is the payment's outcome. Without
|
|
858
|
+
`when`, an `ADD` the machine ignores while paying would be a step without a
|
|
859
|
+
response.
|
|
860
|
+
|
|
861
|
+
When no run satisfies a `sometimes` property or enters a `reachable` state,
|
|
862
|
+
the campaign throws a `TestCampaignError`:
|
|
863
|
+
|
|
864
|
+
```
|
|
865
|
+
Campaign assertions failed:
|
|
866
|
+
- sometimes "declined" did not hold in 100 run(s)
|
|
867
|
+
- reachable "#cart.done" was not entered in 100 run(s)
|
|
868
|
+
```
|
|
869
|
+
|
|
870
|
+
`coverage.temporal.counts` records, per property, how many runs satisfied it,
|
|
871
|
+
failed it, and ended before it was decided. A `reachable` target appears as
|
|
872
|
+
`reachable:<target>`.
|
|
873
|
+
|
|
874
|
+
A bounded property whose `within` is larger than the longest sequence a
|
|
875
|
+
campaign can produce can never fail. The campaign lists it in
|
|
876
|
+
`coverage.temporal.warnings`, and `formatTestCoverage()` prints the warning
|
|
877
|
+
under `temporal`:
|
|
878
|
+
|
|
879
|
+
```
|
|
880
|
+
temporal: 1 satisfied, 0 failed, 0 inconclusive
|
|
881
|
+
- checks-out: 2 satisfied, 0 failed, 23 inconclusive
|
|
882
|
+
warning: eventually "checks-out" has within 20, but the longest sequence is 10 steps, so it can never fail
|
|
883
|
+
```
|
|
884
|
+
|
|
885
|
+
The longest sequence is `maxCommands` for `propertyTest()`, plus the longest
|
|
886
|
+
frontier prefix, and the longest path for `testPaths()`.
|
|
887
|
+
|
|
888
|
+
### Gate CI on coverage
|
|
889
|
+
|
|
890
|
+
`assertTestCoverage()` throws when a dimension's ratio of
|
|
891
|
+
`covered / (covered + uncovered)` is below its threshold. The error message
|
|
892
|
+
contains the formatted report:
|
|
893
|
+
|
|
894
|
+
```ts
|
|
895
|
+
const { coverage } = await propertyTest(cartMachine, { events, sut: cartSut })
|
|
896
|
+
|
|
897
|
+
assertTestCoverage(coverage, { transitions: 1, stateNodes: 1 })
|
|
898
|
+
```
|
|
899
|
+
|
|
900
|
+
To stop a campaign as soon as coverage is reached, use `until`. Runs are
|
|
901
|
+
executed in batches of `batchRuns` (default `25`), up to `maxRuns` (default
|
|
902
|
+
`100`), and the condition is checked between batches:
|
|
903
|
+
|
|
904
|
+
```ts
|
|
905
|
+
const { coverage } = await propertyTest(cartMachine, {
|
|
906
|
+
events,
|
|
907
|
+
sut: cartSut,
|
|
908
|
+
until: { transitions: 1 },
|
|
909
|
+
maxRuns: 500,
|
|
910
|
+
})
|
|
911
|
+
|
|
912
|
+
coverage.exploration.stoppedBecause // 'until', 'budget', or 'failure'
|
|
913
|
+
```
|
|
914
|
+
|
|
915
|
+
Write reports as CI artifacts with the formatters. Each is deterministic for
|
|
916
|
+
the same coverage:
|
|
917
|
+
|
|
918
|
+
```ts
|
|
919
|
+
import { formatTestCoverage, formatTestCoverageHTML, formatTestCoverageJUnit, testCoverageToJSON } from '@xstate/test'
|
|
920
|
+
import { writeFile } from 'node:fs/promises'
|
|
921
|
+
|
|
922
|
+
await writeFile('coverage.md', formatTestCoverage(coverage, { format: 'markdown' }))
|
|
923
|
+
await writeFile('coverage.json', JSON.stringify(testCoverageToJSON(coverage)))
|
|
924
|
+
await writeFile('coverage.xml', formatTestCoverageJUnit(coverage, { suiteName: 'cart' }))
|
|
925
|
+
await writeFile('coverage.html', formatTestCoverageHTML(coverage, { title: 'Cart' }))
|
|
926
|
+
```
|
|
927
|
+
|
|
928
|
+
The JUnit report has one `<testcase>` per transition and per state node:
|
|
929
|
+
`<failure>` for uncovered ids and `<skipped>` for unreachable or unknown ones.
|
|
930
|
+
|
|
931
|
+
To require that generated sequences reach a situation often enough, record it
|
|
932
|
+
with `classify()` or `label()` and set `expectLabels`:
|
|
933
|
+
|
|
934
|
+
```ts
|
|
935
|
+
await propertyTest(cartMachine, {
|
|
936
|
+
events,
|
|
937
|
+
invariant: ({ snapshot, classify }) => {
|
|
938
|
+
classify(Object.keys(snapshot.context.items).length >= 2, 'two-skus')
|
|
939
|
+
},
|
|
940
|
+
expectLabels: { 'two-skus': { min: 0.1 } },
|
|
941
|
+
})
|
|
942
|
+
```
|
|
943
|
+
|
|
944
|
+
`min` is the share of attempted runs, from `0` to `1`, that recorded the label
|
|
945
|
+
at least once. `minCount` is a minimum total number of occurrences. Shrink
|
|
946
|
+
attempts are not counted.
|
|
947
|
+
|
|
948
|
+
### Inspect the distribution of generated data
|
|
949
|
+
|
|
950
|
+
`statistics: true` prints how executed events split across event cases, and
|
|
951
|
+
the share of runs that recorded each label, after a passing campaign:
|
|
952
|
+
|
|
953
|
+
```ts
|
|
954
|
+
await propertyTest(cartMachine, {
|
|
955
|
+
seed: 1,
|
|
956
|
+
events,
|
|
957
|
+
statistics: true,
|
|
958
|
+
invariant: ({ snapshot, classify }) => {
|
|
959
|
+
classify(Object.keys(snapshot.context.items).length >= 2, 'two-skus')
|
|
960
|
+
},
|
|
961
|
+
})
|
|
962
|
+
```
|
|
963
|
+
|
|
964
|
+
```
|
|
965
|
+
Test statistics (100 runs)
|
|
966
|
+
|
|
967
|
+
event cases (share of executed events):
|
|
968
|
+
32.6% ADD / default: 115 executed, 39 ignored
|
|
969
|
+
35.7% CHECKOUT / default: 126 executed, 44 ignored
|
|
970
|
+
31.7% REMOVE / default: 112 executed, 35 ignored
|
|
971
|
+
|
|
972
|
+
labels (share of runs):
|
|
973
|
+
14.0% two-skus: 26 recorded
|
|
974
|
+
```
|
|
975
|
+
|
|
976
|
+
`formatTestStatistics(coverage)` returns the same text for any coverage
|
|
977
|
+
object. Runs started while fast-check shrinks a counterexample are left out
|
|
978
|
+
of the labels and the event cases; `coverage.exploration.shrinkRuns` counts
|
|
979
|
+
them.
|
|
980
|
+
|
|
981
|
+
Pass a function instead of `true` to receive the same text rather than print
|
|
982
|
+
it, for example to attach it to a test report. The campaign awaits it when it
|
|
983
|
+
returns a promise. If it throws or rejects, the campaign fails with that error
|
|
984
|
+
after the `sut.complete` hook ran:
|
|
985
|
+
|
|
986
|
+
```ts
|
|
987
|
+
await propertyTest(cartMachine, {
|
|
988
|
+
events,
|
|
989
|
+
statistics: async (report) => {
|
|
990
|
+
await writeFile('statistics.txt', report)
|
|
991
|
+
},
|
|
992
|
+
})
|
|
993
|
+
```
|
|
994
|
+
|
|
995
|
+
### Replay a failure
|
|
996
|
+
|
|
997
|
+
A `ModelTestFailure` carries a `fixture`: a JSON-safe record of the sequence
|
|
998
|
+
that failed. Save it, and replay it with `replayTest()`:
|
|
999
|
+
|
|
1000
|
+
```ts
|
|
1001
|
+
import { ModelTestFailure, replayTest, type TestFixture } from '@xstate/test'
|
|
1002
|
+
import fixture from './cart-failure.json'
|
|
1003
|
+
|
|
1004
|
+
test('regression: removing an item', async () => {
|
|
1005
|
+
await expect(
|
|
1006
|
+
replayTest(cartMachine, fixture as TestFixture, { sut: cartSut }),
|
|
1007
|
+
).rejects.toBeInstanceOf(ModelTestFailure)
|
|
1008
|
+
})
|
|
1009
|
+
```
|
|
1010
|
+
|
|
1011
|
+
`replayTest()` stops at the recorded failing step and throws the reproduced
|
|
1012
|
+
failure. When the failure no longer happens, it throws
|
|
1013
|
+
`ReplayNotReproducedError`. Pass `expect: 'pass'` to require that the whole
|
|
1014
|
+
fixture replays without failing, which is how a fixed regression is kept:
|
|
1015
|
+
|
|
1016
|
+
```ts
|
|
1017
|
+
await replayTest(cartMachine, fixture as TestFixture, {
|
|
1018
|
+
sut: cartSut,
|
|
1019
|
+
expect: 'pass',
|
|
1020
|
+
})
|
|
1021
|
+
```
|
|
1022
|
+
|
|
1023
|
+
Fixtures from executed runs record every invoke outcome, and replay stubs
|
|
1024
|
+
those sources, so no real service is called. The machine's `id` and `version`
|
|
1025
|
+
are checked against the fixture.
|
|
1026
|
+
|
|
1027
|
+
To rerun the same fast-check sequence instead, pass the recorded `seed`,
|
|
1028
|
+
`path`, and `replayPath` back to `propertyTest()`:
|
|
1029
|
+
|
|
1030
|
+
```ts
|
|
1031
|
+
await propertyTest(cartMachine, {
|
|
1032
|
+
events,
|
|
1033
|
+
sut: cartSut,
|
|
1034
|
+
seed: failure.replay!.seed,
|
|
1035
|
+
path: failure.replay!.path,
|
|
1036
|
+
replayPath: failure.replay!.replayPath,
|
|
1037
|
+
})
|
|
1038
|
+
```
|
|
1039
|
+
|
|
1040
|
+
This depends on the same generators and the same fast-check version, which
|
|
1041
|
+
the fixture does not.
|
|
1042
|
+
|
|
1043
|
+
### Save failures and replay them first
|
|
1044
|
+
|
|
1045
|
+
`failures` saves the fixture of a failing campaign to disk, and replays saved
|
|
1046
|
+
fixtures before the next campaign:
|
|
1047
|
+
|
|
1048
|
+
```ts
|
|
1049
|
+
await propertyTest(cartMachine, {
|
|
1050
|
+
events,
|
|
1051
|
+
sut: cartSut,
|
|
1052
|
+
failures: { dir: '.xstate-test', key: 'cart-store' },
|
|
1053
|
+
})
|
|
1054
|
+
```
|
|
1055
|
+
|
|
1056
|
+
A failing campaign writes `<dir>/<key>/<hash>.json`, with the fixture, the
|
|
1057
|
+
summary, and the fast-check seed, and adds a line to the failure message.
|
|
1058
|
+
`<hash>` is the first 12 hexadecimal digits of the fixture's SHA-256 hash:
|
|
1059
|
+
|
|
1060
|
+
```
|
|
1061
|
+
Saved: .xstate-test/cart-store/<hash>.json
|
|
1062
|
+
```
|
|
1063
|
+
|
|
1064
|
+
The next campaign replays every saved fixture with `replayTest()` before it
|
|
1065
|
+
generates anything. A fixture that still fails throws that failure at once,
|
|
1066
|
+
with `(replayed from <file>)` added to its summary. A fixture that no longer
|
|
1067
|
+
fails is deleted, and the campaign runs.
|
|
1068
|
+
|
|
1069
|
+
| Option | Default | Description |
|
|
1070
|
+
| -------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1071
|
+
| `dir` | `'.xstate-test'` | The directory failures are saved in. |
|
|
1072
|
+
| `replay` | `'first'` | `'first'` replays saved fixtures, then runs the campaign. `'only'` replays them and skips the campaign. `false` saves without replaying. |
|
|
1073
|
+
| `key` | machine id and a hash | The subdirectory of `dir`. Defaults to the machine's `id` plus a hash of the event cases, the oracles configured, and the mode. |
|
|
1074
|
+
|
|
1075
|
+
`failures: true` uses the defaults. `testPaths()` accepts the same option.
|
|
1076
|
+
Two tests that run the same machine with different oracles, such as a
|
|
1077
|
+
`sut` that differs only in its implementation, share the default key; set
|
|
1078
|
+
`key` so each replays only its own failures.
|
|
1079
|
+
|
|
1080
|
+
Add `.xstate-test/` to `.gitignore` to keep failures local, or commit it so
|
|
1081
|
+
CI replays the failures found on developer machines first. A run that
|
|
1082
|
+
starts from a `start` snapshot is not saved.
|
|
1083
|
+
|
|
1084
|
+
`createFailureDatabase(options)` returns the store `failures` creates. Any
|
|
1085
|
+
object with `load(key)`, `onFailure(fixture, key, failure)`, and optionally
|
|
1086
|
+
`remove(stored, key)` works as a store, such as one backed by a database;
|
|
1087
|
+
this is the `TestFailureStore` interface.
|
|
1088
|
+
|
|
1089
|
+
### Record an offline regression suite
|
|
1090
|
+
|
|
1091
|
+
`generateTestSuite()` runs a passing campaign and keeps a small set of fixtures
|
|
1092
|
+
that preserves the covered transitions and state nodes:
|
|
1093
|
+
|
|
1094
|
+
```ts
|
|
1095
|
+
import { generateTestSuite, serializeTestSuite } from '@xstate/test'
|
|
1096
|
+
import { writeFile } from 'node:fs/promises'
|
|
1097
|
+
|
|
1098
|
+
const suite = await generateTestSuite(cartMachine, {
|
|
1099
|
+
seed: 1,
|
|
1100
|
+
numRuns: 200,
|
|
1101
|
+
events,
|
|
1102
|
+
sut: cartSut,
|
|
1103
|
+
})
|
|
1104
|
+
await writeFile('cart.suite.json', serializeTestSuite(suite))
|
|
1105
|
+
```
|
|
1106
|
+
|
|
1107
|
+
Replaying generates nothing, so the suite runs the same way every time:
|
|
1108
|
+
|
|
1109
|
+
```ts
|
|
1110
|
+
import { describeTestSuite, parseTestSuite } from '@xstate/test'
|
|
1111
|
+
import { readFile } from 'node:fs/promises'
|
|
1112
|
+
|
|
1113
|
+
const suite = parseTestSuite(await readFile('cart.suite.json', 'utf8'))
|
|
1114
|
+
|
|
1115
|
+
describeTestSuite(suite, cartMachine, { invariant: () => {}, sut: cartSut })
|
|
1116
|
+
```
|
|
1117
|
+
|
|
1118
|
+
`describeTestSuite()` registers one test per fixture with the global `it` and
|
|
1119
|
+
`describe` of Vitest or Jest; pass `it` and `describe` to use others.
|
|
1120
|
+
`replayTestSuite()` replays every fixture and resolves with
|
|
1121
|
+
`{ passed, failed }` instead. Every fixture is expected to pass.
|
|
1122
|
+
|
|
1123
|
+
### Test concurrency
|
|
1124
|
+
|
|
1125
|
+
A sequential run sends one event at a time, so it never observes a race. Three
|
|
1126
|
+
tools look for concurrency bugs.
|
|
1127
|
+
|
|
1128
|
+
`scheduler: true` gives every run an `fc.scheduler()`. Wrap the SUT with
|
|
1129
|
+
`withScheduledSut()` and its `send`, `read`, `settle`, and `advance` calls
|
|
1130
|
+
resolve in an order fast-check chooses. Schedule the SUT's own background work
|
|
1131
|
+
on `getCurrentScheduler()`. This SUT commits each increment asynchronously, so
|
|
1132
|
+
some orderings let `read()` return a stale count, and the campaign fails:
|
|
1133
|
+
|
|
1134
|
+
```ts
|
|
1135
|
+
import { getCurrentScheduler, withScheduledSut } from '@xstate/test'
|
|
1136
|
+
|
|
1137
|
+
await propertyTest(counterMachine, {
|
|
1138
|
+
scheduler: true,
|
|
1139
|
+
deriveEvents: false,
|
|
1140
|
+
events: { INC: fc.record({ by: fc.constant(1) }) },
|
|
1141
|
+
sut: withScheduledSut({
|
|
1142
|
+
create: () => {
|
|
1143
|
+
const scheduler = getCurrentScheduler()!
|
|
1144
|
+
let count = 0
|
|
1145
|
+
return {
|
|
1146
|
+
send: () => {
|
|
1147
|
+
void scheduler.schedule(Promise.resolve(), 'commit').then(() => {
|
|
1148
|
+
count++
|
|
1149
|
+
})
|
|
1150
|
+
},
|
|
1151
|
+
read: () => count,
|
|
1152
|
+
}
|
|
1153
|
+
},
|
|
1154
|
+
projectModel: (snapshot) => snapshot.context.count,
|
|
1155
|
+
}),
|
|
1156
|
+
})
|
|
1157
|
+
```
|
|
1158
|
+
|
|
1159
|
+
The ordering of a failing run is recorded as `failure.replay.data.scheduler`.
|
|
1160
|
+
Rebuild it with `fc.schedulerFor(ordering)`. Only work that goes through the
|
|
1161
|
+
scheduler is reordered.
|
|
1162
|
+
|
|
1163
|
+
`checkLinearizable(history, model)` decides whether a history of overlapping
|
|
1164
|
+
operations could have come from some sequential order:
|
|
1165
|
+
|
|
1166
|
+
```ts
|
|
1167
|
+
import { checkLinearizable } from '@xstate/test'
|
|
1168
|
+
|
|
1169
|
+
const result = checkLinearizable(
|
|
1170
|
+
[
|
|
1171
|
+
{ id: 'a', invocation: { type: 'write', value: 1 }, response: undefined, start: 0, end: 4 },
|
|
1172
|
+
{ id: 'b', invocation: { type: 'read' }, response: 1, start: 1, end: 5 },
|
|
1173
|
+
],
|
|
1174
|
+
{
|
|
1175
|
+
initial: 0,
|
|
1176
|
+
apply: (state: number, event: { type: string; value?: number }) =>
|
|
1177
|
+
event.type === 'write'
|
|
1178
|
+
? { state: event.value!, response: undefined }
|
|
1179
|
+
: { state, response: state },
|
|
1180
|
+
},
|
|
1181
|
+
)
|
|
1182
|
+
|
|
1183
|
+
result.linearizable // true
|
|
1184
|
+
result.witness // the sequential order that explains the history
|
|
1185
|
+
```
|
|
1186
|
+
|
|
1187
|
+
`truncated: true` means the search stopped at `maxExplored` (default `100000`)
|
|
1188
|
+
before deciding. It is not proof of a bug.
|
|
1189
|
+
|
|
1190
|
+
`runParallelPropertyCommands(machine, options)` runs a sequential `prefix`,
|
|
1191
|
+
then runs `branches` concurrently against the SUT, and checks the recorded
|
|
1192
|
+
history against the machine's own `transition()`:
|
|
1193
|
+
|
|
1194
|
+
```ts
|
|
1195
|
+
import { runParallelPropertyCommands } from '@xstate/test'
|
|
1196
|
+
|
|
1197
|
+
const result = await runParallelPropertyCommands(counterMachine, {
|
|
1198
|
+
prefix: [{ type: 'INC', by: 1 }],
|
|
1199
|
+
branches: [[{ type: 'INC', by: 1 }], [{ type: 'INC', by: 2 }]],
|
|
1200
|
+
sut: {
|
|
1201
|
+
create: () => {
|
|
1202
|
+
let count = 0
|
|
1203
|
+
return {
|
|
1204
|
+
send: async (event) => (count += event.type === 'INC' ? event.by : 0),
|
|
1205
|
+
}
|
|
1206
|
+
},
|
|
1207
|
+
projectModel: (snapshot) => snapshot.context.count,
|
|
1208
|
+
},
|
|
1209
|
+
})
|
|
1210
|
+
|
|
1211
|
+
result.linearizable // true
|
|
1212
|
+
```
|
|
1213
|
+
|
|
1214
|
+
When `send` resolves to `undefined`, the response is read with `read()`.
|
|
1215
|
+
|
|
1216
|
+
### Steer exploration
|
|
1217
|
+
|
|
1218
|
+
These `propertyTest()` options change which sequences a campaign generates.
|
|
1219
|
+
|
|
1220
|
+
`weight` on an event case or command changes how often it is drawn, relative
|
|
1221
|
+
to the others:
|
|
1222
|
+
|
|
1223
|
+
```ts
|
|
1224
|
+
events: {
|
|
1225
|
+
ADD: { generate: fc.record({ sku }), weight: 5 },
|
|
1226
|
+
REMOVE: fc.record({ sku }),
|
|
1227
|
+
CHECKOUT: { generate: fc.constant({}), weight: 0.5 }
|
|
1228
|
+
}
|
|
1229
|
+
```
|
|
1230
|
+
|
|
1231
|
+
`frontiers` starts runs from paths that reach a state of interest. The prefix
|
|
1232
|
+
is replayed as-is, and shrinking only shortens the generated continuation:
|
|
1233
|
+
|
|
1234
|
+
```ts
|
|
1235
|
+
import { getShortestPaths } from 'xstate/graph'
|
|
1236
|
+
|
|
1237
|
+
await propertyTest(cartMachine, {
|
|
1238
|
+
events,
|
|
1239
|
+
frontiers: {
|
|
1240
|
+
paths: getShortestPaths(cartMachine, {
|
|
1241
|
+
toState: (snapshot) => Object.keys(snapshot.context.items).length > 0,
|
|
1242
|
+
stopWhen: (snapshot) => Object.values(snapshot.context.items).some((qty) => qty >= 2),
|
|
1243
|
+
}),
|
|
1244
|
+
runsPerFrontier: 50,
|
|
1245
|
+
},
|
|
1246
|
+
})
|
|
1247
|
+
```
|
|
1248
|
+
|
|
1249
|
+
`frontiers: 'auto'` does this for coverage. Between batches, it finds the
|
|
1250
|
+
shortest path to each state node that still owns an uncovered transition and
|
|
1251
|
+
uses those paths as prefixes. Combine it with `until`:
|
|
1252
|
+
|
|
1253
|
+
```ts
|
|
1254
|
+
await propertyTest(cartMachine, {
|
|
1255
|
+
events,
|
|
1256
|
+
frontiers: 'auto',
|
|
1257
|
+
until: { transitions: 1 },
|
|
1258
|
+
maxRuns: 200,
|
|
1259
|
+
})
|
|
1260
|
+
```
|
|
1261
|
+
|
|
1262
|
+
`swarm: true` enables a seeded random subset of at least half the event cases
|
|
1263
|
+
in each run, so frequent events do not crowd out rare combinations. The
|
|
1264
|
+
enabled cases are recorded on the trace and the fixture, and are frozen while
|
|
1265
|
+
a failure shrinks.
|
|
1266
|
+
|
|
1267
|
+
`target` hill-climbs toward high values of a number. With
|
|
1268
|
+
`frontiers: { strategy: 'target' }`, the prefixes that reached the best values
|
|
1269
|
+
become the next batch's frontiers:
|
|
1270
|
+
|
|
1271
|
+
```ts
|
|
1272
|
+
const { coverage } = await propertyTest(cartMachine, {
|
|
1273
|
+
events,
|
|
1274
|
+
target: ({ snapshot }) => snapshot.context.items.apple ?? 0,
|
|
1275
|
+
frontiers: { strategy: 'target' },
|
|
1276
|
+
until: (coverage) => coverage.exploration.target.best >= 8,
|
|
1277
|
+
maxRuns: 400,
|
|
1278
|
+
})
|
|
1279
|
+
```
|
|
1280
|
+
|
|
1281
|
+
### Start from a snapshot or input
|
|
1282
|
+
|
|
1283
|
+
`input` is passed to the machine's initial transition in every run. `start`
|
|
1284
|
+
starts every run from an existing snapshot instead, and requires
|
|
1285
|
+
`serializeSnapshot` so fixtures can record the start:
|
|
1286
|
+
|
|
1287
|
+
```ts
|
|
1288
|
+
const [cartWithApple] = transition(
|
|
1289
|
+
cartMachine,
|
|
1290
|
+
initialTransition(cartMachine)[0],
|
|
1291
|
+
{ type: 'ADD', sku: 'apple' },
|
|
1292
|
+
)
|
|
1293
|
+
|
|
1294
|
+
await propertyTest(cartMachine, {
|
|
1295
|
+
events,
|
|
1296
|
+
start: {
|
|
1297
|
+
snapshot: cartWithApple,
|
|
1298
|
+
serializeSnapshot: (snapshot) => snapshot.context,
|
|
1299
|
+
},
|
|
1300
|
+
})
|
|
1301
|
+
```
|
|
1302
|
+
|
|
1303
|
+
Replaying such a fixture requires `restoreSnapshot`, which turns the recorded
|
|
1304
|
+
value back into a snapshot.
|
|
1305
|
+
|
|
1306
|
+
### Write a custom adapter
|
|
1307
|
+
|
|
1308
|
+
`propertyTest()` uses the built-in fast-check adapter unless you pass
|
|
1309
|
+
`adapter`: an object with a
|
|
1310
|
+
`run(request)` method that drives runners and returns a result. See
|
|
1311
|
+
[`TestAdapter`](#testadapter) in the reference for the request and the runner
|
|
1312
|
+
lifecycle, and
|
|
1313
|
+
[`test/randomAdapter.ts`](./test/randomAdapter.ts) for a dependency-free
|
|
1314
|
+
adapter without shrinking.
|
|
1315
|
+
|
|
1316
|
+
## Reference
|
|
1317
|
+
|
|
1318
|
+
### Exports
|
|
1319
|
+
|
|
1320
|
+
The fast-check-backed entry points and helpers:
|
|
1321
|
+
|
|
1322
|
+
| Export | Description |
|
|
1323
|
+
| ------------------------------------------------------------ | ---------------------------------------------------------------------------------------- |
|
|
1324
|
+
| `propertyTest(source, options)` | Runs a random-sequence campaign. Resolves with `{ coverage }`. |
|
|
1325
|
+
| `testPaths(source, options?)` | Runs generated paths. Resolves with `{ coverage, results }`. |
|
|
1326
|
+
| `generateTestSuite(source, options)` | Records an offline suite from a passing campaign. |
|
|
1327
|
+
| `pick(select, toPayload?)` | An event case that picks its payload from the current snapshot, with a fast-check index. |
|
|
1328
|
+
| `createFailureDatabase(options?)` | The file-system store behind the `failures` option. |
|
|
1329
|
+
| `fastCheckAdapter(options?)` | The fast-check `TestAdapter`. |
|
|
1330
|
+
| `extractReplayPath(counterexample)` | Reads `replayPath` from a raw `fc.commands()` counterexample. |
|
|
1331
|
+
| `eventsFromSchemas(machine, options?)` | Derives the `events` map from `schemas.events`. |
|
|
1332
|
+
| `arbitraryFromSchema(schema, options?)` | Converts one Zod schema to an arbitrary. |
|
|
1333
|
+
| `mergeEventGenerators(derived, explicit)` | Merges two `events` maps; explicit entries win. |
|
|
1334
|
+
| `withScheduledSut(sut)`, `withScheduledReference(reference)` | Route async boundaries through the run's scheduler. |
|
|
1335
|
+
| `getCurrentScheduler()` | The running `fc.Scheduler`, or `undefined`. |
|
|
1336
|
+
|
|
1337
|
+
`propertyTest()`, `testPaths()`, and `generateTestSuite()` take fast-check
|
|
1338
|
+
options at the top level, derive events from schemas, and accept
|
|
1339
|
+
`failures: true`, `{ dir, replay, key }`, or a `TestFailureStore`.
|
|
1340
|
+
|
|
1341
|
+
Replay, reporting, suites, and linearizability:
|
|
1342
|
+
|
|
1343
|
+
| Export | Description |
|
|
1344
|
+
| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |
|
|
1345
|
+
| `replayTest(source, fixture, options)` | Replays a `TestFixture`. |
|
|
1346
|
+
| `ModelTestFailure`, `ReplayNotReproducedError` | Error classes. |
|
|
1347
|
+
| `serializeTestTrace(trace)` | Renders a trace as JSON-safe data. |
|
|
1348
|
+
| `formatTestCoverage`, `formatTestCoverageJUnit`, `formatTestCoverageHTML`, `testCoverageToJSON` | Coverage reports. |
|
|
1349
|
+
| `formatTestStatistics(coverage)` | The event-case and label distribution `statistics` reports. |
|
|
1350
|
+
| `TestCampaignError` | Thrown when a `sometimes` property or a `reachable` target is never satisfied. |
|
|
1351
|
+
| `assertTestCoverage(coverage, thresholds)` | Throws when coverage is below thresholds. |
|
|
1352
|
+
| `replayTestSuite`, `replayTestSuiteFixture`, `describeTestSuite`, `serializeTestSuite`, `parseTestSuite` | Offline suites. |
|
|
1353
|
+
| `checkLinearizable(history, model, options?)` | Linearizability check. |
|
|
1354
|
+
| `runParallelPropertyCommands(machine, options)` | Concurrent branches checked for linearizability. |
|
|
1355
|
+
|
|
1356
|
+
Subpath entrypoints:
|
|
1357
|
+
|
|
1358
|
+
| Entrypoint | Exports |
|
|
1359
|
+
| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1360
|
+
| `@xstate/test/playwright` | `createPlaywrightSut`, `PlaywrightOracleError`, and the types `PlaywrightPage`, `PlaywrightSutConfig`, `PlaywrightEventAction`, `PlaywrightMock`, `PlaywrightOracles`, `PlaywrightTestInfo`. |
|
|
1361
|
+
| `@xstate/test/vitest` | `it` and `test` with `.model` and `.paths`, and `withModelTests`. |
|
|
1362
|
+
| `@xstate/test/effect-schema` | `eventsFromSchemas` with Effect Schema support, `fromEffectSchema`, `fromEffectSchemas`. |
|
|
1363
|
+
| `@xstate/test/schema` | `eventsFromSchemas`, `arbitraryFromSchema`, `mergeEventGenerators`. |
|
|
1364
|
+
|
|
1365
|
+
### Shared options
|
|
1366
|
+
|
|
1367
|
+
`propertyTest()`, `testPaths()`, and `generateTestSuite()` accept these:
|
|
1368
|
+
|
|
1369
|
+
| Option | Default | Description |
|
|
1370
|
+
| -------------- | -------- | --------------------------------------------------------------------------------------------------------------- |
|
|
1371
|
+
| `events` | `{}` | Event types and their generators. See [Events](#events). |
|
|
1372
|
+
| `deriveEvents` | `true` | Derives generators for event types with a runtime schema that `events` does not configure. `@xstate/test` only. |
|
|
1373
|
+
| `sut` | none | The system under test. See [`sut`](#sut). |
|
|
1374
|
+
| `states` | none | Per-state assertions, run after every stable step. |
|
|
1375
|
+
| `invariant` | none | `(context) => void`. Throws to fail the step. |
|
|
1376
|
+
| `temporal` | `[]` | Temporal properties. See [Temporal properties](#temporal-properties). |
|
|
1377
|
+
| `reachable` | none | State values, `'#id'` state node ids, or tags that at least one run must enter. |
|
|
1378
|
+
| `failures` | none | Saves failing fixtures and replays them first. `true`, `{ dir?, replay?, key? }`, or a `TestFailureStore`. |
|
|
1379
|
+
| `statistics` | `false` | `true` prints `formatTestStatistics(coverage)` after a passing campaign; a function receives it instead. |
|
|
1380
|
+
| `reference` | none | A second implementation compared with the model. |
|
|
1381
|
+
| `mode` | `'pure'` | `'pure'` or `'executed'`. See [Modes](#modes). |
|
|
1382
|
+
| `actors` | none | Logic substituted for named actor sources. Executed mode only. |
|
|
1383
|
+
| `outcomes` | none | Invoke sources replaced by stubs, with generators of `{ ok, output }` or `{ ok, error }`. |
|
|
1384
|
+
| `input` | none | Machine input for every run. |
|
|
1385
|
+
| `start` | none | `{ snapshot, serializeSnapshot }` to start every run from a snapshot. |
|
|
1386
|
+
| `target` | none | `(context) => number`, recorded on every stable step for targeted search. |
|
|
1387
|
+
| `collect` | none | `(trace, { passed, runIndex })`, called after every run. |
|
|
1388
|
+
| `expectLabels` | none | `{ [label]: { min?, minCount? } }`. Fails the campaign when a label is too rare. |
|
|
1389
|
+
|
|
1390
|
+
The `invariant`, `temporal`, and `target` functions receive a
|
|
1391
|
+
`TestInvariantContext`: `snapshot`, `previousSnapshot`, `initialSnapshot`,
|
|
1392
|
+
`event`, `effects` (the actions the step produced), `step`, and `label()`,
|
|
1393
|
+
`classify()`, and `target()`.
|
|
1394
|
+
|
|
1395
|
+
### `testPaths()` options
|
|
1396
|
+
|
|
1397
|
+
In addition to the shared options:
|
|
1398
|
+
|
|
1399
|
+
| Option | Default | Description |
|
|
1400
|
+
| ---------------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1401
|
+
| `pathGenerator` | `'shortest'` | `'shortest'`, `'simple'`, or a custom `PathGenerator`. |
|
|
1402
|
+
| `paths` | none | Runs these paths instead of generating any. |
|
|
1403
|
+
| `fromEvents` | none | Runs the single path built from this event sequence. |
|
|
1404
|
+
| `samples` | `3` | Payloads sampled from each event case and outcome generator. An integer of at least `1`. |
|
|
1405
|
+
| `seed` | `0` | Sampling seed. Each case samples from its own stream, so adding a case leaves the other cases' payloads unchanged. |
|
|
1406
|
+
| `limit` | `10_000` | Traversal steps before path generation throws. A context that grows without bound reaches it; merge states with `serializeState` or prune with `stopWhen`. |
|
|
1407
|
+
| `stopWhen` | none | Stops expanding a state when it returns `true`. |
|
|
1408
|
+
| `toState` | none | Keeps only paths that end in a matching state. |
|
|
1409
|
+
| `fromState` | initial state | Starts traversal from this snapshot. |
|
|
1410
|
+
| `allowDuplicatePaths` | `false` | Keeps paths that are prefixes of longer paths. |
|
|
1411
|
+
| `serializeState`, `serializeEvent` | built in | Identity functions for traversal. |
|
|
1412
|
+
|
|
1413
|
+
`testPaths()` rejects `commands`. It resolves with `results`, one
|
|
1414
|
+
`{ path, passed, error }` per path, and sets `coverage.exploration.strategy`
|
|
1415
|
+
to `'paths'` and `stoppedBecause` to `'paths'`. It throws on the first failing
|
|
1416
|
+
path.
|
|
1417
|
+
|
|
1418
|
+
After every step, the run's snapshot is compared with the snapshot traversal
|
|
1419
|
+
planned for that step, using `serializeState`. A run that departs from its
|
|
1420
|
+
path, such as an event the machine no longer accepts or a service that
|
|
1421
|
+
resolved on its own, fails with
|
|
1422
|
+
`Path diverged at step N: expected <state>, got <state>`.
|
|
1423
|
+
|
|
1424
|
+
### `propertyTest()` options
|
|
1425
|
+
|
|
1426
|
+
In addition to the shared options:
|
|
1427
|
+
|
|
1428
|
+
| Option | Default | Description |
|
|
1429
|
+
| ----------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1430
|
+
| `commands` | none | Generators for `advance` (milliseconds), `checkpoint` (`{ label? }`), and `stop` (`{}`) commands. Each may be `{ generate, weight }`. |
|
|
1431
|
+
| `until` | none | Stop condition. See below. Enables batching. |
|
|
1432
|
+
| `batchRuns` | `25` | Runs per batch when batching. |
|
|
1433
|
+
| `maxRuns` | `100` | Total runs when batching. In `@xstate/test`, defaults to `numRuns` when that is set. |
|
|
1434
|
+
| `frontiers` | none | An array of paths, `{ paths, select?, runsPerFrontier? }`, `'auto'`, `{ strategy: 'uncovered', maxFrontiers?, runsPerFrontier?, limit? }`, or `{ strategy: 'target', maxFrontiers?, runsPerFrontier? }`. |
|
|
1435
|
+
| `swarm` | `false` | `true`, or `{ minCases?, seed? }`. |
|
|
1436
|
+
| `adapter` | fast-check | Replaces the generator engine. |
|
|
1437
|
+
|
|
1438
|
+
Without `until` and without `frontiers: 'auto'` or `{ strategy: 'target' }`,
|
|
1439
|
+
the adapter runs once with its own `numRuns`.
|
|
1440
|
+
|
|
1441
|
+
`until` is a function `(coverage) => boolean`, or an object in which every
|
|
1442
|
+
key must hold:
|
|
1443
|
+
|
|
1444
|
+
| Key | Holds when |
|
|
1445
|
+
| -------------------------------------------------------------- | ---------------------------------------------------------------------- |
|
|
1446
|
+
| `stateNodes`, `transitions`, `transitionPairs`, `requirements` | `covered / (covered + uncovered)` is at least the value. |
|
|
1447
|
+
| `eventCases` | The share of event cases executed at least once is at least the value. |
|
|
1448
|
+
| `runs` | At least this many runs completed. |
|
|
1449
|
+
| `timeMs` | At least this many milliseconds elapsed. |
|
|
1450
|
+
| `any` | At least one of the listed conditions holds. |
|
|
1451
|
+
|
|
1452
|
+
`frontiers` defaults: `maxFrontiers` is `5`, `runsPerFrontier` is an even
|
|
1453
|
+
split of the batch, and `limit` is `1000`. `swarm` defaults: `minCases` is
|
|
1454
|
+
half the declared cases, rounded up, and `seed` is `0`.
|
|
1455
|
+
|
|
1456
|
+
### fast-check options
|
|
1457
|
+
|
|
1458
|
+
`propertyTest()` and `generateTestSuite()` from `@xstate/test` pass these
|
|
1459
|
+
options to fast-check:
|
|
1460
|
+
|
|
1461
|
+
| Option | Description |
|
|
1462
|
+
| ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
1463
|
+
| `seed`, `path`, `replayPath` | Reproduce a run. `replayPath` is the `fc.commands()` replay path. |
|
|
1464
|
+
| `numRuns` | Runs per adapter call. fast-check defaults to `100`. |
|
|
1465
|
+
| `maxCommands` | Maximum generated commands per run. |
|
|
1466
|
+
| `scheduler` | `true` or `{ act }`. See [Test concurrency](#test-concurrency). |
|
|
1467
|
+
| `endOnFailure`, `interruptAfterTimeLimit`, `markInterruptAsFailure`, `skipAllAfterTimeLimit`, `timeout`, `maxSkipsPerRun` | Run limits. |
|
|
1468
|
+
| `reporter`, `asyncReporter` | Called with fast-check's run details after every adapter run: once per campaign, or once per batch with `until` or `frontiers`. |
|
|
1469
|
+
| `verbose`, `includeErrorInReport` | With `verbose` set to `1` (or `true`) or above, the failure message ends with fast-check's report, `fc.defaultReportMessage()`. |
|
|
1470
|
+
| `randomType`, `unbiased`, `skipEqualValues`, `ignoreEqualValues`, `plugins` | Generation. |
|
|
1471
|
+
|
|
1472
|
+
### `sut`
|
|
1473
|
+
|
|
1474
|
+
`TestSut`:
|
|
1475
|
+
|
|
1476
|
+
| Member | Description |
|
|
1477
|
+
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
1478
|
+
| `create(context)` | Creates a session for one run. `context` has `logic`, `input`, `snapshot` (the start snapshot, if any), `label`, `classify`, and `target`. |
|
|
1479
|
+
| `projectModel(snapshot)` | Projects the model snapshot to compare with `read()`. |
|
|
1480
|
+
| `projectSut(observed)` | Normalizes the value from `read()`. Defaults to identity. |
|
|
1481
|
+
| `equivalent(model, observed)` | Compares the projections. Defaults to structural deep equality. |
|
|
1482
|
+
| `complete({ passed, failure })` | Called once when the campaign ends, after every session was disposed. `failure` is the error the campaign throws. |
|
|
1483
|
+
|
|
1484
|
+
`TestSutSession`, returned by `create()`:
|
|
1485
|
+
|
|
1486
|
+
| Member | Description |
|
|
1487
|
+
| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1488
|
+
| `send(event, context)` | Performs the event. `context.snapshot` is the model snapshot after the event. `context.case` is `{ type, name }` for generated events. |
|
|
1489
|
+
| `read()` | Reads the observable state for comparison. |
|
|
1490
|
+
| `states` | Per-state assertions, `(snapshot, session) => void`. Replaces the top-level `states`. |
|
|
1491
|
+
| `settle()` | Waits for quiescence before each comparison. |
|
|
1492
|
+
| `advance(ms)` | Advances the SUT's clock and returns the events that fired. |
|
|
1493
|
+
| `checkpoint(label?)` | Handles a `checkpoint` command. |
|
|
1494
|
+
| `stop()` | Handles a `stop` command. |
|
|
1495
|
+
| `check()` | Runs after every stable step, after the comparison. Throws to fail the step with `SUT check failed after N steps`. |
|
|
1496
|
+
| `dispose({ passed, failure })` | Tears the session down at the end of the run. `failure` is the run's `ModelTestFailure`, when an oracle failed. |
|
|
1497
|
+
|
|
1498
|
+
`reference` has the same structure: `create()` returns
|
|
1499
|
+
`{ transition(event), read(), stop?, dispose? }`, and `projectModel` is
|
|
1500
|
+
required.
|
|
1501
|
+
|
|
1502
|
+
### Temporal properties
|
|
1503
|
+
|
|
1504
|
+
Each property has a `type`, an `id`, and an optional `description`, and is
|
|
1505
|
+
checked on every stable step:
|
|
1506
|
+
|
|
1507
|
+
| `type` | Fields | Fails when |
|
|
1508
|
+
| ------------ | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1509
|
+
| `always` | `predicate` | `predicate` is false on some step. |
|
|
1510
|
+
| `never` | `predicate` | `predicate` is true on some step. |
|
|
1511
|
+
| `eventually` | `predicate`, `within?` | `predicate` is not true within `within` steps, or before the run ends. |
|
|
1512
|
+
| `until` | `hold`, `until`, `within?` | `hold` stops holding before `until` holds, or `until` does not hold within `within` steps or before the run ends. |
|
|
1513
|
+
| `respond` | `trigger`, `response`, `within?` | A step on which `trigger` holds is not followed by a step on which `response` holds within `within` steps, or before the run ends. The response may hold on the trigger step. |
|
|
1514
|
+
| `sometimes` | `predicate` | `predicate` holds on no step of any run. Checked when the campaign ends. |
|
|
1515
|
+
|
|
1516
|
+
With `within`, a run that ends before `within` steps is **inconclusive**, not
|
|
1517
|
+
failed. An id that is inconclusive in every run and satisfied in none is
|
|
1518
|
+
listed in `coverage.temporal.inconclusive`.
|
|
1519
|
+
|
|
1520
|
+
### Coverage
|
|
1521
|
+
|
|
1522
|
+
`TestCoverage`:
|
|
1523
|
+
|
|
1524
|
+
| Field | Description |
|
|
1525
|
+
| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
1526
|
+
| `states`, `stateNodes`, `configurations`, `statuses`, `eventTypes`, `transitions`, `frontiers` | `TestCoverageDimension`: `counts`, `covered`, `uncovered`, `unreachable`, `unknown`. |
|
|
1527
|
+
| `transitionPairs` | A dimension of `"<first> -> <second>"` ids plus `truncated`. Only pairs of transitions with static targets are declared up front, up to 2,000; a pair involving a transition whose target is computed by a function appears once a run takes it. |
|
|
1528
|
+
| `requirements` | A dimension of `meta.requirements` ids plus `sources`, the state nodes and transitions that declare each id. |
|
|
1529
|
+
| `eventCases` | `{ [caseId]: { weight, generated, applicable, executed, ignored } }`. |
|
|
1530
|
+
| `labels` | `{ [name]: { count, values, share } }`. `share` is the fraction of attempted runs that recorded the label. Shrink attempts are not counted. |
|
|
1531
|
+
| `temporal` | `satisfied`, `failed`, and `inconclusive` property ids; `counts`, `{ [id]: { satisfied, failed, inconclusive } }` in runs; and `warnings` for bounds that can never fail. |
|
|
1532
|
+
| `exploration` | `TestExplorationBounds`. See below. |
|
|
1533
|
+
| `runs`, `steps`, `skipped`, `prefixSteps`, `generatedSteps`, `invariantChecks`, `temporalChecks`, `clockAdvances`, `checkpoints`, `stops`, `sutComparisons`, `oracleComparisons` | Counters. |
|
|
1534
|
+
|
|
1535
|
+
`exploration`:
|
|
1536
|
+
|
|
1537
|
+
| Field | Description |
|
|
1538
|
+
| -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1539
|
+
| `strategy` | `'property'` or `'paths'`. |
|
|
1540
|
+
| `pathCount`, `pathGenerator` | Paths only. |
|
|
1541
|
+
| `mode` | `'pure'` or `'executed'`. |
|
|
1542
|
+
| `configuredRuns`, `completedRuns`, `attemptedRuns` | `attemptedRuns` includes shrink attempts. |
|
|
1543
|
+
| `shrinkRuns` | Runs started after the first failing run, while the counterexample was shrunk. |
|
|
1544
|
+
| `maximumSequenceLength`, `maximumObservedSequenceLength` | Configured and observed sequence lengths. |
|
|
1545
|
+
| `frontiers`, `seeds` | Per-frontier budgets, and the adapter seeds and paths used. |
|
|
1546
|
+
| `swarm` | `{ runs, averageEnabled }`, or `null`. |
|
|
1547
|
+
| `target` | `{ best, label, improvements }`. `best` is `-Infinity` when unused. |
|
|
1548
|
+
| `stoppedBecause` | `'until'`, `'budget'`, `'failure'`, `'paths'` (`testPaths()` ran every path), or `'replay'` (`failures.replay` was `'only'`). |
|
|
1549
|
+
| `truncated`, `truncationReasons` | Why exploration was cut short. |
|
|
1550
|
+
| `pendingActorSteps` | Executed-mode steps that settled while an invoked or spawned actor's asynchronous work was still in flight. Those timeline entries list the actors in `pendingActors`. |
|
|
1551
|
+
|
|
1552
|
+
Add requirement ids with `meta.requirements` (a string or an array) on state
|
|
1553
|
+
nodes and transitions. A requirement is covered when any state node or
|
|
1554
|
+
transition that declares it is covered.
|
|
1555
|
+
|
|
1556
|
+
### `ModelTestFailure`
|
|
1557
|
+
|
|
1558
|
+
| Field | Description |
|
|
1559
|
+
| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
1560
|
+
| `summary` | The short message, such as `Property observation diverged`. `testPaths()` prefixes it with the failing path: `Path 2 (ADD → REMOVE) failed: …`. |
|
|
1561
|
+
| `message` | `summary` and the cause's message; a `Reproduce:` line with the fast-check `seed`, `path`, and `replayPath`; a `Fixture:` line when `fixture` is set; `Shrunk N time(s)` when fast-check shrank the counterexample; `Saved: <file>` when `failures` saved it; then the trace, one line per entry, and fast-check's report when `verbose` is set. |
|
|
1562
|
+
| `trace` | `TestTrace`: `start`, `initialSnapshot`, `timeline`, `events`, `commands`, `steps`, `finalSnapshot`, `finalObservation`, `swarm`, `mode`, `outcomes`. |
|
|
1563
|
+
| `cause` | The error thrown by the oracle or the SUT. |
|
|
1564
|
+
| `fixture` | A `TestFixture` for `replayTest()`. |
|
|
1565
|
+
| `replay` | fast-check metadata: `engine`, `engineVersion`, `seed`, `path`, `replayPath`, `numShrinks`, `data`. |
|
|
1566
|
+
| `coverage` | Coverage up to the failure. |
|
|
1567
|
+
|
|
1568
|
+
`trace.timeline` entries have a `kind`: `'event'`, `'command'` (`advance`,
|
|
1569
|
+
`checkpoint`, `outcome`, `stop`), or, in executed mode, `'actorEvent'` for a
|
|
1570
|
+
transition the actor system made on its own.
|
|
1571
|
+
|
|
1572
|
+
The trace in `message` has one line per entry:
|
|
1573
|
+
`N. <origin> <event> -> <state>`. The origin is `generator`, `prefix`, or
|
|
1574
|
+
`clock` for sent events, `timer` for `xstate.after` events and `advance`
|
|
1575
|
+
commands, `outcome` for invoke results, and `actor(<id>)` for a child actor's
|
|
1576
|
+
own transition. `<state>` is `formatSnapshot(snapshot)`, which defaults to
|
|
1577
|
+
`{ value, context }`; pass `formatSnapshot` to `propertyTest()`,
|
|
1578
|
+
`testPaths()`, or `replayTest()` to print something else. A step whose SUT or
|
|
1579
|
+
reference observation differs from the model adds `model:` and `observed:`
|
|
1580
|
+
lines.
|
|
1581
|
+
|
|
1582
|
+
### `TestFixture`
|
|
1583
|
+
|
|
1584
|
+
```ts
|
|
1585
|
+
interface TestFixture {
|
|
1586
|
+
formatVersion: 2
|
|
1587
|
+
machine?: { id?: string; version?: string }
|
|
1588
|
+
start: { type: 'input'; input: unknown } | { type: 'snapshot'; snapshot: unknown }
|
|
1589
|
+
timeline: { kind: 'event' | 'command'; command: TestCommand }[]
|
|
1590
|
+
failedAt?: number // absent on fixtures from passing runs
|
|
1591
|
+
temporalFailure?: { type; id; description?; within?; atStep: number }
|
|
1592
|
+
swarm?: string[]
|
|
1593
|
+
mode?: 'pure' | 'executed'
|
|
1594
|
+
outcomes?: { src: string; occurrence: number; outcome: TestActorOutcome }[]
|
|
1595
|
+
stubs?: string[] // invoke sources the run stubbed, resolved or not
|
|
1596
|
+
}
|
|
1597
|
+
```
|
|
1598
|
+
|
|
1599
|
+
Errors in commands and outcomes are recorded as
|
|
1600
|
+
`{ xstate$$error: true, name, message }`, since `JSON.stringify()` turns an
|
|
1601
|
+
`Error` into `{}`. `replayTest()` turns them back into `Error` objects.
|
|
1602
|
+
|
|
1603
|
+
A `TestCommand` is `{ type: 'event', event, phase, origin, caseId? }`,
|
|
1604
|
+
`{ type: 'advance', milliseconds, deliveredEvents }`,
|
|
1605
|
+
`{ type: 'checkpoint', label? }`, `{ type: 'outcome', src, outcome }`, or
|
|
1606
|
+
`{ type: 'stop' }`. Events delivered by a SUT clock appear twice: in the
|
|
1607
|
+
`advance` command's `deliveredEvents`, and as the following `event` entries
|
|
1608
|
+
with `origin: 'clock'`. `replayTest()` rejects a fixture in which the two
|
|
1609
|
+
disagree.
|
|
1610
|
+
|
|
1611
|
+
### `replayTest()`
|
|
1612
|
+
|
|
1613
|
+
`replayTest(source, fixture, options)` resolves with the replayed `TestTrace`.
|
|
1614
|
+
|
|
1615
|
+
| Option | Default | Description |
|
|
1616
|
+
| ----------------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1617
|
+
| `expect` | `'failure'` | `'failure'` stops at `failedAt` and throws the reproduced failure, or `ReplayNotReproducedError`. `'pass'` replays everything and lets any failure through. |
|
|
1618
|
+
| `invariant`, `temporal`, `sut`, `reference`, `states` | none | The oracles to replay against. |
|
|
1619
|
+
| `mode` | the fixture's mode | `'executed'` replays against a real actor with recorded outcomes stubbed. |
|
|
1620
|
+
| `actors` | none | Logic to provide before replaying. Executed mode only. |
|
|
1621
|
+
| `restoreSnapshot` | none | Required for fixtures that start from a snapshot. |
|
|
1622
|
+
|
|
1623
|
+
### `TestSuite`
|
|
1624
|
+
|
|
1625
|
+
`{ formatVersion: 1, machineId?, machineVersion?, generatedAt?, fixtures, coverage }`,
|
|
1626
|
+
where `coverage` is `testCoverageToJSON()` of the whole campaign.
|
|
1627
|
+
`generateTestSuite()` takes the `propertyTest()` options plus:
|
|
1628
|
+
|
|
1629
|
+
| Option | Default | Description |
|
|
1630
|
+
| ------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------- |
|
|
1631
|
+
| `select` | `'minimal'` | `'minimal'` keeps the smallest greedy subset of traces that preserves the covered set. `'all'` keeps every distinct trace. |
|
|
1632
|
+
| `maxFixtures` | none | Upper bound on fixtures. |
|
|
1633
|
+
| `generatedAt` | none | Recorded as-is. Omit it to keep the file byte-stable. |
|
|
1634
|
+
|
|
1635
|
+
### `TestAdapter`
|
|
1636
|
+
|
|
1637
|
+
`TestAdapter.run(request)` receives:
|
|
1638
|
+
|
|
1639
|
+
| Field | Description |
|
|
1640
|
+
| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1641
|
+
| `events` | `{ type, caseId, generator, weight }` per event case. |
|
|
1642
|
+
| `commands` | `{ type, generator, weight, src? }` per configured command. `type` is `'advance'`, `'checkpoint'`, `'stop'`, or `'outcome'`; `src` is set for `'outcome'`. |
|
|
1643
|
+
| `runBudget` | Runs to use, when a frontier or batch fixes it. |
|
|
1644
|
+
| `runOffset` | Runs completed by earlier batches. Offset a fixed seed by it. |
|
|
1645
|
+
| `createEvent(type, payload)` | Builds a typed event. |
|
|
1646
|
+
| `createRunner(run?)` | Creates a `PropertyScenarioRunner` for one run or shrink attempt. `run` is the adapter's per-run context — `shrink`, the frozen swarm subset, and a `report(outcome)` the runner calls on disposal. An adapter that never shrinks may omit it. |
|
|
1647
|
+
|
|
1648
|
+
Each run:
|
|
1649
|
+
|
|
1650
|
+
1. `createRunner(run?)`, then `await runner.start()`.
|
|
1651
|
+
2. For each step: an event (`runner.canRunGenerated(type, generated, caseId)`,
|
|
1652
|
+
then `await runner.runGenerated(type, generated, caseId)`), a command
|
|
1653
|
+
(`runner.canRunCommand(true)`, then `advance(ms)`, `checkpoint(label)`, or
|
|
1654
|
+
`stop()`), or an outcome (`runner.canRunOutcome()`, then
|
|
1655
|
+
`outcome(src, outcome)`).
|
|
1656
|
+
3. `runner.finish()`.
|
|
1657
|
+
4. `await runner.dispose()`, always.
|
|
1658
|
+
|
|
1659
|
+
`run()` resolves with `{ runs, exploration, error?, replay? }`.
|
|
1660
|
+
`exploration` has `configuredRuns`, `maximumSequenceLength`, and optionally
|
|
1661
|
+
`engine`, `seed`, `path`, `truncated`, and `truncationReasons`.
|
|
1662
|
+
|
|
1663
|
+
### `eventsFromSchemas()`
|
|
1664
|
+
|
|
1665
|
+
`eventsFromSchemas(machine, options?)`:
|
|
1666
|
+
|
|
1667
|
+
| Option | Default | Description |
|
|
1668
|
+
| --------------------- | --------- | ------------------------------------------------------------------------------------------ |
|
|
1669
|
+
| `eventsWithoutSchema` | `'empty'` | `'empty'` generates `{}` for handled event types with no schema. `'skip'` leaves them out. |
|
|
1670
|
+
| `fallback` | none | `(schema, path) => arbitrary \| undefined` for unrecognized schemas. |
|
|
1671
|
+
|
|
1672
|
+
Supported Zod (v3 and v4) kinds: `object`, `interface`, `string`, `number`,
|
|
1673
|
+
`int`, `bigint`, `boolean`, `date`, `literal`, `enum`, `nativeEnum`, `union`,
|
|
1674
|
+
`array`, `set`, `tuple`, `record`, `optional`, `nullable`, `default`,
|
|
1675
|
+
`prefault`, `catch`, `readonly`, `nonoptional`, `lazy`, `null`, `undefined`,
|
|
1676
|
+
`void`, `any`, and `unknown`. Mapped checks: lengths, numeric ranges, `int`,
|
|
1677
|
+
`multipleOf`, `email`, `uuid`, `url`, `regex`, `startsWith`, `endsWith`,
|
|
1678
|
+
`includes`, `trim`, `toLowerCase`, and `toUpperCase`. An unsupported kind or
|
|
1679
|
+
check, a recursive schema, a string with two formats, or a schema that no value
|
|
1680
|
+
satisfies throws an error naming the schema path, such as `'SET.when'`. A
|
|
1681
|
+
`type` field in a generated payload is removed.
|
|
1682
|
+
|
|
1683
|
+
A handled event type without its own schema key uses the schema of the first
|
|
1684
|
+
wildcard key it matches, such as `'user.*'`, in both `eventsWithoutSchema`
|
|
1685
|
+
modes.
|
|
1686
|
+
|
|
1687
|
+
### `createPlaywrightSut()`
|
|
1688
|
+
|
|
1689
|
+
`createPlaywrightSut(page, config)` returns a `TestSut`. `page` is any object
|
|
1690
|
+
with the parts of Playwright's `Page` the configuration uses; a real `Page` is
|
|
1691
|
+
assignable.
|
|
1692
|
+
|
|
1693
|
+
| Option | Default | Description |
|
|
1694
|
+
| --------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
1695
|
+
| `events` | required | `(page, event) => void` per event type. |
|
|
1696
|
+
| `read` | none | Projects the page to a value comparable with `projectModel`. |
|
|
1697
|
+
| `projectModel` | none | Projects the model snapshot. |
|
|
1698
|
+
| `states` | none | `(page, snapshot) => void` per state key. |
|
|
1699
|
+
| `projectSut` | identity | Normalizes the value from `read`. |
|
|
1700
|
+
| `equivalent` | deep equality | Compares the projections. |
|
|
1701
|
+
| `settle` | `page.waitForLoadState('load')`, then a microtask flush | Runs before each comparison. |
|
|
1702
|
+
| `advance` | `page.clock.runFor(ms)` | Handles `advance` commands. May return delivered events. |
|
|
1703
|
+
| `checkpoint` | screenshot | Writes `<screenshotDir>/<label>.png`, or `checkpoint-<n>.png` without a label. |
|
|
1704
|
+
| `screenshotDir` | `'property-screenshots'` | Directory for checkpoint screenshots. |
|
|
1705
|
+
| `reset` | none | Runs when a run's session is created. |
|
|
1706
|
+
| `stop` | none | Handles `stop` commands. |
|
|
1707
|
+
| `dispose` | none | Runs when a run's session is disposed. |
|
|
1708
|
+
| `mocks` | none | `(page) => void` per case key, run before the event action. |
|
|
1709
|
+
| `caseOf` | `event.case ?? event.type` | Mock key for events without a generated case: prefix, clock, and replayed events. |
|
|
1710
|
+
| `oracles` | `'defaults'` | Page oracles checked after every stable step. `'defaults'` is `{ pageError: true, console: 'error', http: 400, unhandledRejection: true }`. `false`, or an object in which omitted oracles are off. |
|
|
1711
|
+
| `testInfo` | none | Playwright's `testInfo`. Failure artifacts are attached to it. Required when `trace` or `screenshots` is on. |
|
|
1712
|
+
| `trace` | `'retain-on-failure'` with `testInfo`, else `'off'` | Records a trace per run with `page.context().tracing`. `'retain-on-failure'` attaches the failing run's trace; `'on'` also attaches the last run's trace of a passing campaign. Left alone when tracing is already running. |
|
|
1713
|
+
| `screenshots` | `'on-failure'` with `testInfo`, else `'off'` | `'on-failure'` attaches a screenshot of the page at the end of the failing run; `'every-step'` attaches one per stable step of the failing run. |
|
|
1714
|
+
| `step` | none | `(name, body) => Promise`, such as `(name, body) => test.step(name, body)`. Wraps each event action. |
|
|
1715
|
+
|
|
1716
|
+
A failing campaign with `testInfo` also attaches `fixture.json`, the failure's
|
|
1717
|
+
`TestFixture`.
|
|
1718
|
+
|
|
1719
|
+
## Migrating from `@xstate/test` 0.x and 1.0 beta
|
|
1720
|
+
|
|
1721
|
+
`@xstate/test` 2.0 requires XState v6. The path functions
|
|
1722
|
+
(`getShortestPaths()`, `getSimplePaths()`) moved to `xstate/graph`.
|
|
1723
|
+
|
|
1724
|
+
### From 1.0 beta
|
|
1725
|
+
|
|
1726
|
+
The `sut` option of `testPaths()` replaces `path.test({ events, states })`,
|
|
1727
|
+
and adds coverage and replay fixtures. Pass `paths` to run specific paths.
|
|
1728
|
+
|
|
1729
|
+
Before:
|
|
1730
|
+
|
|
1731
|
+
```ts
|
|
1732
|
+
const model = createTestModel(machine)
|
|
1733
|
+
|
|
1734
|
+
for (const path of model.getShortestPaths()) {
|
|
1735
|
+
it(path.description, async () => {
|
|
1736
|
+
await path.test({
|
|
1737
|
+
events: { SUBMIT: ({ event }) => page.click('#submit') },
|
|
1738
|
+
states: { submitted: () => expect(page.locator('#done')).toBeVisible() },
|
|
1739
|
+
})
|
|
1740
|
+
})
|
|
1741
|
+
}
|
|
1742
|
+
```
|
|
1743
|
+
|
|
1744
|
+
After:
|
|
1745
|
+
|
|
1746
|
+
```ts
|
|
1747
|
+
import { getShortestPaths } from 'xstate/graph'
|
|
1748
|
+
|
|
1749
|
+
for (const path of getShortestPaths(machine)) {
|
|
1750
|
+
const title = path.steps.map((step) => step.event.type).join(' → ')
|
|
1751
|
+
it(title, async () => {
|
|
1752
|
+
await testPaths(machine, {
|
|
1753
|
+
paths: [path],
|
|
1754
|
+
sut: {
|
|
1755
|
+
create: () => ({
|
|
1756
|
+
send: (event) => (event.type === 'SUBMIT' ? page.click('#submit') : undefined),
|
|
1757
|
+
states: { submitted: () => expect(page.locator('#done')).toBeVisible() },
|
|
1758
|
+
}),
|
|
1759
|
+
},
|
|
1760
|
+
})
|
|
1761
|
+
})
|
|
1762
|
+
}
|
|
1763
|
+
```
|
|
1764
|
+
|
|
1765
|
+
Other changes:
|
|
1766
|
+
|
|
1767
|
+
| 1.0 beta | 2.0 |
|
|
1768
|
+
| ---------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- |
|
|
1769
|
+
| `createTestMachine(config)` | `createMachine(config)` |
|
|
1770
|
+
| `path.testSync(params)` | `await testPaths(machine, { paths: [path], sut })` |
|
|
1771
|
+
| `model.testState(state, params)`, `model.testTransition(step, params)` | The `states` option, checked on every stable step. |
|
|
1772
|
+
| `TestPathResult`, `TestStepResult` | `TestPathRunResult`: `{ path, passed, error }`. Use `ModelTestFailure.trace` for step detail. |
|
|
1773
|
+
| `meta.test(testContext, state)` | `meta.test(session, snapshot)`: the first argument is the SUT session. |
|
|
1774
|
+
|
|
1775
|
+
### From 0.x
|
|
1776
|
+
|
|
1777
|
+
| 0.x | 2.0 |
|
|
1778
|
+
| --------------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
|
1779
|
+
| `createModel(machine).withEvents({ E: { exec, cases } })` | `events: { E: [...] }` for payloads, and `sut.create().send` for `exec`. |
|
|
1780
|
+
| `model.getShortestPathPlans()`, `plan.paths` | `testPaths(machine, options)`, or `getShortestPaths(machine)` from `xstate/graph`. |
|
|
1781
|
+
| `model.getSimplePathPlans()` | `testPaths(machine, { pathGenerator: 'simple' })`. |
|
|
1782
|
+
| `path.test(page)` | `testPaths(machine, { sut })`. The SUT session holds the page. |
|
|
1783
|
+
| `meta.test(page, state)` | `meta.test(session, snapshot)`, or `states` on the session. |
|
|
1784
|
+
| `model.testCoverage()` | `assertTestCoverage(coverage, { stateNodes: 1 })`. |
|
|
1785
|
+
|
|
1786
|
+
`cases` become event cases:
|
|
1787
|
+
|
|
1788
|
+
```ts
|
|
1789
|
+
// 0.x
|
|
1790
|
+
createModel(machine).withEvents({
|
|
1791
|
+
ADD: {
|
|
1792
|
+
exec: (page, event) => page.fill('#sku', event.sku),
|
|
1793
|
+
cases: [{ sku: 'apple' }, { sku: 'pear' }],
|
|
1794
|
+
},
|
|
1795
|
+
})
|
|
1796
|
+
|
|
1797
|
+
// 2.0
|
|
1798
|
+
await testPaths(machine, {
|
|
1799
|
+
events: {
|
|
1800
|
+
ADD: [
|
|
1801
|
+
{ case: 'apple', generate: fc.constant({ sku: 'apple' }) },
|
|
1802
|
+
{ case: 'pear', generate: fc.constant({ sku: 'pear' }) },
|
|
1803
|
+
],
|
|
1804
|
+
},
|
|
1805
|
+
samples: 1,
|
|
1806
|
+
sut: { create: () => ({ send: (event) => page.fill('#sku', event.sku) }) },
|
|
1807
|
+
})
|
|
1808
|
+
```
|
|
1809
|
+
|
|
1810
|
+
## Comparison
|
|
1811
|
+
|
|
1812
|
+
| | `@xstate/test` 2.0 | fast-check alone | Playwright alone | `@xstate/test` 0.x and 1.0 beta |
|
|
1813
|
+
| ------------------------- | ------------------------------------------------------------------------------- | ---------------------------------------- | ------------------------------ | ------------------------------- |
|
|
1814
|
+
| Model | An XState machine | Hand-written model and `Command` classes | None | An XState machine |
|
|
1815
|
+
| Sequences | Graph paths and random sequences | Random sequences | Hand-written scenarios | Graph paths |
|
|
1816
|
+
| Payloads | Generated, or derived from Zod and Effect schemas | Generated | Hand-written | Fixed `cases` |
|
|
1817
|
+
| Shrinking | Yes, with `propertyTest()` | Yes | No | No |
|
|
1818
|
+
| Oracles | SUT comparison, invariants, temporal properties, state assertions, reference | Assertions in each command | Assertions in each test | State assertions |
|
|
1819
|
+
| Coverage | States, transitions, pairs, requirements, event cases | None | None | State nodes |
|
|
1820
|
+
| Replay | Portable JSON fixtures, a failure database replayed first, and fast-check seeds | Seeds and paths | Traces | None |
|
|
1821
|
+
| Invoked actors and delays | `mode: 'executed'` with stubbed outcomes and a simulated clock | Hand-written | Real services and `page.clock` | Not modeled |
|
|
1822
|
+
| UI | `@xstate/test/playwright` | Manual | Native | Manual executors |
|