@typeonce/ritmo-devtools 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +283 -0
- package/README.md +303 -55
- package/dist/Desync.d.ts +23 -3
- package/dist/Desync.d.ts.map +1 -1
- package/dist/Desync.js +17 -15
- package/dist/Desync.js.map +1 -1
- package/dist/Determinism.d.ts +20 -6
- package/dist/Determinism.d.ts.map +1 -1
- package/dist/Determinism.js +11 -3
- package/dist/Determinism.js.map +1 -1
- package/dist/Format.d.ts +43 -8
- package/dist/Format.d.ts.map +1 -1
- package/dist/Format.js +66 -31
- package/dist/Format.js.map +1 -1
- package/dist/Fuzz.d.ts +275 -0
- package/dist/Fuzz.d.ts.map +1 -0
- package/dist/Fuzz.js +362 -0
- package/dist/Fuzz.js.map +1 -0
- package/dist/Invariant.d.ts +9 -2
- package/dist/Invariant.d.ts.map +1 -1
- package/dist/Invariant.js.map +1 -1
- package/dist/Lint.d.ts +11 -5
- package/dist/Lint.d.ts.map +1 -1
- package/dist/Lint.js.map +1 -1
- package/dist/PlaywrightEngine.d.ts +29 -0
- package/dist/PlaywrightEngine.d.ts.map +1 -0
- package/dist/PlaywrightEngine.js +196 -0
- package/dist/PlaywrightEngine.js.map +1 -0
- package/dist/Replay.d.ts +47 -23
- package/dist/Replay.d.ts.map +1 -1
- package/dist/Replay.js +26 -30
- package/dist/Replay.js.map +1 -1
- package/dist/ReplayNodeSink.d.ts.map +1 -1
- package/dist/ReplayNodeSink.js +4 -1
- package/dist/ReplayNodeSink.js.map +1 -1
- package/dist/ReplayVitePlugin.d.ts +1 -0
- package/dist/ReplayVitePlugin.d.ts.map +1 -1
- package/dist/ReplayVitePlugin.js.map +1 -1
- package/dist/Session.d.ts +224 -18
- package/dist/Session.d.ts.map +1 -1
- package/dist/Session.js +571 -58
- package/dist/Session.js.map +1 -1
- package/dist/Verify.d.ts +150 -0
- package/dist/Verify.d.ts.map +1 -0
- package/dist/Verify.js +154 -0
- package/dist/Verify.js.map +1 -0
- package/dist/cli.js +157 -16
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/internal/dependencyScan.d.ts +44 -0
- package/dist/internal/dependencyScan.d.ts.map +1 -0
- package/dist/internal/dependencyScan.js +313 -0
- package/dist/internal/dependencyScan.js.map +1 -0
- package/dist/internal/fuzzNetwork.d.ts +56 -0
- package/dist/internal/fuzzNetwork.d.ts.map +1 -0
- package/dist/internal/fuzzNetwork.js +96 -0
- package/dist/internal/fuzzNetwork.js.map +1 -0
- package/dist/internal/simulationCoverage.d.ts.map +1 -1
- package/dist/internal/simulationCoverage.js +2 -1
- package/dist/internal/simulationCoverage.js.map +1 -1
- package/dist/internal/skillLink.d.ts +14 -0
- package/dist/internal/skillLink.d.ts.map +1 -0
- package/dist/internal/skillLink.js +46 -0
- package/dist/internal/skillLink.js.map +1 -0
- package/dist/internal/values.d.ts +9 -1
- package/dist/internal/values.d.ts.map +1 -1
- package/dist/internal/values.js +11 -1
- package/dist/internal/values.js.map +1 -1
- package/dist/internal/verifyRunner.d.ts +19 -0
- package/dist/internal/verifyRunner.d.ts.map +1 -0
- package/dist/internal/verifyRunner.js +47 -0
- package/dist/internal/verifyRunner.js.map +1 -0
- package/dist/internal/version.d.ts +1 -1
- package/dist/internal/version.js +1 -1
- package/package.json +28 -3
- package/src/Desync.ts +40 -17
- package/src/Determinism.ts +49 -11
- package/src/Format.ts +101 -34
- package/src/Fuzz.ts +551 -0
- package/src/Invariant.ts +9 -2
- package/src/Lint.ts +11 -5
- package/src/PlaywrightEngine.ts +232 -0
- package/src/Replay.ts +100 -47
- package/src/ReplayNodeSink.ts +4 -1
- package/src/ReplayVitePlugin.ts +1 -0
- package/src/Session.ts +816 -73
- package/src/Verify.ts +276 -0
- package/src/cli.ts +150 -15
- package/src/index.ts +3 -2
- package/src/internal/dependencyScan.ts +356 -0
- package/src/internal/fuzzNetwork.ts +130 -0
- package/src/internal/simulationCoverage.ts +2 -1
- package/src/internal/skillLink.ts +53 -0
- package/src/internal/values.ts +20 -1
- package/src/internal/verifyRunner.ts +46 -0
- package/src/internal/version.ts +1 -1
- package/dist/Monitor.d.ts +0 -101
- package/dist/Monitor.d.ts.map +0 -1
- package/dist/Monitor.js +0 -602
- package/dist/Monitor.js.map +0 -1
- package/dist/internal/monitor.d.ts +0 -126
- package/dist/internal/monitor.d.ts.map +0 -1
- package/dist/internal/monitor.js +0 -279
- package/dist/internal/monitor.js.map +0 -1
- package/src/Monitor.ts +0 -713
- package/src/internal/monitor.ts +0 -370
package/src/Fuzz.ts
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Seeded fuzzing that shrinks to a replay: finds a bug before anyone plays
|
|
3
|
+
* into it, and hands back the small input timeline that shows it.
|
|
4
|
+
*
|
|
5
|
+
* `run(make, options)` builds a fresh simulation per run with the same
|
|
6
|
+
* `make(playback)` factory `Replay.session` opens replays with, feeds each
|
|
7
|
+
* fuzzed source a value drawn from a seeded `Random` (from
|
|
8
|
+
* `@typeonce/ritmo`) every frame, and ticks the schedule in a `Session`
|
|
9
|
+
* until an invariant breaks, a system fails, or something throws. The
|
|
10
|
+
* failing timeline is then shrunk by delta debugging: ranges of frames are
|
|
11
|
+
* dropped, then each remaining value is moved toward the input's `neutral`
|
|
12
|
+
* (the whole value, then one field at a time), re-running the deterministic
|
|
13
|
+
* simulation for every candidate and keeping it only when it fails the same
|
|
14
|
+
* way. The result is a `Replay.ReplayFile` of the shrunk timeline, so the
|
|
15
|
+
* test that locks the bug in is `Replay.session(make, file, { invariants })`.
|
|
16
|
+
*
|
|
17
|
+
* A source the factory asks `playback.source` for must be a fuzzed `input`,
|
|
18
|
+
* and every `input` must be asked for; provide plain-data services (rules,
|
|
19
|
+
* config) directly. Use `Determinism.check` to check that the same input
|
|
20
|
+
* repeats; use `Session` to explain a failure by hand.
|
|
21
|
+
*
|
|
22
|
+
* With `network`, the same `run` fuzzes a rollback game over a faulty
|
|
23
|
+
* network: every seat of a `seats` input is its own peer, a fresh runtime
|
|
24
|
+
* from `make`, joined to the others by a seeded `MemoryNetworkLink`
|
|
25
|
+
* (latency, jitter, loss) and driven by a `Netcode.rollback` session. A
|
|
26
|
+
* desync, or any other session failure, is a `SessionFailed` violation, and
|
|
27
|
+
* shrinking shrinks the input timeline with the network seed fixed. Run
|
|
28
|
+
* `Netcode.syncTest` first: it finds state a rewind cannot put back on one
|
|
29
|
+
* peer, faster.
|
|
30
|
+
*
|
|
31
|
+
* @module Fuzz
|
|
32
|
+
* @docGroup devtools
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```ts
|
|
36
|
+
* const Input = Descriptor.Service<{ snapshot: () => { dx: number } }>()("Game/Input")
|
|
37
|
+
* declare const make: (playback: Replay.Playback) => Replay.Target<typeof Game.schema, typeof Root, { readonly setup: typeof setup; readonly update: typeof update }>
|
|
38
|
+
* const InsideArena = Invariant.fromInspector(Game.Inspector("inside arena", {
|
|
39
|
+
* queries: { bodies: Game.Query({ selection: { position: Game.Query.read(Position) } }) }
|
|
40
|
+
* }, ({ queries }) => queries.bodies.each().some(({ data }) => data.position.get().x < 0) ? "left the arena" : undefined))
|
|
41
|
+
* declare const sink: Replay.Sink
|
|
42
|
+
* // ---cut---
|
|
43
|
+
* const Move = Fuzz.input(Input, {
|
|
44
|
+
* generate: (random) => ({ dx: random.integer(-2, 3) }),
|
|
45
|
+
* neutral: { dx: 0 }
|
|
46
|
+
* })
|
|
47
|
+
* const found = Fuzz.run(make, { schedule: "update", inputs: [Move], invariants: [InsideArena], frames: 600, seed: 7 })
|
|
48
|
+
* if (!found.ok && found.error._tag === "Found") {
|
|
49
|
+
* sink.save(found.error.replay) // the shrunk timeline that still fails
|
|
50
|
+
* console.log(found.error.violation) // the invariant, failure, or defect it hit
|
|
51
|
+
* }
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
import { Clock } from "@typeonce/ritmo"
|
|
55
|
+
import type { Descriptor } from "@typeonce/ritmo/Descriptor"
|
|
56
|
+
import * as Result from "@typeonce/ritmo/Result"
|
|
57
|
+
import type { AnyScheduleOrder, ExecutableScheduleDefinition, Source } from "@typeonce/ritmo/Schedule"
|
|
58
|
+
import type { Schema } from "@typeonce/ritmo/Schema"
|
|
59
|
+
import type * as Snapshot from "@typeonce/ritmo/Snapshot"
|
|
60
|
+
import type * as Checkpoint from "@typeonce/ritmo/Checkpoint"
|
|
61
|
+
import type * as CoreDescriptor from "@typeonce/ritmo/Descriptor"
|
|
62
|
+
import * as Random from "@typeonce/ritmo/Random"
|
|
63
|
+
import type * as Netcode from "@typeonce/ritmo-netcode/Netcode"
|
|
64
|
+
|
|
65
|
+
import * as FuzzNetwork from "./internal/fuzzNetwork.ts"
|
|
66
|
+
|
|
67
|
+
import type { Invariant } from "./Invariant.ts"
|
|
68
|
+
import * as Rendered from "./Rendered.ts"
|
|
69
|
+
import * as Replay from "./Replay.ts"
|
|
70
|
+
import * as Session from "./Session.ts"
|
|
71
|
+
|
|
72
|
+
type AnySchedule<S extends Schema.Any, Root> = ExecutableScheduleDefinition<S, any, Root, any, any, AnyScheduleOrder>
|
|
73
|
+
|
|
74
|
+
/** One fuzzed source, made with {@link input}: what `run` feeds a `snapshot()` service every frame. */
|
|
75
|
+
export interface Input<Name extends string = string, Data = unknown> {
|
|
76
|
+
/** The source service's name. */
|
|
77
|
+
readonly name: Name
|
|
78
|
+
/** The value for `frame` (counting from 1), drawn from the run's seeded generator. */
|
|
79
|
+
readonly generate: (random: Random.Random, frame: number) => Data
|
|
80
|
+
/** The input of a player who does nothing: what shrinking moves values toward. */
|
|
81
|
+
readonly neutral: Data
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Declares how to fuzz one `snapshot()` source: `generate` draws its value
|
|
86
|
+
* for a frame, of the type the source returns, and `neutral` is the idle
|
|
87
|
+
* value. Pass the result in `run`'s `inputs`. The source is a
|
|
88
|
+
* `snapshot()`-only service (`Schedule.Source`, `InputCapture.SourceOf<...>`
|
|
89
|
+
* for browser input, whose `Controls` action set draws a value with
|
|
90
|
+
* `random` and has `idle` for `neutral`). Use `Keyboard.scripted` (or a
|
|
91
|
+
* scripted source) instead to replay one known sequence.
|
|
92
|
+
*/
|
|
93
|
+
export const input = <const Name extends string, Value extends Source<unknown>>(
|
|
94
|
+
descriptor: Descriptor<"service", Name, Value> & Replay.ValidateSource<Name, Value>,
|
|
95
|
+
options: {
|
|
96
|
+
readonly generate: (random: Random.Random, frame: number) => Replay.SourceData<Value>
|
|
97
|
+
readonly neutral: Replay.SourceData<Value>
|
|
98
|
+
}
|
|
99
|
+
): Input<Name, Replay.SourceData<Value>> => ({ name: descriptor.name, generate: options.generate, neutral: options.neutral })
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Every seat's input of a networked game, made with {@link seats}: per
|
|
103
|
+
* frame, one value per seat. Alone it fuzzes the seats source like any
|
|
104
|
+
* `input`; with `run`'s `network`, each seat's value is that peer's local
|
|
105
|
+
* input.
|
|
106
|
+
*/
|
|
107
|
+
export interface Seats<Name extends string = string, Data = unknown> extends Input<Name, ReadonlyArray<Data>> {
|
|
108
|
+
/** The schema remote input is decoded with, as given to `Netcode.seats`. */
|
|
109
|
+
readonly codec: CoreDescriptor.Codec<any, any>
|
|
110
|
+
/** How many seats: over a network, how many peers. */
|
|
111
|
+
readonly count: number
|
|
112
|
+
/** The input of a seat that pressed nothing, as given to `Netcode.seats`. */
|
|
113
|
+
readonly idle: Data
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Declares how to fuzz the seats of a networked game: the
|
|
118
|
+
* `Netcode.Source` service its simulation captures, the input `codec` and
|
|
119
|
+
* `idle` input the game gives `Netcode.seats`, `count` seats, and
|
|
120
|
+
* `generate` for one seat's input on a frame. Pass it as the only input of a
|
|
121
|
+
* `run` with `network`; `make` provides the service with
|
|
122
|
+
* `playback.source(descriptor)` (over a network, that is the peer's
|
|
123
|
+
* `seats.source`). Use `input` for a game without seats.
|
|
124
|
+
*/
|
|
125
|
+
export const seats = <const Name extends string, const C extends CoreDescriptor.Codec<any, any>>(
|
|
126
|
+
descriptor: Descriptor<"service", Name, Netcode.Source<CoreDescriptor.CodecType<C>>>,
|
|
127
|
+
options: {
|
|
128
|
+
readonly codec: C & CoreDescriptor.CheckedCodec<C>
|
|
129
|
+
readonly count: number
|
|
130
|
+
readonly generate: (random: Random.Random, frame: number, seat: number) => NoInfer<CoreDescriptor.CodecType<C>>
|
|
131
|
+
readonly idle: NoInfer<CoreDescriptor.CodecType<C>>
|
|
132
|
+
}
|
|
133
|
+
): Seats<Name, CoreDescriptor.CodecType<C>> => ({
|
|
134
|
+
name: descriptor.name,
|
|
135
|
+
codec: options.codec,
|
|
136
|
+
count: options.count,
|
|
137
|
+
idle: options.idle,
|
|
138
|
+
neutral: Array.from({ length: options.count }, () => options.idle),
|
|
139
|
+
generate: (random, frame) => Array.from({ length: options.count }, (_, seat) => options.generate(random, frame, seat))
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* How `run` joins the peers when it fuzzes over a network: the seeded link
|
|
144
|
+
* faults (`MemoryNetworkLink.make`'s), kept fixed while shrinking, and the
|
|
145
|
+
* `Netcode.rollback` options every peer's session takes.
|
|
146
|
+
*/
|
|
147
|
+
export interface Network {
|
|
148
|
+
/** The link's seed: drops and delays repeat for the same seed. */
|
|
149
|
+
readonly seed: number
|
|
150
|
+
readonly latencyMS: number
|
|
151
|
+
/** Up to this much extra delay per message (0 by default). */
|
|
152
|
+
readonly jitterMS?: number
|
|
153
|
+
/** The share of messages dropped, from 0 to 1 (0 by default). */
|
|
154
|
+
readonly loss?: number
|
|
155
|
+
/** The simulation's fixed step: the test clock moves one step per frame. */
|
|
156
|
+
readonly step: Clock.Step
|
|
157
|
+
/** Idle frames played after the fuzzed ones, so late input and checksums arrive (120 by default). */
|
|
158
|
+
readonly settle?: number
|
|
159
|
+
readonly inputDelay?: number
|
|
160
|
+
readonly maxRollback?: number
|
|
161
|
+
/** Ticks between checksum comparisons (10 by default here, to find a desync close to its cause). */
|
|
162
|
+
readonly checksumEvery?: number
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** What `run` fuzzes, how long, and how hard it shrinks. */
|
|
166
|
+
export interface Options<
|
|
167
|
+
Name extends string,
|
|
168
|
+
Names extends string,
|
|
169
|
+
Invariants extends ReadonlyArray<Invariant<Invariant.CheckInspector>>
|
|
170
|
+
> {
|
|
171
|
+
/** The schedule to fuzz: the one ticked once per simulation step. Its name in the replay. */
|
|
172
|
+
readonly schedule: Name
|
|
173
|
+
/** Ticked once before the fuzzed frames, with every input neutral: the replay starts after it. */
|
|
174
|
+
readonly setup?: Names
|
|
175
|
+
/** The fuzzed sources, each from `input`. */
|
|
176
|
+
readonly inputs: readonly [Input, ...Array<Input>]
|
|
177
|
+
/** Checked after every frame, in order, as in `Session.make`. */
|
|
178
|
+
readonly invariants?: Invariants
|
|
179
|
+
/** Frames per run, a whole number above 0. */
|
|
180
|
+
readonly frames: number
|
|
181
|
+
/** Runs to try, each with a fresh simulation; run `i` draws from seed `seed + i`. Defaults to 10. */
|
|
182
|
+
readonly runs?: number
|
|
183
|
+
/** The first run's seed (an integer). A found bug reports the seed of its run. */
|
|
184
|
+
readonly seed: number
|
|
185
|
+
/** Simulations shrinking may run before it stops with the smallest timeline so far. Defaults to 500. */
|
|
186
|
+
readonly shrinkAttempts?: number
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** What a fuzzed run hit. */
|
|
190
|
+
export type Violation =
|
|
191
|
+
| { readonly _tag: "InvariantViolated"; readonly invariant: string; readonly message: string }
|
|
192
|
+
| { readonly _tag: "SystemFailed"; readonly system: string; readonly error: unknown }
|
|
193
|
+
/** A system threw, or `invariant` names the invariant whose check threw (a bug in the check). */
|
|
194
|
+
| { readonly _tag: "Defect"; readonly invariant: string | undefined; readonly error: unknown }
|
|
195
|
+
/**
|
|
196
|
+
* Over a `network`: the session of `seat` failed. `Desynced` is a
|
|
197
|
+
* determinism bug (`Desync.compare` its `snapshot` with the other seat's:
|
|
198
|
+
* the sessions keep them); `TickFailed` and `CheckpointLost` stop a real
|
|
199
|
+
* session too.
|
|
200
|
+
*/
|
|
201
|
+
| { readonly _tag: "SessionFailed"; readonly seat: number; readonly error: Netcode.SessionError<unknown> }
|
|
202
|
+
|
|
203
|
+
/** A bug the fuzzer found: the error side of `run`. */
|
|
204
|
+
export interface Found {
|
|
205
|
+
readonly _tag: "Found"
|
|
206
|
+
readonly violation: Violation
|
|
207
|
+
/** The seed of the run that found it: `run` with this `seed` and `runs: 1` finds it again. */
|
|
208
|
+
readonly seed: number
|
|
209
|
+
/** The frame the shrunk timeline fails at: its length, or later over a `network` (the frame a peer saw it). */
|
|
210
|
+
readonly frame: number
|
|
211
|
+
/**
|
|
212
|
+
* The shrunk timeline as a replay; `Replay.session(make, replay, { invariants })` plays it to the violation.
|
|
213
|
+
* Over a `network`, it plays every seat's input on one runtime: step it to see what each tick did,
|
|
214
|
+
* and find the failure again with `run` and the same seeds.
|
|
215
|
+
*/
|
|
216
|
+
readonly replay: Replay.ReplayFile
|
|
217
|
+
/**
|
|
218
|
+
* Whether the replay, opened with `Replay.session`, hits the violation
|
|
219
|
+
* again. `false` means restoring the start lost state the bug needs
|
|
220
|
+
* (transient components): reproduce it with `seed` instead.
|
|
221
|
+
*/
|
|
222
|
+
readonly reproduces: boolean
|
|
223
|
+
/**
|
|
224
|
+
* How shrinking went: simulations run, candidates kept, and the frames of input before and after
|
|
225
|
+
* (the failing frame, but over a `network`, where a desync shows a few frames after its cause).
|
|
226
|
+
*/
|
|
227
|
+
readonly shrink: { readonly attempts: number; readonly steps: number; readonly fromFrames: number; readonly toFrames: number }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** A run failed, but the same input did not fail again: the simulation reads something besides its input. */
|
|
231
|
+
export interface Flaky {
|
|
232
|
+
readonly _tag: "Flaky"
|
|
233
|
+
readonly violation: Violation
|
|
234
|
+
readonly seed: number
|
|
235
|
+
readonly frame: number
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** The options or the factory cannot be fuzzed; `message` says why. */
|
|
239
|
+
export interface InvalidOptions {
|
|
240
|
+
readonly _tag: "InvalidOptions"
|
|
241
|
+
readonly message: string
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** The world after setup cannot be saved (a `"~encode"` rejected it), so no replay could start there. */
|
|
245
|
+
export interface Unreplayable {
|
|
246
|
+
readonly _tag: "Unreplayable"
|
|
247
|
+
readonly error: Snapshot.SnapshotError
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Why `run` did not pass. */
|
|
251
|
+
export type FuzzError = Found | Flaky | InvalidOptions | Unreplayable
|
|
252
|
+
|
|
253
|
+
/** Every run completed: how many, of how many frames, from which seed. */
|
|
254
|
+
export interface Passed {
|
|
255
|
+
readonly runs: number
|
|
256
|
+
readonly frames: number
|
|
257
|
+
readonly seed: number
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** A timeline: per frame, the value of each input, in `inputs` order. */
|
|
261
|
+
type Timeline = ReadonlyArray<ReadonlyArray<unknown>>
|
|
262
|
+
|
|
263
|
+
type Outcome =
|
|
264
|
+
| { readonly kind: "passed"; readonly file?: Replay.ReplayFile | undefined }
|
|
265
|
+
| { readonly kind: "violated"; readonly violation: Violation; readonly frame: number; readonly file: Replay.ReplayFile | undefined }
|
|
266
|
+
| { readonly kind: "invalid"; readonly error: InvalidOptions | Unreplayable }
|
|
267
|
+
|
|
268
|
+
/** Two violations are the same bug when they name the same invariant, system, or kind of defect. */
|
|
269
|
+
const signature = (violation: Violation): string =>
|
|
270
|
+
violation._tag === "InvariantViolated" ? `invariant ${violation.invariant}`
|
|
271
|
+
: violation._tag === "SystemFailed" ? `system ${violation.system}`
|
|
272
|
+
: violation._tag === "SessionFailed" ? `session ${violation.error._tag}`
|
|
273
|
+
: `defect ${violation.invariant ?? ""}`
|
|
274
|
+
|
|
275
|
+
const describe = (violation: Violation): string => {
|
|
276
|
+
if (violation._tag === "InvariantViolated") return `invariant ${violation.invariant} broke: ${violation.message}`
|
|
277
|
+
if (violation._tag === "SessionFailed") return `seat ${violation.seat}'s session failed with ${violation.error._tag}${"tick" in violation.error ? ` at tick ${violation.error.tick}` : ""}`
|
|
278
|
+
const error = violation.error instanceof Error ? violation.error.message : JSON.stringify(violation.error) ?? String(violation.error)
|
|
279
|
+
if (violation._tag === "SystemFailed") return `${violation.system} failed: ${error}`
|
|
280
|
+
return violation.invariant === undefined ? `a system threw: ${error}` : `invariant ${violation.invariant} threw: ${error}`
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const violationOf = (stop: Session.Stop): Violation | InvalidOptions | undefined => {
|
|
284
|
+
switch (stop.reason) {
|
|
285
|
+
case "completed":
|
|
286
|
+
case "until":
|
|
287
|
+
return undefined
|
|
288
|
+
case "invariant":
|
|
289
|
+
return { _tag: "InvariantViolated", invariant: stop.invariant, message: stop.message }
|
|
290
|
+
case "failure":
|
|
291
|
+
return { _tag: "SystemFailed", system: stop.system, error: stop.error }
|
|
292
|
+
case "ignored-failure":
|
|
293
|
+
return { _tag: "SystemFailed", system: stop.system ?? stop.schedule, error: stop.error }
|
|
294
|
+
case "defect":
|
|
295
|
+
return { _tag: "Defect", invariant: undefined, error: stop.error }
|
|
296
|
+
case "invariant-threw":
|
|
297
|
+
return { _tag: "Defect", invariant: stop.invariant, error: stop.error }
|
|
298
|
+
case "missingRequirements":
|
|
299
|
+
return { _tag: "InvalidOptions", message: `the fuzzed schedule's runtime is missing requirements: ${JSON.stringify(stop.requirements)}` }
|
|
300
|
+
case "tickOrder":
|
|
301
|
+
return { _tag: "InvalidOptions", message: `the fuzzed schedule's tick order is invalid: ${stop.issues.join("; ")}` }
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const same = (left: unknown, right: unknown): boolean => left === right || JSON.stringify(left) === JSON.stringify(right)
|
|
306
|
+
|
|
307
|
+
const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
|
|
308
|
+
typeof value === "object" && value !== null && !Array.isArray(value)
|
|
309
|
+
|
|
310
|
+
/** Values closer to `neutral` than `value`: `neutral` itself, then `value` with one field set to neutral's. */
|
|
311
|
+
const simpler = (value: unknown, neutral: unknown): Array<unknown> => {
|
|
312
|
+
if (same(value, neutral)) return []
|
|
313
|
+
const candidates: Array<unknown> = [neutral]
|
|
314
|
+
// Every seat's input: one seat at a time.
|
|
315
|
+
if (Array.isArray(value) && Array.isArray(neutral) && value.length === neutral.length) {
|
|
316
|
+
value.forEach((item, index) => {
|
|
317
|
+
for (const candidate of simpler(item, neutral[index])) {
|
|
318
|
+
const next = value.map((other, at) => at === index ? candidate : other)
|
|
319
|
+
if (!same(next, neutral)) candidates.push(next)
|
|
320
|
+
}
|
|
321
|
+
})
|
|
322
|
+
}
|
|
323
|
+
if (isRecord(value) && isRecord(neutral)) {
|
|
324
|
+
const keys = Object.keys(value).filter((key) => !same(value[key], neutral[key]))
|
|
325
|
+
if (keys.length > 1) for (const key of keys) candidates.push({ ...value, [key]: neutral[key] })
|
|
326
|
+
}
|
|
327
|
+
return candidates
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Over a `network`: the only input is a `Fuzz.seats`, the fuzzed schedule
|
|
332
|
+
* must rewind and the runtime save (as for `Netcode.rollback`).
|
|
333
|
+
*/
|
|
334
|
+
type NetworkChecks<Net, S extends Schema.Any, Fuzzed> =
|
|
335
|
+
[Net] extends [Network]
|
|
336
|
+
? { readonly inputs: readonly [Seats] } & Checkpoint.ValidateRewindable<Fuzzed> & Snapshot.Gate<S, unknown>
|
|
337
|
+
: unknown
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Fuzzes `options.schedule` of the simulation `make` builds. Each run makes
|
|
341
|
+
* a fresh runtime with `make(playback)` (the `Replay.session` factory),
|
|
342
|
+
* ticks `setup` once, then ticks the schedule up to `frames` times with
|
|
343
|
+
* every input's `generate`d value, checking `invariants` after each frame.
|
|
344
|
+
* The first run that fails is shrunk and returned as a `Found` with a
|
|
345
|
+
* replay; a schedule name that `make` does not return, an invariant that
|
|
346
|
+
* reads what the runtime does not provide, or a schedule service that could
|
|
347
|
+
* not be replayed does not compile.
|
|
348
|
+
*
|
|
349
|
+
* With `network`, every seat of the one `Fuzz.seats` input plays on its own
|
|
350
|
+
* peer (a fresh `make` each, seat 0 ticking `setup`) through
|
|
351
|
+
* `Netcode.rollback` over a seeded `MemoryNetworkLink`, and the invariants
|
|
352
|
+
* are checked on every peer after each frame. A session failure is a
|
|
353
|
+
* `SessionFailed` violation; a schedule that cannot be rewound, or a
|
|
354
|
+
* runtime that cannot save, does not compile.
|
|
355
|
+
*/
|
|
356
|
+
export const run = <
|
|
357
|
+
S extends Schema.Any,
|
|
358
|
+
Root,
|
|
359
|
+
const Schedules extends Readonly<Record<string, AnySchedule<S, Root>>>,
|
|
360
|
+
Services extends Record<string, unknown>,
|
|
361
|
+
Resources extends object,
|
|
362
|
+
Machines extends Record<string, unknown>,
|
|
363
|
+
const Name extends keyof Schedules & string,
|
|
364
|
+
const Invariants extends ReadonlyArray<Invariant<Invariant.CheckInspector>> = readonly [],
|
|
365
|
+
const Net extends Network | undefined = undefined
|
|
366
|
+
>(
|
|
367
|
+
make: (playback: Replay.Playback) => Replay.Target<S, Root, Schedules, Services, Resources, Machines>,
|
|
368
|
+
options: Options<Name, NoInfer<keyof Schedules & string>, Invariants>
|
|
369
|
+
& Session.ValidateInvariants<Invariants, Services, Resources, Machines>
|
|
370
|
+
& Replay.ValidateRecordable<Schedules[Name]>
|
|
371
|
+
& {
|
|
372
|
+
/** Fuzz every seat as its own peer over a seeded faulty network, with `Netcode.rollback` sessions. */
|
|
373
|
+
readonly network?: Net
|
|
374
|
+
}
|
|
375
|
+
& NetworkChecks<Net, S, Schedules[Name]>
|
|
376
|
+
): Result.Result<Rendered.Rendered<Passed>, FuzzError> => {
|
|
377
|
+
const invalid = (message: string) => Result.failure<FuzzError>({ _tag: "InvalidOptions", message })
|
|
378
|
+
const runs = options.runs ?? 10
|
|
379
|
+
const budget = options.shrinkAttempts ?? 500
|
|
380
|
+
if (!Number.isSafeInteger(options.frames) || options.frames < 1) return invalid("frames must be a whole number above 0")
|
|
381
|
+
if (!Number.isSafeInteger(runs) || runs < 1) return invalid("runs must be a whole number above 0")
|
|
382
|
+
if (!Number.isSafeInteger(options.seed)) return invalid("seed must be an integer")
|
|
383
|
+
if (!Number.isSafeInteger(budget) || budget < 0) return invalid("shrinkAttempts must be a whole number, 0 or more")
|
|
384
|
+
const inputs: ReadonlyArray<Input> = options.inputs
|
|
385
|
+
const indexOf = new Map(inputs.map((entry, index) => [entry.name, index] as const))
|
|
386
|
+
if (indexOf.size !== inputs.length) return invalid("each input must fuzz a different source")
|
|
387
|
+
const neutral: ReadonlyArray<unknown> = inputs.map((entry) => entry.neutral)
|
|
388
|
+
// `run`'s signature checked the invariants against this runtime's provisions.
|
|
389
|
+
const invariants = (options.invariants ?? []) as readonly []
|
|
390
|
+
const network: Network | undefined = options.network
|
|
391
|
+
let plan: FuzzNetwork.NetworkPlan | undefined
|
|
392
|
+
if (network !== undefined) {
|
|
393
|
+
const only = inputs[0]!
|
|
394
|
+
if (inputs.length !== 1 || !("codec" in only) || !("count" in only)) return invalid("over a network, give inputs exactly one Fuzz.seats")
|
|
395
|
+
const seated = only as Seats
|
|
396
|
+
const settle = network.settle ?? 120
|
|
397
|
+
if (!Number.isSafeInteger(settle) || settle < 0) return invalid("network.settle must be a whole number, 0 or more")
|
|
398
|
+
plan = {
|
|
399
|
+
name: seated.name, codec: seated.codec, count: seated.count, idle: seated.idle,
|
|
400
|
+
make, schedule: options.schedule, setup: options.setup, invariants: options.invariants ?? [],
|
|
401
|
+
step: network.step, seed: network.seed, latencyMS: network.latencyMS, jitterMS: network.jitterMS ?? 0, loss: network.loss ?? 0, settle,
|
|
402
|
+
inputDelay: network.inputDelay, maxRollback: network.maxRollback, checksumEvery: network.checksumEvery ?? 10
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* Runs one fresh simulation over `timeline`; `record` also saves it as a replay with that reason
|
|
408
|
+
* (a text: saved even when it passes).
|
|
409
|
+
*/
|
|
410
|
+
const simulateOne = (timeline: Timeline, record: ((violation: Violation, frame: number) => string) | string | undefined): Outcome => {
|
|
411
|
+
let current = neutral
|
|
412
|
+
const asked = new Set<string>()
|
|
413
|
+
const unknown: Array<string> = []
|
|
414
|
+
const playback: Replay.Playback = {
|
|
415
|
+
source: <const SourceName extends string, Value extends Source<unknown>>(descriptor: Descriptor<"service", SourceName, Value>): Source<Replay.SourceData<Value>> => {
|
|
416
|
+
asked.add(descriptor.name)
|
|
417
|
+
const index = indexOf.get(descriptor.name)
|
|
418
|
+
if (index === undefined) unknown.push(descriptor.name)
|
|
419
|
+
// The value was generated for this source's `input`, whose type `input` checked against the descriptor.
|
|
420
|
+
return { snapshot: () => (index === undefined ? undefined : current[index]) as Replay.SourceData<Value> }
|
|
421
|
+
},
|
|
422
|
+
constant: () => undefined
|
|
423
|
+
}
|
|
424
|
+
const target = make(playback)
|
|
425
|
+
if (unknown.length > 0) return { kind: "invalid", error: { _tag: "InvalidOptions", message: `make asked playback.source for ${unknown.join(", ")}, which no input fuzzes: add Fuzz.input for it, or provide it directly` } }
|
|
426
|
+
const unread = inputs.filter((entry) => !asked.has(entry.name)).map((entry) => entry.name)
|
|
427
|
+
if (unread.length > 0) return { kind: "invalid", error: { _tag: "InvalidOptions", message: `make never asked playback.source for ${unread.join(", ")}: provide the fuzzed source with playback.source(...)` } }
|
|
428
|
+
const schedules: Readonly<Record<string, AnySchedule<S, Root> | undefined>> = target.schedules
|
|
429
|
+
const schedule = schedules[options.schedule]
|
|
430
|
+
if (schedule === undefined) return { kind: "invalid", error: { _tag: "InvalidOptions", message: `make returned no schedule ${options.schedule}` } }
|
|
431
|
+
const runtime: Session.Target<S, Root> & Replay.Recordable<S, Root> = target.runtime
|
|
432
|
+
if (options.setup !== undefined) {
|
|
433
|
+
const setup = schedules[options.setup]
|
|
434
|
+
if (setup === undefined) return { kind: "invalid", error: { _tag: "InvalidOptions", message: `make returned no schedule ${options.setup}` } }
|
|
435
|
+
const ticked = runtime.tryTick(setup)
|
|
436
|
+
if (!ticked.ok) return { kind: "invalid", error: { _tag: "InvalidOptions", message: `setup failed with every input neutral: ${JSON.stringify(ticked.error)}` } }
|
|
437
|
+
}
|
|
438
|
+
const start = runtime.snapshot()
|
|
439
|
+
if (!start.ok) return { kind: "invalid", error: { _tag: "Unreplayable", error: start.error } }
|
|
440
|
+
|
|
441
|
+
const sink = Replay.memorySink()
|
|
442
|
+
// One window over the whole timeline, with its only snapshot at the start.
|
|
443
|
+
// `run`'s signature checked that the schedule's services can be recorded.
|
|
444
|
+
const recorder = record === undefined ? undefined : Replay.recorder(runtime, {
|
|
445
|
+
schedule, name: options.schedule, keep: timeline.length + 1, every: timeline.length + 1, step: Clock.hz(1), sink, hostClock: target.hostClock
|
|
446
|
+
} as never)
|
|
447
|
+
const session = Session.make(runtime, { schedules: { [options.schedule]: schedule }, hostClock: target.hostClock, invariants, history: 1 })
|
|
448
|
+
let ticked = 0
|
|
449
|
+
current = timeline[0] ?? neutral
|
|
450
|
+
const played = session.run(options.schedule, {
|
|
451
|
+
frames: timeline.length,
|
|
452
|
+
until: () => {
|
|
453
|
+
ticked += 1
|
|
454
|
+
current = timeline[ticked] ?? neutral
|
|
455
|
+
return false
|
|
456
|
+
}
|
|
457
|
+
})
|
|
458
|
+
session.close()
|
|
459
|
+
const found = violationOf(played.data.stop)
|
|
460
|
+
const frame = ticked + 1
|
|
461
|
+
const reason = found?._tag === "InvalidOptions" || record === undefined ? undefined
|
|
462
|
+
: typeof record === "string" ? record
|
|
463
|
+
: found === undefined ? undefined
|
|
464
|
+
: record(found, frame)
|
|
465
|
+
const file = reason === undefined ? undefined : recorder?.save(reason)
|
|
466
|
+
recorder?.stop()
|
|
467
|
+
if (found === undefined) return { kind: "passed", file }
|
|
468
|
+
if (found._tag === "InvalidOptions") return { kind: "invalid", error: found }
|
|
469
|
+
return { kind: "violated", violation: found, frame, file }
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Over the network: every seat on its own peer. The replay plays the same timeline on one runtime. */
|
|
473
|
+
const simulate = (timeline: Timeline, record: ((violation: Violation, frame: number) => string) | undefined): Outcome => {
|
|
474
|
+
if (plan === undefined) return simulateOne(timeline, record)
|
|
475
|
+
const played = FuzzNetwork.play(plan, timeline)
|
|
476
|
+
if (played.kind === "invalid") return { kind: "invalid", error: { _tag: "InvalidOptions", message: played.message } }
|
|
477
|
+
if (played.kind === "passed") return played
|
|
478
|
+
const one = record === undefined ? undefined : simulateOne(timeline.slice(0, played.frame), record(played.violation, played.frame))
|
|
479
|
+
return { kind: "violated", violation: played.violation, frame: played.frame, file: one?.kind === "invalid" ? undefined : one?.file }
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
for (let index = 0; index < runs; index++) {
|
|
483
|
+
const seed = options.seed + index
|
|
484
|
+
const random = Random.make(seed)
|
|
485
|
+
const timeline: Array<ReadonlyArray<unknown>> = []
|
|
486
|
+
for (let frame = 1; frame <= options.frames; frame++) timeline.push(inputs.map((entry) => entry.generate(random, frame)))
|
|
487
|
+
const first = simulate(timeline, undefined)
|
|
488
|
+
if (first.kind === "invalid") return Result.failure(first.error)
|
|
489
|
+
if (first.kind === "passed") continue
|
|
490
|
+
const target = signature(first.violation)
|
|
491
|
+
let attempts = 0
|
|
492
|
+
let steps = 0
|
|
493
|
+
let best: Timeline = timeline.slice(0, first.frame)
|
|
494
|
+
/** Whether `candidate` still fails the same way; on success, `best` becomes it, cut at its failing frame. */
|
|
495
|
+
const fails = (candidate: Timeline): boolean => {
|
|
496
|
+
attempts += 1
|
|
497
|
+
const outcome = simulate(candidate, undefined)
|
|
498
|
+
if (outcome.kind !== "violated" || signature(outcome.violation) !== target) return false
|
|
499
|
+
best = candidate.slice(0, outcome.frame)
|
|
500
|
+
steps += 1
|
|
501
|
+
return true
|
|
502
|
+
}
|
|
503
|
+
if (!fails(best)) return Result.failure({ _tag: "Flaky", violation: first.violation, seed, frame: first.frame })
|
|
504
|
+
steps = 0
|
|
505
|
+
shrink: for (let changed = true; changed;) {
|
|
506
|
+
changed = false
|
|
507
|
+
// Drop ranges of frames, halving the range when no range of that size can go.
|
|
508
|
+
for (let size = Math.max(1, Math.floor(best.length / 2)); size >= 1; size = Math.floor(size / 2)) {
|
|
509
|
+
for (let from = 0; from < best.length && best.length > 1;) {
|
|
510
|
+
if (attempts >= budget) break shrink
|
|
511
|
+
if (fails([...best.slice(0, from), ...best.slice(from + size)])) changed = true
|
|
512
|
+
else from += size
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
// Move each remaining value toward its input's neutral.
|
|
516
|
+
for (let frame = 0; frame < best.length; frame++) {
|
|
517
|
+
for (let slot = 0; slot < inputs.length; slot++) {
|
|
518
|
+
for (const candidate of simpler(best[frame]![slot], neutral[slot])) {
|
|
519
|
+
if (attempts >= budget) break shrink
|
|
520
|
+
const values = [...best[frame]!]
|
|
521
|
+
values[slot] = candidate
|
|
522
|
+
if (fails([...best.slice(0, frame), values, ...best.slice(frame + 1)])) {
|
|
523
|
+
changed = true
|
|
524
|
+
break
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
const final = simulate(best, (violation, frame) =>
|
|
531
|
+
`fuzz seed ${seed}${network === undefined ? "" : ` over network seed ${network.seed}`}: ${describe(violation)} at frame ${frame} (shrunk from ${Math.min(first.frame, timeline.length)} frames)`)
|
|
532
|
+
if (final.kind !== "violated" || final.file === undefined || signature(final.violation) !== target) {
|
|
533
|
+
return Result.failure({ _tag: "Flaky", violation: first.violation, seed, frame: first.frame })
|
|
534
|
+
}
|
|
535
|
+
const opened = Replay.session(make, final.file, { invariants })
|
|
536
|
+
const replayed = opened.ok ? violationOf(opened.value.play().data.run.stop) : undefined
|
|
537
|
+
return Result.failure({
|
|
538
|
+
_tag: "Found",
|
|
539
|
+
violation: final.violation,
|
|
540
|
+
seed,
|
|
541
|
+
frame: final.frame,
|
|
542
|
+
replay: final.file,
|
|
543
|
+
reproduces: replayed !== undefined && replayed._tag !== "InvalidOptions" && signature(replayed) === target,
|
|
544
|
+
shrink: { attempts, steps, fromFrames: Math.min(first.frame, timeline.length), toFrames: best.length }
|
|
545
|
+
})
|
|
546
|
+
}
|
|
547
|
+
return Result.success(Rendered.make(
|
|
548
|
+
{ runs, frames: options.frames, seed: options.seed },
|
|
549
|
+
`${runs} run${runs === 1 ? "" : "s"} of ${options.frames} frames from seed ${options.seed}: no invariant broke, no system failed, nothing threw`
|
|
550
|
+
))
|
|
551
|
+
}
|
package/src/Invariant.ts
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
* invariant that reads a resource, service, or machine the runtime does not
|
|
9
9
|
* provide. The invariant is named after its inspector.
|
|
10
10
|
*
|
|
11
|
+
* Write entities in the message as `e${id}`: a session with `labels` prints
|
|
12
|
+
* each one with its label (`player e12`) in the run summary; the run's
|
|
13
|
+
* `stop.message` keeps the text the inspector returned.
|
|
14
|
+
*
|
|
11
15
|
* Use `Invariant.fromInspector` for checks over the world during a session.
|
|
12
16
|
* Use `Game.Inspector` with `runtime.inspect(...)` directly for one-off reads
|
|
13
17
|
* in tests and scripts.
|
|
@@ -17,12 +21,15 @@
|
|
|
17
21
|
*
|
|
18
22
|
* @example
|
|
19
23
|
* ```ts
|
|
24
|
+
* const WORLD_WIDTH = 800
|
|
25
|
+
* const hostClock = HostClock.test()
|
|
26
|
+
* // ---cut---
|
|
20
27
|
* const PlayerInsideWorld = Invariant.fromInspector(Game.Inspector("player inside world", {
|
|
21
28
|
* queries: { players: Game.Query({ selection: { position: Game.Query.read(Position) }, with: [Player] }) }
|
|
22
29
|
* }, ({ queries }) => {
|
|
23
|
-
* for (const { data } of queries.players.each()) {
|
|
30
|
+
* for (const { entity, data } of queries.players.each()) {
|
|
24
31
|
* const { x } = data.position.get()
|
|
25
|
-
* if (x < 0 || x > WORLD_WIDTH) return `
|
|
32
|
+
* if (x < 0 || x > WORLD_WIDTH) return `e${entity.id} at x=${x}`
|
|
26
33
|
* }
|
|
27
34
|
* return undefined
|
|
28
35
|
* }))
|
package/src/Lint.ts
CHANGED
|
@@ -25,11 +25,17 @@
|
|
|
25
25
|
*
|
|
26
26
|
* @example
|
|
27
27
|
* ```ts
|
|
28
|
-
* // src/
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
28
|
+
* // The game's own modules: src/simulation.ts and src/render.ts.
|
|
29
|
+
* declare const createSimulation: (devices: { readonly keyboard: Keyboard.Actions<typeof bindings> }) => {
|
|
30
|
+
* readonly runtime: typeof runtime
|
|
31
|
+
* readonly setup: typeof setup
|
|
32
|
+
* readonly update: typeof update
|
|
33
|
+
* }
|
|
34
|
+
* declare const importRender: () => Promise<{ readonly render: typeof update }> // import("./render.ts")
|
|
35
|
+
* // ---cut---
|
|
36
|
+
* // src/lint.ts: render code loads only inside lintTarget, never in the simulation.
|
|
37
|
+
* export const lintTarget = async (): Promise<Lint.Target<typeof Game.schema, typeof Root>> => {
|
|
38
|
+
* const { render } = await importRender()
|
|
33
39
|
* const sim = createSimulation({ keyboard: Keyboard.scripted(bindings, []) })
|
|
34
40
|
* return { runtime: sim.runtime, schedules: { setup: sim.setup, update: sim.update }, describe: { render } }
|
|
35
41
|
* }
|