@bytewaveco/pudding 0.0.0-stage → 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +363 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +143 -0
- package/dist/crosscheck.d.ts +63 -0
- package/dist/crosscheck.js +164 -0
- package/dist/index.d.ts +76 -0
- package/dist/index.js +72 -0
- package/dist/install.d.ts +25 -0
- package/dist/install.js +192 -0
- package/dist/ir.d.ts +93 -0
- package/dist/ir.js +331 -0
- package/dist/lean.d.ts +56 -0
- package/dist/lean.js +438 -0
- package/dist/report.d.ts +7 -0
- package/dist/report.js +74 -0
- package/dist/runner.d.ts +79 -0
- package/dist/runner.js +672 -0
- package/dist/search.d.ts +24 -0
- package/dist/search.js +137 -0
- package/dist/translate.d.ts +247 -0
- package/dist/translate.js +1938 -0
- package/dist/vitest.d.ts +8 -0
- package/dist/vitest.js +96 -0
- package/package.json +77 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kaden Griffith
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,364 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Pudding
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Pudding proves laws about TypeScript functions you already have. You write a
|
|
4
|
+
law the way you would write a Vitest test, against the real implementation, and
|
|
5
|
+
Pudding proves it for **every** input instead of the ones you thought of. Lean 4
|
|
6
|
+
checks the proofs.
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
// cart.law.ts
|
|
10
|
+
import { describe, domain, expect, law, type Integer } from "@bytewaveco/pudding"
|
|
11
|
+
import { subtotal, pageCount, type Item } from "./cart"
|
|
12
|
+
|
|
13
|
+
describe("cart", () => {
|
|
14
|
+
law("adding an item adds its cost", domain<[Item, Item]>(), ([first, second]) => {
|
|
15
|
+
expect(subtotal([first, second])).toEqual(subtotal([first]) + second.price * second.quantity)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
law("pages cover every item", domain<[Integer<0, 1000000>, Integer<1, 50>]>(), ([items, perPage]) => {
|
|
19
|
+
expect(pageCount(items, perPage) * perPage >= items).toBe(true)
|
|
20
|
+
})
|
|
21
|
+
})
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The implementation is not annotated, rewritten or moved. Law files are ordinary
|
|
25
|
+
TypeScript: they type-check, format and refactor with the rest of the project.
|
|
26
|
+
|
|
27
|
+
## Getting started
|
|
28
|
+
|
|
29
|
+
Pudding needs Node 24. It brings its own TypeScript 7 for reading your code,
|
|
30
|
+
and it needs Lean 4.36 for the proofs.
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
npm install --save-dev @bytewaveco/pudding@beta
|
|
34
|
+
npx pudding # check every *.law.ts
|
|
35
|
+
npx pudding watch # check again on every change
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
There is nothing to configure. Law files do not have to be listed in a
|
|
39
|
+
tsconfig, and a project without a tsconfig works.
|
|
40
|
+
|
|
41
|
+
On macOS and Linux, arm64 and x64, Lean arrives with `npm install` as an
|
|
42
|
+
optional dependency: a package of about 110 MB holding the part of Lean that
|
|
43
|
+
Pudding uses. Where it did not arrive, because optional dependencies are
|
|
44
|
+
switched off or the package manager skipped it, `pudding` offers to download
|
|
45
|
+
Lean the first time, and `npx pudding install-lean` does the same without
|
|
46
|
+
asking. That fetches the official Lean release, checks it against a recorded
|
|
47
|
+
SHA-256, and keeps the same part of it in a cache shared by all your projects.
|
|
48
|
+
To use a Lean you already have, set `PUDDING_LEAN` to the binary; a Lean
|
|
49
|
+
installed with elan is found on its own.
|
|
50
|
+
|
|
51
|
+
## The command line
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
pudding [filters...] check law files whose path contains a filter
|
|
55
|
+
pudding watch [filters...] check again whenever a file changes
|
|
56
|
+
pudding install-lean download the Lean runtime
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
| Option | Effect |
|
|
60
|
+
|---|---|
|
|
61
|
+
| `-t, --name <pattern>` | only laws whose full name matches the regular expression |
|
|
62
|
+
| `-w, --watch` | same as the `watch` command |
|
|
63
|
+
| `--strict` | fail when a law was only executed, not proved |
|
|
64
|
+
| `--reporter <name>` | `default`, `verbose`, `json` or `github` |
|
|
65
|
+
| `--no-cache` | ignore results from earlier runs |
|
|
66
|
+
| `--no-cross-check` | do not execute laws against the real code |
|
|
67
|
+
| `--emit <file>` | also write the generated Lean source |
|
|
68
|
+
| `--cwd <dir>` | project directory |
|
|
69
|
+
|
|
70
|
+
The `github` reporter is chosen automatically inside GitHub Actions and
|
|
71
|
+
annotates the lines that failed. The same run is available from code as
|
|
72
|
+
`run()` in `@bytewaveco/pudding/runner`.
|
|
73
|
+
|
|
74
|
+
## Running under Vitest
|
|
75
|
+
|
|
76
|
+
A law file is also a Vitest test file. Add the pattern to the files Vitest
|
|
77
|
+
collects, and every law shows up as a test next to your unit tests:
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
// vitest.config.ts
|
|
81
|
+
import { defineConfig } from "vitest/config"
|
|
82
|
+
|
|
83
|
+
export default defineConfig({
|
|
84
|
+
test: { include: ["**/*.test.ts", "**/*.law.ts"] },
|
|
85
|
+
})
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
✓ src/cart.test.ts (1 test)
|
|
90
|
+
❯ src/cart.law.ts (2 tests | 1 failed)
|
|
91
|
+
✓ the order of two items does not matter
|
|
92
|
+
× FALSE: quantity is ignored
|
|
93
|
+
|
|
94
|
+
FAIL src/cart.law.ts > FALSE: quantity is ignored
|
|
95
|
+
LawFailed: the expectation does not hold
|
|
96
|
+
expect(subtotal([item])).toEqual(item.price)
|
|
97
|
+
with item.price = 1n, item.quantity = 0n
|
|
98
|
+
actual: 0n
|
|
99
|
+
expected: 1n
|
|
100
|
+
❯ src/cart.law.ts:8:3
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
A passing test means what it means on the command line: the law was proved,
|
|
104
|
+
or executed when it could not be, and a law that was only executed carries a
|
|
105
|
+
note saying so. `describe`, `law.skip`, `law.only` and `law.todo` become
|
|
106
|
+
their Vitest counterparts, and `pudding.config.ts` still applies, so `strict`
|
|
107
|
+
turns executed-only laws into failures here too. Watch mode, `-t`, the UI and
|
|
108
|
+
editor integrations work as they do for any test.
|
|
109
|
+
|
|
110
|
+
Each file is proved by its own Pudding run when its first law starts, so a
|
|
111
|
+
law file costs a few hundred milliseconds more under Vitest than under
|
|
112
|
+
`pudding`, which reads all of them in one go. Vitest is an optional peer
|
|
113
|
+
dependency, version 3 or newer.
|
|
114
|
+
|
|
115
|
+
## Configuration
|
|
116
|
+
|
|
117
|
+
A `pudding.config.ts` is optional.
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
import { defineConfig } from "@bytewaveco/pudding"
|
|
121
|
+
|
|
122
|
+
export default defineConfig({
|
|
123
|
+
include: ["src/**/*.law.ts"], // default: **/*.law.ts
|
|
124
|
+
exclude: ["src/legacy/**"],
|
|
125
|
+
tsconfig: ".nuxt/tsconfig.app.json", // whose paths and options to read the laws with
|
|
126
|
+
strict: false, // fail laws that were only executed
|
|
127
|
+
crossCheck: true, // execute laws against the real code
|
|
128
|
+
reporter: "default",
|
|
129
|
+
})
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`tsconfig` matters when your imports use path aliases that the root
|
|
133
|
+
`tsconfig.json` does not define, as in a Nuxt project.
|
|
134
|
+
|
|
135
|
+
## Writing laws
|
|
136
|
+
|
|
137
|
+
```ts
|
|
138
|
+
law(name, domain<T>(), input => { ... }, options?)
|
|
139
|
+
law(name, () => { ... }, options?) // fixed values, no domain
|
|
140
|
+
law.skip(...) law.only(...) law.todo(name)
|
|
141
|
+
describe(name, () => { ... }) // also describe.skip, describe.only
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`domain<T>()` describes the inputs by type.
|
|
145
|
+
|
|
146
|
+
| Type | Inputs covered |
|
|
147
|
+
|---|---|
|
|
148
|
+
| `boolean`, a union of string, bigint or number literals | every member |
|
|
149
|
+
| `bigint`, `string` | every value |
|
|
150
|
+
| `Integer` | every whole number a `number` holds exactly |
|
|
151
|
+
| `Integer<Min, Max>` | every whole number from `Min` to `Max` |
|
|
152
|
+
| an object type | every plain object with exactly its properties, in every combination of their values |
|
|
153
|
+
| a tuple | every combination of its elements |
|
|
154
|
+
| a union of object types | every alternative |
|
|
155
|
+
|
|
156
|
+
A plain `number` cannot be an input, because it also covers fractions, NaN and
|
|
157
|
+
infinities. Functions that take a `number` accept an `Integer`. An object type
|
|
158
|
+
with an index signature, such as `Record<string, boolean>`, cannot be an input
|
|
159
|
+
either, because its values cannot be listed.
|
|
160
|
+
|
|
161
|
+
A law's callback is an ordinary synchronous function, and a law file contains
|
|
162
|
+
only `law()` and `describe()` calls and declarations at its top level, so that
|
|
163
|
+
every law that would be registered is also read.
|
|
164
|
+
|
|
165
|
+
Inside a law you can use anything from the supported subset below: declare
|
|
166
|
+
values, loop, guard with `if`, leave early with `return`, call helpers that
|
|
167
|
+
assert, and assert with `expect(a).toEqual(b)`, `expect(a).toBe(b)` and their
|
|
168
|
+
`.not` forms.
|
|
169
|
+
|
|
170
|
+
### Supplying a proof
|
|
171
|
+
|
|
172
|
+
Laws over finite inputs are decided by evaluation and never need help. Laws
|
|
173
|
+
over unbounded inputs use Lean's `grind`, which handles linear arithmetic,
|
|
174
|
+
case splits, and division and remainder by constants. When that is not enough,
|
|
175
|
+
pass a Lean tactic script:
|
|
176
|
+
|
|
177
|
+
```ts
|
|
178
|
+
law("a product of positives is positive", domain<[bigint, bigint]>(), ([a, b]) => {
|
|
179
|
+
if (a > 0n && b > 0n) expect(a * b > 0n).toBe(true)
|
|
180
|
+
}, { proof: "intro h; simp_all; exact Int.mul_pos h.1 h.2" })
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
`pudding --emit laws.lean` writes the theorems, so you or an agent can work on
|
|
184
|
+
a proof with ordinary Lean tooling. Nobody has to review the script: Lean
|
|
185
|
+
rejects it if it is wrong or unfinished. A script is a sequence of tactics and
|
|
186
|
+
nothing else. It is checked in a file of its own, and it may not contain
|
|
187
|
+
comments, string literals or unbalanced parentheses, so that it cannot go on
|
|
188
|
+
to define anything. The trimmed Lean that `install-lean` leaves holds
|
|
189
|
+
only the part of Lean's library Pudding itself uses; a proof that names other
|
|
190
|
+
lemmas needs `pudding install-lean --full` or your own Lean.
|
|
191
|
+
|
|
192
|
+
## What a run reports
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
✗ money.law.ts (3 laws)
|
|
196
|
+
✓ clamp stays within its bounds
|
|
197
|
+
✗ clamp is idempotent
|
|
198
|
+
the expectation does not hold
|
|
199
|
+
money.law.ts:12:3 expect(clamp(clamp(x, lo, hi), lo, hi)).toEqual(clamp(x, lo, hi))
|
|
200
|
+
with x = 0n, lo = 0n, hi = -1n
|
|
201
|
+
actual: 0n
|
|
202
|
+
expected: -1n
|
|
203
|
+
~ cents survive a round trip executed on sampled inputs, not proved
|
|
204
|
+
note: not proved: calling `.padStart` on a string is not supported (money.ts:38:56)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
| State | Meaning |
|
|
208
|
+
|---|---|
|
|
209
|
+
| **proved** | Lean accepted a proof for every input. |
|
|
210
|
+
| **checked** | The code is outside the supported subset, so the law was executed against the real code for every input of a finite domain. |
|
|
211
|
+
| **sampled** | As above, but the domain is unbounded, so the law was executed on a sample. |
|
|
212
|
+
| **failed** | The law is false, proves nothing, or could not be verified at all. |
|
|
213
|
+
| **skipped**, **todo** | Marked that way, or filtered out. |
|
|
214
|
+
|
|
215
|
+
A failed law says why:
|
|
216
|
+
|
|
217
|
+
- **A counterexample.** The inputs are named as the law names them, with both sides of the failed equality.
|
|
218
|
+
- **A throw that can be reached**, including division by zero, a lookup with a key the table does not have, and a function that ends without returning. A law holds only if the code it calls cannot throw for the inputs it covers.
|
|
219
|
+
- **A law file that throws when it is loaded.** Nothing in it holds at run time, whatever was proved.
|
|
220
|
+
- **A number that stops being exact.** A result beyond 2^53 − 1, or a remainder by zero.
|
|
221
|
+
- **An expectation that is never reached.** A guard that is never true would otherwise pass silently. When Pudding can neither find an input that reaches an expectation nor prove that none exists, the law keeps its verdict and carries a note saying so.
|
|
222
|
+
- **No proof and no counterexample.** The law may be true but beyond the automation. Supply a proof.
|
|
223
|
+
- **A disagreement between the model and the running code.** See "What a proof means".
|
|
224
|
+
|
|
225
|
+
`--strict` turns *checked* and *sampled* into failures, for CI on modules where
|
|
226
|
+
only a proof will do.
|
|
227
|
+
|
|
228
|
+
## The supported subset
|
|
229
|
+
|
|
230
|
+
Pudding reads only the code a law reaches. A module may contain anything else.
|
|
231
|
+
|
|
232
|
+
- **Values:** `boolean`, `bigint`, `string`, whole `number`s, `undefined`, object literals, array literals and tuples, functions.
|
|
233
|
+
- **Operators:** `===` `!==` `&&` `||` `!` `?:`; `+ - * % < <= > >=` on bigint and number; `/` on bigint; string `+` and template literals.
|
|
234
|
+
- **Statements:** `const` and `let`, assignment and `+=`-style updates of local variables, `if`/`else`, `switch`, `for…of` over an array of known length, `for`, `while` and `do` loops whose condition is settled at each round, `break`, `continue`, `return`, `throw`.
|
|
235
|
+
- **Functions:** declarations and arrows, in any module of the project, with defaults and destructured parameters.
|
|
236
|
+
- **Data:** module-level `const` tables, indexed by a key whose type guarantees it is present; `.includes`, `.some`, `.every`, `.map`, `.reduce` and `.length` on arrays of known length.
|
|
237
|
+
- **Numbers:** `Math.floor`, `ceil` and `trunc` of a quotient or a value, `Math.abs`, `sign`, `min`, `max`, and `String`, `Number` and `BigInt` conversions.
|
|
238
|
+
|
|
239
|
+
Refused, with the reason, at the exact expression:
|
|
240
|
+
|
|
241
|
+
- fractions, plain `number` division and `Math.round` of a quotient;
|
|
242
|
+
- changing an object or array in place, because two names can refer to the same object;
|
|
243
|
+
- a function that uses a variable which is reassigned outside it;
|
|
244
|
+
- reading a `const` or `let` before its declaration has run, and `__proto__`;
|
|
245
|
+
- an import such as `./x.js` when both `x.js` and `x.ts` exist, because TypeScript reads one and Node runs the other;
|
|
246
|
+
- `var`, loops whose length depends on an input, recursion, classes, async code, regular expressions and library calls.
|
|
247
|
+
|
|
248
|
+
A refused law is executed instead and reported as *checked* or *sampled*, so a
|
|
249
|
+
law can always be written and gets promoted to *proved* when the subset grows.
|
|
250
|
+
|
|
251
|
+
## What a proof means
|
|
252
|
+
|
|
253
|
+
A *proved* law is a theorem about a model of your code. These things stand
|
|
254
|
+
between the theorem and the code that runs.
|
|
255
|
+
|
|
256
|
+
1. **The translation.** Pudding's translator and its constant folding are trusted. To catch mistakes, every law is also executed against the real code and compared with the model, on all inputs of a small finite domain or a sample of a large one that includes the constants the law mentions. A disagreement fails the law. If the law file cannot be executed, the run says so in a warning. With `--no-cross-check` none of this happens, including noticing that a module throws when it loads.
|
|
257
|
+
2. **Inputs are plain data.** An object input has exactly the properties its type declares, held as ordinary values. TypeScript would also accept an object with extra properties, or one whose properties are getters; those are not covered.
|
|
258
|
+
3. **Module data.** A table such as `const transitions = {...}` is assumed to hold its initial value when a law runs. Anywhere in its module, a table may only be read in ways known to change nothing; anything else, such as passing it to a function, calling a method it defines, or giving it another name, makes Pudding refuse it. Two things are taken on trust: an installed package given a table declared `as const` does not change it, and no other module changes an exported table.
|
|
259
|
+
4. **Built-ins.** `Math`, `String`, array methods, iteration and the like are modeled as the language defines them. If any file of the project assigns to a built-in, no law is proved; they are executed instead. Pudding does not look inside installed packages.
|
|
260
|
+
5. **The files that run are the files TypeScript read.** Laws are executed against the files TypeScript resolved each import to. Where Node on its own would load a different file, as with some conditional `imports` and `exports`, the run says so in a warning.
|
|
261
|
+
6. **Numbers.** A `number` is modeled as the whole number it holds, and every result must be proved to stay within 2^53 − 1, where that model is exact. Zero and negative zero are one value, as `===` treats them.
|
|
262
|
+
7. **Strings.** `string` inputs range over well-formed Unicode text. JavaScript strings with unpaired surrogates are not covered.
|
|
263
|
+
8. **Lean.** Finite laws are checked by Lean's kernel. Other proofs are found by tactics and checked by the kernel. A theorem counts only when Lean reports no error for it and confirms that it rests on nothing but Lean's standard axioms, so an unfinished proof (`sorry`) is rejected.
|
|
264
|
+
|
|
265
|
+
## How it works
|
|
266
|
+
|
|
267
|
+
1. **Read.** The TypeScript 7 compiler API type-checks the law files and resolves every name and type. Type errors stop the run.
|
|
268
|
+
2. **Translate.** Each law is evaluated symbolically. Functions are inlined, objects, arrays and loops are resolved, and what remains is a set of obligations over booleans, integers and strings: one per `expect`, plus one per reachable `throw`, division, or number that could stop being exact.
|
|
269
|
+
3. **Prove.** Obligations over finite inputs become one Lean theorem each, decided by evaluating every case in the kernel. Obligations with unbounded inputs are handed to `grind`, with a slower fallback under the full library before a rejection is reported.
|
|
270
|
+
4. **Explain.** For a rejected obligation Pudding searches for a counterexample by evaluating the model in JavaScript.
|
|
271
|
+
5. **Cross-check.** In a worker thread, each law is executed against the real code and compared with the model.
|
|
272
|
+
|
|
273
|
+
Proved theorems are remembered by their full text and the Lean version, so
|
|
274
|
+
Lean is only asked about what changed. The TypeScript side is read afresh on
|
|
275
|
+
every run: reusing a whole result would mean guessing that nothing affecting
|
|
276
|
+
module resolution had changed.
|
|
277
|
+
|
|
278
|
+
## Developing
|
|
279
|
+
|
|
280
|
+
```sh
|
|
281
|
+
npm install
|
|
282
|
+
npx pudding install-lean
|
|
283
|
+
npm test # builds, then runs the tests
|
|
284
|
+
npm run proof # builds, then checks the examples
|
|
285
|
+
npm run fuzz -- --seed 3 --programs 2000
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
`npm run fuzz` is a differential fuzzer. It generates random functions inside
|
|
289
|
+
the supported subset, runs each in Node on every input of a small finite
|
|
290
|
+
domain, and checks that Pudding proves the law stating exactly those results
|
|
291
|
+
and never proves the same law with one result changed. It leans on the
|
|
292
|
+
scoping, hoisting and `undefined` rules that are easiest to model wrongly.
|
|
293
|
+
|
|
294
|
+
With `--unbounded` the inputs are all bigints or a wide range of whole
|
|
295
|
+
numbers, which Pudding proves with tactics. No table of results exists there,
|
|
296
|
+
so each function is compared with a copy that has one constant or operator
|
|
297
|
+
changed; when Node finds an input on which the two differ, the law that they
|
|
298
|
+
are equal must not be proved. Programs that expose a problem are kept under
|
|
299
|
+
`.lab/fuzz/failures/`.
|
|
300
|
+
|
|
301
|
+
- [src/translate.ts](src/translate.ts): TypeScript to obligations.
|
|
302
|
+
- [src/ir.ts](src/ir.ts): the expression language, its simplifier and its evaluator.
|
|
303
|
+
- [src/lean.ts](src/lean.ts): obligations to Lean theorems, and running Lean.
|
|
304
|
+
- [src/runner.ts](src/runner.ts): the run, caching and explaining failures.
|
|
305
|
+
- [src/crosscheck.ts](src/crosscheck.ts): executing laws against the real code.
|
|
306
|
+
- [scripts/pack-runtime.mjs](scripts/pack-runtime.mjs): builds the trimmed Lean runtime for a platform.
|
|
307
|
+
- [scripts/fuzz.mjs](scripts/fuzz.mjs): the differential fuzzer.
|
|
308
|
+
- [examples/](examples): laws that pass, over small modules of the kind business code is made of: money, orders, stock, leave requests.
|
|
309
|
+
- [test/fixtures/](test/fixtures): laws that are meant to fail, be vacuous, or fall back, with the tests that pin each outcome.
|
|
310
|
+
|
|
311
|
+
### Shipping Lean as a package
|
|
312
|
+
|
|
313
|
+
Pudding looks for a package named `<its own name>-lean-<platform>-<arch>`,
|
|
314
|
+
so Lean is an ordinary optional dependency instead of a separate download.
|
|
315
|
+
Only `PUDDING_LEAN`, the `lean` setting and a whole Lean from
|
|
316
|
+
`install-lean --full` go before it. `scripts/pack-runtime.mjs` builds that package from an
|
|
317
|
+
official toolchain: the `lean` binary, its stripped shared libraries and the
|
|
318
|
+
264 compiled modules the generated theorems import.
|
|
319
|
+
|
|
320
|
+
```sh
|
|
321
|
+
npx pudding install-lean --full --local
|
|
322
|
+
npm run pack:runtime -- .toolchain/lean-4.36.0-darwin_aarch64 runtime --package
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
The result is about 360 MB on disk and 108 to 116 MB as the `.tar.gz` npm
|
|
326
|
+
uploads, depending on the platform. On macOS arm64 it gives the same verdicts
|
|
327
|
+
as the full toolchain on the tests, the examples and a benchmark of 4,661
|
|
328
|
+
generated theorems, and on Linux arm64 and x64 the tests and the examples pass
|
|
329
|
+
against it. The Intel macOS package has been built but not run. The package is versioned
|
|
330
|
+
after Lean: Lean 4.36.0 gives 4.36.0, and `--build 1` gives 4.36.1 when the
|
|
331
|
+
set of modules changes under the same Lean.
|
|
332
|
+
|
|
333
|
+
### Releasing
|
|
334
|
+
|
|
335
|
+
Two workflows in `.github/workflows` do the work. `ci.yml` runs the tests, the
|
|
336
|
+
examples, the package smoke test and a short fuzzing run on Linux and macOS,
|
|
337
|
+
arm64 and x64. `release.yml` is started by hand and publishes one of two
|
|
338
|
+
things.
|
|
339
|
+
|
|
340
|
+
1. **The runtime packages**, only when Lean or the imports in `src/lean.ts`
|
|
341
|
+
change. Run Release with target `runtime`. Each platform builds its own
|
|
342
|
+
package, runs the tests against it and publishes it. Then pin the new
|
|
343
|
+
version here and commit:
|
|
344
|
+
|
|
345
|
+
```sh
|
|
346
|
+
npm install --save-optional --save-exact \
|
|
347
|
+
@bytewaveco/pudding-lean-darwin-arm64@4.36.0 @bytewaveco/pudding-lean-darwin-x64@4.36.0 \
|
|
348
|
+
@bytewaveco/pudding-lean-linux-arm64@4.36.0 @bytewaveco/pudding-lean-linux-x64@4.36.0
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
2. **Pudding itself.** Set the version in `package.json`, commit, and run
|
|
352
|
+
Release with target `package`. It is published under the dist-tag in
|
|
353
|
+
`publishConfig`, which is `beta` for now.
|
|
354
|
+
|
|
355
|
+
Both need an `NPM_TOKEN` secret that may publish to the `@bytewaveco` scope.
|
|
356
|
+
Tick `dry-run` to go through everything except the upload.
|
|
357
|
+
|
|
358
|
+
## Limits worth knowing
|
|
359
|
+
|
|
360
|
+
- The TypeScript API Pudding uses is published under `typescript/unstable`, so the TypeScript version is pinned.
|
|
361
|
+
- Division by an unknown and products of two unknowns are outside `grind`'s arithmetic. Bound the divisor with a small `Integer<Min, Max>`, which is checked value by value, or supply a proof.
|
|
362
|
+
- Symbolic strings support equality and concatenation only.
|
|
363
|
+
- Each obligation is printed as a tree, so deeply nested calls of functions that use an argument several times grow quickly. Pudding refuses terms past a size limit.
|
|
364
|
+
- Prebuilt Lean downloads exist for macOS and Linux only.
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync, watch as watchFiles } from "node:fs";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { parseArgs } from "node:util";
|
|
6
|
+
import { createInterface } from "node:readline/promises";
|
|
7
|
+
import { installLean, LEAN_VERSION } from "./install.js";
|
|
8
|
+
import { format } from "./report.js";
|
|
9
|
+
import { LeanMissing, run } from "./runner.js";
|
|
10
|
+
const HELP = `pudding: prove laws about TypeScript functions
|
|
11
|
+
|
|
12
|
+
Usage
|
|
13
|
+
pudding [filters...] [options] check law files; filters match parts of their paths
|
|
14
|
+
pudding watch [filters...] check again whenever a file changes
|
|
15
|
+
pudding install-lean download the Lean runtime
|
|
16
|
+
|
|
17
|
+
Options
|
|
18
|
+
-t, --name <pattern> only laws whose name matches this regular expression
|
|
19
|
+
-w, --watch same as the watch command
|
|
20
|
+
--strict fail when a law was only executed, not proved
|
|
21
|
+
--reporter <name> default, verbose, json or github
|
|
22
|
+
--no-cache ignore results from earlier runs
|
|
23
|
+
--no-cross-check do not execute laws against the real code
|
|
24
|
+
--emit <file> also write the generated Lean source to <file>
|
|
25
|
+
--cwd <dir> project directory (default: current directory)
|
|
26
|
+
--full with install-lean: keep the whole toolchain
|
|
27
|
+
--local with install-lean: install into ./.toolchain, not the shared cache
|
|
28
|
+
-v, --version
|
|
29
|
+
-h, --help
|
|
30
|
+
`;
|
|
31
|
+
const REPORTERS = ["default", "verbose", "json", "github"];
|
|
32
|
+
const IGNORED = /(^|[\\/])(node_modules|\.git|dist|\.pudding|\.toolchain|\.lab)([\\/]|$)/;
|
|
33
|
+
async function main() {
|
|
34
|
+
const { values, positionals } = parseArgs({
|
|
35
|
+
allowPositionals: true,
|
|
36
|
+
options: {
|
|
37
|
+
name: { type: "string", short: "t" }, watch: { type: "boolean", short: "w" }, strict: { type: "boolean" },
|
|
38
|
+
reporter: { type: "string" }, "no-cache": { type: "boolean" }, "no-cross-check": { type: "boolean" }, emit: { type: "string" },
|
|
39
|
+
cwd: { type: "string" }, full: { type: "boolean" }, local: { type: "boolean" }, version: { type: "boolean", short: "v" }, help: { type: "boolean", short: "h" },
|
|
40
|
+
// Kept so earlier invocations still work.
|
|
41
|
+
json: { type: "boolean" }, verbose: { type: "boolean" },
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
const cwd = resolve(values.cwd ?? process.cwd());
|
|
45
|
+
if (values.help)
|
|
46
|
+
return void process.stdout.write(HELP);
|
|
47
|
+
if (values.version) {
|
|
48
|
+
const manifest = JSON.parse(readFileSync(join(dirname(dirname(fileURLToPath(import.meta.url))), "package.json"), "utf8"));
|
|
49
|
+
return void console.log(manifest.version);
|
|
50
|
+
}
|
|
51
|
+
if (positionals[0] === "install-lean") {
|
|
52
|
+
return void (await installLean(cwd, { full: !!values.full, local: !!values.local, log: line => console.log(line) }));
|
|
53
|
+
}
|
|
54
|
+
const watching = values.watch || positionals[0] === "watch";
|
|
55
|
+
const filters = positionals[0] === "watch" ? positionals.slice(1) : positionals;
|
|
56
|
+
const named = values.reporter ?? (values.json ? "json" : values.verbose ? "verbose" : undefined);
|
|
57
|
+
if (named !== undefined && !REPORTERS.includes(named))
|
|
58
|
+
throw new Error(`unknown reporter \`${named}\`; choose one of ${REPORTERS.join(", ")}`);
|
|
59
|
+
const options = {
|
|
60
|
+
cwd,
|
|
61
|
+
...(filters.length ? { files: filters } : {}),
|
|
62
|
+
...(values.name ? { name: values.name } : {}),
|
|
63
|
+
...(values.strict ? { strict: true } : {}),
|
|
64
|
+
...(values["no-cache"] ? { cache: false } : {}),
|
|
65
|
+
...(values["no-cross-check"] ? { crossCheck: false } : {}),
|
|
66
|
+
...(values.emit ? { emit: values.emit } : {}),
|
|
67
|
+
};
|
|
68
|
+
const once = async () => {
|
|
69
|
+
const report = await run(options);
|
|
70
|
+
const reporter = (named ?? report.reporter ?? (process.env["GITHUB_ACTIONS"] ? "github" : "default"));
|
|
71
|
+
process.stdout.write(format(report, { cwd, color: process.stdout.isTTY === true && reporter !== "json", reporter }));
|
|
72
|
+
return report.ok;
|
|
73
|
+
};
|
|
74
|
+
if (!watching)
|
|
75
|
+
return once().then(ok => void (process.exitCode = ok ? 0 : 1));
|
|
76
|
+
return watch(cwd, once);
|
|
77
|
+
}
|
|
78
|
+
/** Check, then check again after each change to a source file, until interrupted. */
|
|
79
|
+
async function watch(cwd, once) {
|
|
80
|
+
let running = false;
|
|
81
|
+
let again = false;
|
|
82
|
+
let timer;
|
|
83
|
+
const interactive = process.stdout.isTTY === true;
|
|
84
|
+
const cycle = async () => {
|
|
85
|
+
if (running)
|
|
86
|
+
return void (again = true);
|
|
87
|
+
running = true;
|
|
88
|
+
do {
|
|
89
|
+
again = false;
|
|
90
|
+
if (interactive)
|
|
91
|
+
process.stdout.write("\x1b[2J\x1b[3J\x1b[H");
|
|
92
|
+
try {
|
|
93
|
+
await once();
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
process.stderr.write(`pudding: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
97
|
+
}
|
|
98
|
+
process.stdout.write(interactive ? "\n Watching for changes. Press q to quit, enter to check again.\n" : "\n Watching for changes.\n");
|
|
99
|
+
} while (again);
|
|
100
|
+
running = false;
|
|
101
|
+
};
|
|
102
|
+
watchFiles(cwd, { recursive: true }, (_event, file) => {
|
|
103
|
+
if (!file || IGNORED.test(file) || !/\.(m?ts|cts|json)$/.test(file))
|
|
104
|
+
return;
|
|
105
|
+
clearTimeout(timer);
|
|
106
|
+
timer = setTimeout(() => void cycle(), 60);
|
|
107
|
+
});
|
|
108
|
+
if (process.stdin.isTTY) {
|
|
109
|
+
process.stdin.setRawMode(true);
|
|
110
|
+
process.stdin.resume();
|
|
111
|
+
process.stdin.on("data", data => {
|
|
112
|
+
const key = data.toString();
|
|
113
|
+
if (key === "q" || key === "\x03")
|
|
114
|
+
process.exit(0);
|
|
115
|
+
if (key === "\r" || key === "\n")
|
|
116
|
+
void cycle();
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
await cycle();
|
|
120
|
+
}
|
|
121
|
+
/** Offer to download Lean when it is missing and someone is there to answer. */
|
|
122
|
+
async function offerInstall() {
|
|
123
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY)
|
|
124
|
+
return false;
|
|
125
|
+
const prompt = createInterface({ input: process.stdin, output: process.stdout });
|
|
126
|
+
const answer = await prompt.question(`Lean ${LEAN_VERSION} is not installed. Download it now? [Y/n] `);
|
|
127
|
+
prompt.close();
|
|
128
|
+
if (answer.trim() !== "" && !/^y/i.test(answer.trim()))
|
|
129
|
+
return false;
|
|
130
|
+
await installLean(process.cwd(), { full: false, local: false, log: line => console.log(line) });
|
|
131
|
+
return true;
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
await main();
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (error instanceof LeanMissing && await offerInstall())
|
|
138
|
+
await main();
|
|
139
|
+
else {
|
|
140
|
+
process.stderr.write(`pudding: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
141
|
+
process.exitCode = 2;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { Binder, Expr } from "./ir.ts";
|
|
2
|
+
import type { ArgShape } from "./translate.ts";
|
|
3
|
+
export interface RuntimeMap {
|
|
4
|
+
/** For each source file, where each of its import specifiers resolves. */
|
|
5
|
+
resolutions: Map<string, Map<string, string>>;
|
|
6
|
+
/** For each source file, the text ranges of imported or exported names that are types only. */
|
|
7
|
+
elided: Map<string, [number, number][]>;
|
|
8
|
+
}
|
|
9
|
+
interface Instance {
|
|
10
|
+
binders: Binder[];
|
|
11
|
+
arg: ArgShape | undefined;
|
|
12
|
+
obligations: {
|
|
13
|
+
guard: Expr;
|
|
14
|
+
claim: Expr;
|
|
15
|
+
inexact: boolean;
|
|
16
|
+
}[];
|
|
17
|
+
}
|
|
18
|
+
export interface Job {
|
|
19
|
+
map: RuntimeMap;
|
|
20
|
+
/** The most inputs to try per law instance. */
|
|
21
|
+
budget: number;
|
|
22
|
+
files: {
|
|
23
|
+
file: string;
|
|
24
|
+
/** How many laws the translator saw; a different count at run time means the two cannot be paired up. */
|
|
25
|
+
registered: number;
|
|
26
|
+
/** `mentioned` holds the constants of a law that could not be translated, so sampling can still aim at them. */
|
|
27
|
+
laws: {
|
|
28
|
+
index: number;
|
|
29
|
+
unproved: boolean;
|
|
30
|
+
instances: Instance[];
|
|
31
|
+
mentioned: Expr[];
|
|
32
|
+
}[];
|
|
33
|
+
}[];
|
|
34
|
+
}
|
|
35
|
+
export interface LawOutcome {
|
|
36
|
+
index: number;
|
|
37
|
+
executed: number;
|
|
38
|
+
exhaustive: boolean;
|
|
39
|
+
failure?: {
|
|
40
|
+
kind: "counterexample" | "mismatch";
|
|
41
|
+
inputs: Record<string, string>;
|
|
42
|
+
actual: boolean;
|
|
43
|
+
expected: boolean;
|
|
44
|
+
};
|
|
45
|
+
/** An input for which the model itself says the law fails. If Lean accepted the law, something is wrong. */
|
|
46
|
+
modelFails?: Record<string, string>;
|
|
47
|
+
}
|
|
48
|
+
export interface FileOutcome {
|
|
49
|
+
file: string;
|
|
50
|
+
/** Why the file's laws could not be executed. */
|
|
51
|
+
error?: string;
|
|
52
|
+
/** True when the file itself throws while it loads, as opposed to Node being unable to run TypeScript like it. */
|
|
53
|
+
throws?: boolean;
|
|
54
|
+
/** Imports Node would have resolved to another file than the one TypeScript read, which is the one that was run. */
|
|
55
|
+
rerouted?: string[];
|
|
56
|
+
/** Set when the file registered a different number of laws than were read from its source. */
|
|
57
|
+
miscounted?: {
|
|
58
|
+
ran: number;
|
|
59
|
+
read: number;
|
|
60
|
+
};
|
|
61
|
+
laws: LawOutcome[];
|
|
62
|
+
}
|
|
63
|
+
export {};
|