@letthembuild/run-receipts 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +4 -0
- package/README.md +110 -2
- package/SPEC.md +199 -0
- package/examples/agreed-verified.json +26 -0
- package/examples/not-unanimous-failing.json +38 -0
- package/package.json +46 -4
- package/receipt.schema.json +114 -0
- package/verify.mjs +229 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "{}"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026 GPULogy AI LLC
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/NOTICE
ADDED
package/README.md
CHANGED
|
@@ -1,3 +1,111 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Run receipts
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A run receipt is a short, machine-readable account of how well an AI run checked its own work.
|
|
4
|
+
|
|
5
|
+
Several models agreeing on an answer is not evidence that the answer is right. What matters is what was actually run: which commands, by whom, how they exited, and what nobody verified. A receipt records exactly that, in one JSON file that a person can read and a tool can check.
|
|
6
|
+
|
|
7
|
+
This repository holds:
|
|
8
|
+
|
|
9
|
+
- **[SPEC.md](SPEC.md)**: the format, version 0.1 (draft), and the rules a valid receipt obeys.
|
|
10
|
+
- **[receipt.schema.json](receipt.schema.json)**: the JSON Schema.
|
|
11
|
+
- **[verify.mjs](verify.mjs)**: a verifier with no dependencies.
|
|
12
|
+
- **[examples/](examples)**: two receipts to read and test against.
|
|
13
|
+
|
|
14
|
+
The format is not tied to any one product. Any tool that runs one model to do work and others to review it can write a receipt.
|
|
15
|
+
|
|
16
|
+
## What a receipt looks like
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"receiptVersion": "0.1",
|
|
21
|
+
"producer": { "name": "example-producer" },
|
|
22
|
+
"endedAt": "2026-11-10T16:42:07Z",
|
|
23
|
+
"seats": [
|
|
24
|
+
{ "name": "Claude", "role": "lead", "model": "claude-sonnet-5-5", "...": "..." },
|
|
25
|
+
{ "name": "Gemini", "role": "reviewer", "model": "gemini-3-pro", "stance": "agreed", "...": "..." }
|
|
26
|
+
],
|
|
27
|
+
"verdict": { "outcome": "agreed", "agreed": 2, "reviewers": 2, "holdouts": [] },
|
|
28
|
+
"checks": [
|
|
29
|
+
{ "by": "Gemini", "command": "npm test", "exitCode": 0, "where": "local", "deployed": false, "state": "checked", "...": "..." }
|
|
30
|
+
],
|
|
31
|
+
"totals": { "reviewers": 2, "verifiers": 2, "ran": 5, "passed": 4, "failed": 1, "failingNow": 0, "...": "..." },
|
|
32
|
+
"grade": "verified",
|
|
33
|
+
"gaps": ["Behaviour under production load"]
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The full versions are in [examples/](examples). A receipt holds figures, names and commands. It never holds a prompt, a model's reply or a file's contents.
|
|
38
|
+
|
|
39
|
+
## Verify a receipt
|
|
40
|
+
|
|
41
|
+
You need Node 18 or later. Nothing else.
|
|
42
|
+
|
|
43
|
+
With npm, no clone is needed:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
npx @letthembuild/run-receipts receipt.json
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or from a clone of this repository:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
git clone <this repository> run-receipts
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
node run-receipts/verify.mjs run-receipts/examples/agreed-verified.json
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
VALID run-receipts/examples/agreed-verified.json
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
A receipt that does not hold together is rejected, with the rule it broke:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
INVALID edited.json
|
|
67
|
+
error G1: grade is failed; the totals earn verified
|
|
68
|
+
error D1: digest does not match the receipt's contents
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
| Command | What it does |
|
|
72
|
+
|---|---|
|
|
73
|
+
| `verify.mjs <file>` | Check a receipt. Exits 0 when valid, 1 when invalid, 2 when the file cannot be read. |
|
|
74
|
+
| `verify.mjs --json <file>` | The same, with the result as JSON. |
|
|
75
|
+
| `verify.mjs --digest <file>` | Print the digest the receipt should carry. |
|
|
76
|
+
|
|
77
|
+
The verifier is also a module:
|
|
78
|
+
|
|
79
|
+
```js
|
|
80
|
+
import { verifyReceipt } from "./verify.mjs";
|
|
81
|
+
|
|
82
|
+
const { valid, errors, warnings } = verifyReceipt(receipt);
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## What verifying proves, and what it does not
|
|
86
|
+
|
|
87
|
+
A valid receipt is well-formed, consistent with itself, and unchanged since its digest was taken. Its grade follows from its totals, its verdict follows from where each reviewer ended, and each check's state matches its exit code.
|
|
88
|
+
|
|
89
|
+
Verifying does **not** prove that the commands were really run or that they exited as stated. A receipt is the producer's own account. Trusting a receipt you did not produce needs a signature, which version 0.1 leaves out.
|
|
90
|
+
|
|
91
|
+
## Roadmap
|
|
92
|
+
|
|
93
|
+
Not in 0.1:
|
|
94
|
+
|
|
95
|
+
- **Signing**, so a reader can tell who produced a receipt.
|
|
96
|
+
- **A GitHub Action** that verifies a receipt on a pull request and posts its grade, checks and gaps as a comment.
|
|
97
|
+
- **A guide for producers**: how to write a receipt from your own tool.
|
|
98
|
+
|
|
99
|
+
## Run the tests
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
npm test
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Contributing
|
|
106
|
+
|
|
107
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Security reports go to the address in [SECURITY.md](SECURITY.md), not to public issues.
|
|
108
|
+
|
|
109
|
+
## Licence
|
|
110
|
+
|
|
111
|
+
Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). The licence covers the code and the specification; it does not grant use of any names or logos, as [TRADEMARKS.md](TRADEMARKS.md) explains.
|
package/SPEC.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Run receipt, version 0.1 (draft)
|
|
2
|
+
|
|
3
|
+
Status: draft. Field names and rules may change before 1.0.
|
|
4
|
+
|
|
5
|
+
## What a receipt is
|
|
6
|
+
|
|
7
|
+
Several models agreeing on an answer is not evidence that the answer is right. A run receipt is the short, machine-readable account of what a multi-model run actually did to check its own work: who took part, which commands were run and how they exited, where each reviewer ended up, and what nobody verified.
|
|
8
|
+
|
|
9
|
+
A receipt holds figures, names and commands. It never holds a prompt, a model's reply, a file's contents or a command's full output.
|
|
10
|
+
|
|
11
|
+
A receipt is the producer's account of the run. A verifier can prove that the account is well-formed, consistent with itself and unchanged since its digest was taken. It cannot prove the run happened as described. See "What a verifier cannot tell you".
|
|
12
|
+
|
|
13
|
+
## Vocabulary
|
|
14
|
+
|
|
15
|
+
- **Run**: one request worked on by one lead and zero or more reviewers until it ends.
|
|
16
|
+
- **Seat**: one participant in the run, with a display name and the model behind it.
|
|
17
|
+
- **Lead**: the seat that does the work. **Reviewer**: a seat that examines it.
|
|
18
|
+
- **Check**: a command a seat ran to test the work.
|
|
19
|
+
- **Stance**: where a reviewer ended the run. `agreed`, `objected`, `silent` (answered without saying either way) or `unavailable` (could not answer at all).
|
|
20
|
+
- **Gap**: something that was not verified, stated in words.
|
|
21
|
+
|
|
22
|
+
## The document
|
|
23
|
+
|
|
24
|
+
A receipt is one JSON object, UTF-8. `receipt.schema.json` is the normative shape; unknown fields are not allowed.
|
|
25
|
+
|
|
26
|
+
### Top level
|
|
27
|
+
|
|
28
|
+
| Field | Required | Meaning |
|
|
29
|
+
|---|---|---|
|
|
30
|
+
| `receiptVersion` | yes | `"0.1"`. |
|
|
31
|
+
| `id` | no | Opaque id chosen by the producer. |
|
|
32
|
+
| `producer` | yes | `name` and optional `version` of the tool that wrote the receipt. |
|
|
33
|
+
| `endedAt` | yes | When the run ended, UTC, `YYYY-MM-DDTHH:MM:SSZ`. |
|
|
34
|
+
| `revision` | no | The commit the work ended on, or null outside a repository. May say that uncommitted changes were present. |
|
|
35
|
+
| `durationMs` | yes | Wall-clock length of the run. |
|
|
36
|
+
| `firstReplyMs` | no | Time until the first seat began answering. |
|
|
37
|
+
| `tokens` | yes | Tokens used by all seats, in and out. |
|
|
38
|
+
| `costUsd` | yes | Metered cost of the run in US dollars. Seats on a flat subscription add nothing. |
|
|
39
|
+
| `seats` | yes | One entry per seat, 1 to 32. |
|
|
40
|
+
| `verdict` | yes | How the reviewers ended, or null when nobody reviewed. |
|
|
41
|
+
| `checks` | yes | The commands run to test the work. May be empty. |
|
|
42
|
+
| `totals` | yes | The counts the grade is worked out from. |
|
|
43
|
+
| `grade` | yes | One word for how well the result was checked. |
|
|
44
|
+
| `gaps` | yes | What was not verified. May be empty, and is listed even when everyone agreed. |
|
|
45
|
+
| `digest` | no | `sha256:` and 64 hex characters over the canonical form. |
|
|
46
|
+
|
|
47
|
+
### Seat
|
|
48
|
+
|
|
49
|
+
| Field | Required | Meaning |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `name` | yes | Display name, unique within the run. Checks and holdouts refer to it. |
|
|
52
|
+
| `role` | yes | `lead` or `reviewer`. |
|
|
53
|
+
| `model` | yes | The model identifier the seat ran. |
|
|
54
|
+
| `family` | no | The model's vendor family, lower case. |
|
|
55
|
+
| `turns` | yes | Answers the seat completed. |
|
|
56
|
+
| `errors` | yes | Attempts that failed. |
|
|
57
|
+
| `timeouts` | yes | How many of those errors were the seat going quiet. |
|
|
58
|
+
| `durationMs` | yes | Time the seat spent answering. Seats may run in parallel, so these do not add up to the run's length. |
|
|
59
|
+
| `costUsd` | yes | Metered cost of this seat. |
|
|
60
|
+
| `costBasis` | no | `metered`, `subscription` (a flat plan bought elsewhere) or `unpriced` (no published price). |
|
|
61
|
+
| `stance` | reviewers only | Where the reviewer ended. |
|
|
62
|
+
| `commandsVisible` | no | False when the seat worked inside a tool whose commands the producer could not observe. Such a seat is not counted as having run nothing; it is counted as not visible. |
|
|
63
|
+
|
|
64
|
+
### Verdict
|
|
65
|
+
|
|
66
|
+
| Field | Meaning |
|
|
67
|
+
|---|---|
|
|
68
|
+
| `outcome` | `agreed`, `majority` (more than half agree) or `none` (half or fewer; a tie is not a majority). |
|
|
69
|
+
| `agreed` | How many reviewers agreed. |
|
|
70
|
+
| `reviewers` | How many reviewers the outcome is counted over. |
|
|
71
|
+
| `holdouts` | Every reviewer that did not agree, by name, with its stance. |
|
|
72
|
+
|
|
73
|
+
A reviewer that could not answer is not a reviewer that disagreed. When every reviewer that answered agreed and no check is still failing, the outcome is `agreed`, counted over those that answered, and the missing seats stay named in `holdouts`.
|
|
74
|
+
|
|
75
|
+
### Check
|
|
76
|
+
|
|
77
|
+
| Field | Required | Meaning |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| `by` | yes | The seat that ran it, or null when unknown. |
|
|
80
|
+
| `alsoBy` | no | Up to four other seats that re-ran it with the same result. |
|
|
81
|
+
| `purpose` | no | What the seat said the command was for. |
|
|
82
|
+
| `command` | yes | The command as run, at most 300 characters. |
|
|
83
|
+
| `exitCode` | yes | Its exit code, or null when none was seen. |
|
|
84
|
+
| `result` | no | One line of its output, at most 200 characters. |
|
|
85
|
+
| `revision` | no | The commit it ran against. |
|
|
86
|
+
| `where` | yes | `local`, `ci`, `cloud` or `other`. A category, never a machine name. |
|
|
87
|
+
| `deployed` | yes | True when the command reached the place the work actually runs. |
|
|
88
|
+
| `missingSetup` | yes | What the command needed and did not have, in words, or null. |
|
|
89
|
+
| `state` | yes | `checked` (ran and answered), `reproduced` (a failure was shown), `blocked` (could not run for want of setup), `unknown` (ran, result not seen) or `untested` (named but never run). |
|
|
90
|
+
| `durationMs` | no | How long it took. |
|
|
91
|
+
|
|
92
|
+
`checks` is a readable list, not a ledger: identical re-runs are merged into one entry with `alsoBy`, and a producer may cap the list. `totals` holds the full counts.
|
|
93
|
+
|
|
94
|
+
### Totals
|
|
95
|
+
|
|
96
|
+
| Field | Required | Meaning |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| `reviewers` | yes | Reviewers that answered. |
|
|
99
|
+
| `verifiers` | yes | Reviewers that ran at least one command. |
|
|
100
|
+
| `notVisible` | no | Reviewers whose commands could not be observed. |
|
|
101
|
+
| `contradicted` | yes | Reviewers that said they agreed while their own checks failed. Their agreement does not count. |
|
|
102
|
+
| `ran` | yes | Commands run to test the work. |
|
|
103
|
+
| `passed` | yes | Of those, how many passed. |
|
|
104
|
+
| `failed` | yes | How many failed at any point in the run. |
|
|
105
|
+
| `failingNow` | yes | How many were still failing when the run ended. |
|
|
106
|
+
| `expected` | no | Failed as intended: a probe answering "not installed", or the failure the person asked to see. Neither passed nor failing. |
|
|
107
|
+
| `resolved` | no | Failed earlier and passing at the end: caught and fixed. |
|
|
108
|
+
| `filesChanged` | no | Files the lead changed. Absent means not measured; 0 means measured and none. |
|
|
109
|
+
|
|
110
|
+
### Grade
|
|
111
|
+
|
|
112
|
+
One of six words, decided by the first rule that applies:
|
|
113
|
+
|
|
114
|
+
1. `failed`: a check is still failing, or a reviewer contradicted its own checks.
|
|
115
|
+
2. `verified`: commands were run, and every reviewer ran some.
|
|
116
|
+
3. `partial`: commands were run, by some reviewers only.
|
|
117
|
+
4. `reviewed`: nothing was run and no file was changed. A question was answered and read.
|
|
118
|
+
5. `not-visible`: nothing was run where it could be seen, and every reviewer worked out of view.
|
|
119
|
+
6. `unverified`: none of the above.
|
|
120
|
+
|
|
121
|
+
The grade is a word and not a score on purpose. A failed check is a different answer from an unrun one, not fewer points.
|
|
122
|
+
|
|
123
|
+
## What a verifier must check
|
|
124
|
+
|
|
125
|
+
A receipt is valid when it satisfies the schema and every rule below. Rule ids are stable, so tools can report them.
|
|
126
|
+
|
|
127
|
+
**Seats**
|
|
128
|
+
- S1. Seat names are unique.
|
|
129
|
+
- S2. At least one seat is the lead.
|
|
130
|
+
- S3. `timeouts` does not exceed `errors`.
|
|
131
|
+
- S4. Every reviewer has a `stance`; the lead has none.
|
|
132
|
+
- S5. A seat whose `costBasis` is `subscription` or `unpriced` has `costUsd` 0.
|
|
133
|
+
|
|
134
|
+
**Verdict**
|
|
135
|
+
- V1. `verdict` is null exactly when the run has no reviewers.
|
|
136
|
+
- V2. `verdict.agreed` equals the number of reviewers whose stance is `agreed`.
|
|
137
|
+
- V3. `holdouts` is exactly the reviewers whose stance is not `agreed`, each with that stance.
|
|
138
|
+
- V4. `outcome` is `agreed` only if at least one reviewer answered and every reviewer that answered agreed; and, if any reviewer was `unavailable`, only if `totals.failingNow` is 0. Otherwise `outcome` is `majority` when more than half of all reviewers agreed, else `none`.
|
|
139
|
+
- V5. `verdict.reviewers` is the number of reviewers that answered when the outcome is `agreed`, and the number of all reviewers otherwise.
|
|
140
|
+
|
|
141
|
+
A producer may record `majority` or `none` where V4 would allow `agreed`: a run can withhold agreement for reasons of its own. It may never record `agreed` where V4 forbids it.
|
|
142
|
+
|
|
143
|
+
**Totals**
|
|
144
|
+
- T1. `reviewers` equals the number of reviewers whose stance is not `unavailable`.
|
|
145
|
+
- T2. `verifiers`, `notVisible` and `contradicted` do not exceed `reviewers`; `notVisible` does not exceed the reviewers marked `commandsVisible: false`.
|
|
146
|
+
- T3. `passed + failed + expected` does not exceed `ran`.
|
|
147
|
+
- T4. `failingNow` does not exceed `failed`; `resolved` does not exceed `failed - failingNow`.
|
|
148
|
+
- T5. `ran` is above 0 exactly when `verifiers` is above 0.
|
|
149
|
+
|
|
150
|
+
**Grade**
|
|
151
|
+
- G1. `grade` is the word the totals earn under the six rules above.
|
|
152
|
+
|
|
153
|
+
**Checks**
|
|
154
|
+
- C1. `by` and every name in `alsoBy` is a seat in the run.
|
|
155
|
+
- C2. A check that exited 0 has state `checked`.
|
|
156
|
+
- C3. State `blocked` and a non-null `missingSetup` go together.
|
|
157
|
+
- C4. States `unknown` and `untested` have a null `exitCode`.
|
|
158
|
+
- C5 (warning). More checks are listed as run than `totals.ran` counts.
|
|
159
|
+
|
|
160
|
+
**Money and time**
|
|
161
|
+
- M1 (warning). Seat costs differ from `costUsd` by more than one cent.
|
|
162
|
+
- M2. `firstReplyMs` does not exceed `durationMs`.
|
|
163
|
+
- M3. `endedAt` is a real date.
|
|
164
|
+
|
|
165
|
+
**Digest**
|
|
166
|
+
- D1. If `digest` is present it equals the digest of the receipt.
|
|
167
|
+
- D2 (warning). No digest is present.
|
|
168
|
+
|
|
169
|
+
Warnings do not make a receipt invalid.
|
|
170
|
+
|
|
171
|
+
## Digest
|
|
172
|
+
|
|
173
|
+
The digest is SHA-256 over the receipt's canonical form, written `sha256:` followed by lower-case hex. The canonical form is the receipt without its top-level `digest` field, serialised as JSON with object keys sorted by code unit at every level, no whitespace, and numbers and strings as `JSON.stringify` writes them, encoded as UTF-8.
|
|
174
|
+
|
|
175
|
+
A digest lets a copy of a receipt, such as one quoted in a pull-request comment, be matched against the file it came from. It is not a signature: anyone who edits a receipt can retake its digest.
|
|
176
|
+
|
|
177
|
+
## What a verifier cannot tell you
|
|
178
|
+
|
|
179
|
+
- That the commands were really run, or exited as stated.
|
|
180
|
+
- That the seats were the models named.
|
|
181
|
+
- That the receipt belongs to the change it is attached to, beyond the `revision` it names.
|
|
182
|
+
|
|
183
|
+
Closing those gaps needs a signature from the producer and a way to check the key, which this version leaves out. See the roadmap in README.md.
|
|
184
|
+
|
|
185
|
+
## What a receipt must not carry
|
|
186
|
+
|
|
187
|
+
A tool that runs models holds far more about a run than a receipt should. A producer must keep all of the following out of a receipt, and the schema rejects unknown fields so that none of it can be added by accident.
|
|
188
|
+
|
|
189
|
+
| Left out | Why |
|
|
190
|
+
|---|---|
|
|
191
|
+
| Prompts, replies, conversation titles, file contents | A receipt is figures, not the work. |
|
|
192
|
+
| Full command output | Output carries paths, tokens and data. One line (`result`) is the limit, and the producer must mask secrets in it and in `command`. |
|
|
193
|
+
| Account email, user id, team or organisation id | Personal and customer data. |
|
|
194
|
+
| The id of the conversation or job the run belonged to, or anything derived from it | Links a public file to a private record. `id` must be opaque. |
|
|
195
|
+
| Machine names and host names for where a check ran | `where` is a four-word category for this reason. |
|
|
196
|
+
| How a seat was paid for beyond `costBasis` | Billing detail. |
|
|
197
|
+
| Network destinations blocked or allowed, browser permissions, sandbox decisions | Describe the person's machine and browsing, not the result. |
|
|
198
|
+
| Team policy and configuration | Belongs to the team, not to the run. |
|
|
199
|
+
| Free-text narrative: why a seat was silent, what failed, notes for later | Free text can quote the work. `gaps` and `purpose` are the only prose, and both are short. |
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"receiptVersion": "0.1",
|
|
3
|
+
"id": "example-agreed-verified",
|
|
4
|
+
"producer": { "name": "example-producer", "version": "0.0.1" },
|
|
5
|
+
"endedAt": "2026-11-10T16:42:07Z",
|
|
6
|
+
"revision": "4f1c2ab90d3e",
|
|
7
|
+
"durationMs": 184300,
|
|
8
|
+
"firstReplyMs": 2100,
|
|
9
|
+
"tokens": 61240,
|
|
10
|
+
"costUsd": 0.4312,
|
|
11
|
+
"seats": [
|
|
12
|
+
{ "name": "Claude", "role": "lead", "model": "claude-sonnet-5-5", "family": "claude", "turns": 3, "errors": 0, "timeouts": 0, "durationMs": 96200, "costUsd": 0.2871, "costBasis": "metered" },
|
|
13
|
+
{ "name": "Gemini", "role": "reviewer", "model": "gemini-3-pro", "family": "gemini", "turns": 2, "errors": 0, "timeouts": 0, "durationMs": 41800, "costUsd": 0.0935, "costBasis": "metered", "stance": "agreed" },
|
|
14
|
+
{ "name": "ChatGPT", "role": "reviewer", "model": "gpt-5.2", "family": "openai", "turns": 2, "errors": 1, "timeouts": 1, "durationMs": 46300, "costUsd": 0.0506, "costBasis": "metered", "stance": "agreed" }
|
|
15
|
+
],
|
|
16
|
+
"verdict": { "outcome": "agreed", "agreed": 2, "reviewers": 2, "holdouts": [] },
|
|
17
|
+
"checks": [
|
|
18
|
+
{ "by": "Gemini", "alsoBy": ["ChatGPT"], "purpose": "Run the unit tests", "command": "npm test", "exitCode": 0, "result": "42 passing", "revision": "4f1c2ab90d3e", "where": "local", "deployed": false, "missingSetup": null, "state": "checked", "durationMs": 8400 },
|
|
19
|
+
{ "by": "ChatGPT", "purpose": "Check types", "command": "npm run typecheck", "exitCode": 0, "result": "", "revision": "4f1c2ab90d3e", "where": "local", "deployed": false, "missingSetup": null, "state": "checked", "durationMs": 5100 },
|
|
20
|
+
{ "by": "Gemini", "purpose": "Ask the staging health endpoint", "command": "curl -fsS https://staging.example.com/healthz", "exitCode": 0, "result": "ok", "revision": null, "where": "local", "deployed": true, "missingSetup": null, "state": "checked" }
|
|
21
|
+
],
|
|
22
|
+
"totals": { "reviewers": 2, "verifiers": 2, "notVisible": 0, "contradicted": 0, "ran": 5, "passed": 4, "failed": 1, "failingNow": 0, "resolved": 1, "filesChanged": 3 },
|
|
23
|
+
"grade": "verified",
|
|
24
|
+
"gaps": ["Behaviour under production load"],
|
|
25
|
+
"digest": "sha256:12b44c77459b21d7d99682b9a75a620d7ee5aae1ef0f581ce5e0e5df677334ab"
|
|
26
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"receiptVersion": "0.1",
|
|
3
|
+
"id": "example-not-unanimous-failing",
|
|
4
|
+
"producer": { "name": "example-producer", "version": "0.0.1" },
|
|
5
|
+
"endedAt": "2026-11-10T18:05:51Z",
|
|
6
|
+
"revision": "9a07de41c6b2 + uncommitted changes",
|
|
7
|
+
"durationMs": 402900,
|
|
8
|
+
"tokens": 138700,
|
|
9
|
+
"costUsd": 0.2204,
|
|
10
|
+
"seats": [
|
|
11
|
+
{ "name": "Codex", "role": "lead", "model": "gpt-5.2-codex", "family": "openai", "turns": 4, "errors": 0, "timeouts": 0, "durationMs": 211000, "costUsd": 0, "costBasis": "subscription" },
|
|
12
|
+
{ "name": "Claude", "role": "reviewer", "model": "claude-sonnet-5-5", "family": "claude", "turns": 3, "errors": 0, "timeouts": 0, "durationMs": 98400, "costUsd": 0.2204, "costBasis": "metered", "stance": "objected" },
|
|
13
|
+
{ "name": "Gemini", "role": "reviewer", "model": "gemini-3-pro", "family": "gemini", "turns": 3, "errors": 0, "timeouts": 0, "durationMs": 77100, "costUsd": 0, "costBasis": "subscription", "stance": "agreed", "commandsVisible": false },
|
|
14
|
+
{ "name": "Grok", "role": "reviewer", "model": "grok-5", "family": "xai", "turns": 0, "errors": 2, "timeouts": 2, "durationMs": 0, "costUsd": 0, "costBasis": "unpriced", "stance": "unavailable" }
|
|
15
|
+
],
|
|
16
|
+
"verdict": {
|
|
17
|
+
"outcome": "none",
|
|
18
|
+
"agreed": 1,
|
|
19
|
+
"reviewers": 3,
|
|
20
|
+
"holdouts": [
|
|
21
|
+
{ "name": "Claude", "stance": "objected" },
|
|
22
|
+
{ "name": "Grok", "stance": "unavailable" }
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
"checks": [
|
|
26
|
+
{ "by": "Claude", "purpose": "Run the migration tests", "command": "pytest tests/test_migrate.py -q", "exitCode": 1, "result": "1 failed, 11 passed", "revision": "9a07de41c6b2 + uncommitted changes", "where": "local", "deployed": false, "missingSetup": null, "state": "reproduced", "durationMs": 12900 },
|
|
27
|
+
{ "by": "Claude", "purpose": "Run the tests against Postgres", "command": "pytest tests/test_migrate.py -q --db postgres", "exitCode": 1, "result": "connection refused", "revision": "9a07de41c6b2 + uncommitted changes", "where": "local", "deployed": false, "missingSetup": "a service it needs was not reachable", "state": "blocked" },
|
|
28
|
+
{ "by": null, "command": "make lint", "exitCode": null, "result": "", "revision": null, "where": "ci", "deployed": false, "missingSetup": null, "state": "unknown" }
|
|
29
|
+
],
|
|
30
|
+
"totals": { "reviewers": 2, "verifiers": 1, "notVisible": 1, "contradicted": 0, "ran": 3, "passed": 0, "failed": 2, "failingNow": 2, "filesChanged": 6 },
|
|
31
|
+
"grade": "failed",
|
|
32
|
+
"gaps": [
|
|
33
|
+
"Checked locally only; deployed behaviour not verified",
|
|
34
|
+
"Behaviour on the hosted database",
|
|
35
|
+
"Gemini's commands could not be seen"
|
|
36
|
+
],
|
|
37
|
+
"digest": "sha256:1e471ce1e3b03fb7fef58f3d56bd08cee3684c499f002e3ffdd5b64334f5170f"
|
|
38
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,48 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@letthembuild/run-receipts",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "A format for recording how well an AI run checked its own work, and a verifier for it.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "GPULogy AI LLC",
|
|
7
|
+
"homepage": "https://github.com/gpulogyai/run-receipts#readme",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/gpulogyai/run-receipts.git"
|
|
11
|
+
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/gpulogyai/run-receipts/issues"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"main": "./verify.mjs",
|
|
17
|
+
"exports": {
|
|
18
|
+
".": "./verify.mjs",
|
|
19
|
+
"./schema": "./receipt.schema.json"
|
|
20
|
+
},
|
|
21
|
+
"bin": {
|
|
22
|
+
"run-receipt-verify": "./verify.mjs"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"verify.mjs",
|
|
26
|
+
"receipt.schema.json",
|
|
27
|
+
"SPEC.md",
|
|
28
|
+
"examples",
|
|
29
|
+
"NOTICE"
|
|
30
|
+
],
|
|
31
|
+
"engines": {
|
|
32
|
+
"node": ">=18"
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"test": "node --test verify.test.mjs"
|
|
36
|
+
},
|
|
37
|
+
"keywords": [
|
|
38
|
+
"receipt",
|
|
39
|
+
"verification",
|
|
40
|
+
"ai",
|
|
41
|
+
"agents",
|
|
42
|
+
"code-review",
|
|
43
|
+
"json-schema"
|
|
44
|
+
],
|
|
45
|
+
"publishConfig": {
|
|
46
|
+
"access": "public"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://letthembuild.com/schemas/run-receipt/0.1/receipt.schema.json",
|
|
4
|
+
"title": "Run receipt",
|
|
5
|
+
"description": "What one multi-model run did and how well its result was checked: who took part, what was run, what passed, and what nobody verified. Figures and commands only; never a prompt, a reply or a file's contents.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"additionalProperties": false,
|
|
8
|
+
"required": ["receiptVersion", "producer", "endedAt", "durationMs", "tokens", "costUsd", "seats", "verdict", "checks", "totals", "grade", "gaps"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"receiptVersion": { "const": "0.1" },
|
|
11
|
+
"id": { "type": "string", "minLength": 1, "maxLength": 200, "description": "An opaque id chosen by the producer. Carries no meaning to a verifier." },
|
|
12
|
+
"producer": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"additionalProperties": false,
|
|
15
|
+
"required": ["name"],
|
|
16
|
+
"properties": {
|
|
17
|
+
"name": { "type": "string", "minLength": 1, "maxLength": 60 },
|
|
18
|
+
"version": { "type": "string", "minLength": 1, "maxLength": 40 }
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"endedAt": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,3})?Z$", "description": "When the run ended, UTC, RFC 3339." },
|
|
22
|
+
"revision": { "type": ["string", "null"], "maxLength": 60, "description": "The commit the work ended on, if it was in a repository." },
|
|
23
|
+
"durationMs": { "type": "integer", "minimum": 0, "maximum": 86400000 },
|
|
24
|
+
"firstReplyMs": { "type": "integer", "minimum": 0, "maximum": 86400000 },
|
|
25
|
+
"tokens": { "type": "integer", "minimum": 0, "maximum": 2000000000 },
|
|
26
|
+
"costUsd": { "type": "number", "minimum": 0, "maximum": 1000000 },
|
|
27
|
+
"seats": { "type": "array", "minItems": 1, "maxItems": 32, "items": { "$ref": "#/$defs/seat" } },
|
|
28
|
+
"verdict": { "anyOf": [{ "type": "null" }, { "$ref": "#/$defs/verdict" }] },
|
|
29
|
+
"checks": { "type": "array", "maxItems": 200, "items": { "$ref": "#/$defs/check" } },
|
|
30
|
+
"totals": { "$ref": "#/$defs/totals" },
|
|
31
|
+
"grade": { "enum": ["failed", "verified", "partial", "not-visible", "reviewed", "unverified"] },
|
|
32
|
+
"gaps": { "type": "array", "maxItems": 50, "items": { "type": "string", "minLength": 1, "maxLength": 200 } },
|
|
33
|
+
"digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }
|
|
34
|
+
},
|
|
35
|
+
"$defs": {
|
|
36
|
+
"stance": { "enum": ["agreed", "objected", "silent", "unavailable"] },
|
|
37
|
+
"seat": {
|
|
38
|
+
"type": "object",
|
|
39
|
+
"additionalProperties": false,
|
|
40
|
+
"required": ["name", "role", "model", "turns", "errors", "timeouts", "durationMs", "costUsd"],
|
|
41
|
+
"properties": {
|
|
42
|
+
"name": { "type": "string", "minLength": 1, "maxLength": 60 },
|
|
43
|
+
"role": { "enum": ["lead", "reviewer"] },
|
|
44
|
+
"model": { "type": "string", "minLength": 1, "maxLength": 120 },
|
|
45
|
+
"family": { "type": "string", "minLength": 1, "maxLength": 40 },
|
|
46
|
+
"turns": { "type": "integer", "minimum": 0, "maximum": 1000 },
|
|
47
|
+
"errors": { "type": "integer", "minimum": 0, "maximum": 1000 },
|
|
48
|
+
"timeouts": { "type": "integer", "minimum": 0, "maximum": 1000 },
|
|
49
|
+
"durationMs": { "type": "integer", "minimum": 0, "maximum": 86400000 },
|
|
50
|
+
"costUsd": { "type": "number", "minimum": 0, "maximum": 1000000 },
|
|
51
|
+
"costBasis": { "enum": ["metered", "subscription", "unpriced"] },
|
|
52
|
+
"stance": { "$ref": "#/$defs/stance" },
|
|
53
|
+
"commandsVisible": { "type": "boolean", "description": "False when the seat worked inside a tool whose commands the producer could not see." }
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"holdout": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"additionalProperties": false,
|
|
59
|
+
"required": ["name", "stance"],
|
|
60
|
+
"properties": {
|
|
61
|
+
"name": { "type": "string", "minLength": 1, "maxLength": 60 },
|
|
62
|
+
"stance": { "enum": ["objected", "silent", "unavailable"] }
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
"verdict": {
|
|
66
|
+
"type": "object",
|
|
67
|
+
"additionalProperties": false,
|
|
68
|
+
"required": ["outcome", "agreed", "reviewers", "holdouts"],
|
|
69
|
+
"properties": {
|
|
70
|
+
"outcome": { "enum": ["agreed", "majority", "none"] },
|
|
71
|
+
"agreed": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
72
|
+
"reviewers": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
73
|
+
"holdouts": { "type": "array", "maxItems": 32, "items": { "$ref": "#/$defs/holdout" } }
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"check": {
|
|
77
|
+
"type": "object",
|
|
78
|
+
"additionalProperties": false,
|
|
79
|
+
"required": ["by", "command", "exitCode", "where", "deployed", "missingSetup", "state"],
|
|
80
|
+
"properties": {
|
|
81
|
+
"by": { "type": ["string", "null"], "maxLength": 60 },
|
|
82
|
+
"alsoBy": { "type": "array", "maxItems": 4, "items": { "type": "string", "minLength": 1, "maxLength": 60 } },
|
|
83
|
+
"purpose": { "type": "string", "minLength": 1, "maxLength": 100 },
|
|
84
|
+
"command": { "type": "string", "minLength": 1, "maxLength": 300 },
|
|
85
|
+
"exitCode": { "type": ["integer", "null"] },
|
|
86
|
+
"result": { "type": "string", "maxLength": 200, "description": "One line of the command's output. Never the whole output." },
|
|
87
|
+
"revision": { "type": ["string", "null"], "maxLength": 60 },
|
|
88
|
+
"where": { "enum": ["local", "ci", "cloud", "other"] },
|
|
89
|
+
"deployed": { "type": "boolean", "description": "True when the command reached the place the work actually runs." },
|
|
90
|
+
"missingSetup": { "type": ["string", "null"], "maxLength": 120 },
|
|
91
|
+
"state": { "enum": ["checked", "reproduced", "blocked", "unknown", "untested"] },
|
|
92
|
+
"durationMs": { "type": "integer", "minimum": 0, "maximum": 86400000 }
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
"totals": {
|
|
96
|
+
"type": "object",
|
|
97
|
+
"additionalProperties": false,
|
|
98
|
+
"required": ["reviewers", "verifiers", "ran", "passed", "failed", "failingNow", "contradicted"],
|
|
99
|
+
"properties": {
|
|
100
|
+
"reviewers": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
101
|
+
"verifiers": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
102
|
+
"notVisible": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
103
|
+
"contradicted": { "type": "integer", "minimum": 0, "maximum": 32 },
|
|
104
|
+
"ran": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
105
|
+
"passed": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
106
|
+
"failed": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
107
|
+
"failingNow": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
108
|
+
"expected": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
109
|
+
"resolved": { "type": "integer", "minimum": 0, "maximum": 100000 },
|
|
110
|
+
"filesChanged": { "type": "integer", "minimum": 0, "maximum": 100000 }
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
package/verify.mjs
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Run-receipt verifier, v0.1.
|
|
4
|
+
*
|
|
5
|
+
* Answers one question about a receipt file: does it hold together? First
|
|
6
|
+
* against the schema (shape, bounds), then against itself (the rules in
|
|
7
|
+
* SPEC.md: a grade that follows from the totals, a verdict that follows from
|
|
8
|
+
* where each reviewer ended, checks whose state matches their exit code).
|
|
9
|
+
*
|
|
10
|
+
* It does not prove the run happened. A receipt is the producer's account;
|
|
11
|
+
* this proves the account is well-formed, internally consistent and, when it
|
|
12
|
+
* carries a digest, unaltered since the digest was taken.
|
|
13
|
+
*
|
|
14
|
+
* No dependencies: Node 18 or later and this file, with the schema beside it.
|
|
15
|
+
*/
|
|
16
|
+
import { readFileSync, realpathSync } from "node:fs";
|
|
17
|
+
import { createHash } from "node:crypto";
|
|
18
|
+
import { fileURLToPath } from "node:url";
|
|
19
|
+
|
|
20
|
+
const SCHEMA_URL = new URL("./receipt.schema.json", import.meta.url);
|
|
21
|
+
let cachedSchema = null;
|
|
22
|
+
export function loadSchema() {
|
|
23
|
+
return (cachedSchema ??= JSON.parse(readFileSync(SCHEMA_URL, "utf8")));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/* ── Schema: the subset of JSON Schema that receipt.schema.json uses ───── */
|
|
27
|
+
|
|
28
|
+
const typeOf = (v) => (v === null ? "null" : Array.isArray(v) ? "array" : Number.isInteger(v) ? "integer" : typeof v);
|
|
29
|
+
const isType = (v, t) => typeOf(v) === t || (t === "number" && typeof v === "number");
|
|
30
|
+
|
|
31
|
+
/** Every way `value` fails `schema`, as "path: what is wrong". Empty when it passes. */
|
|
32
|
+
export function validateSchema(value, schema, root = schema, path = "$") {
|
|
33
|
+
if (schema.$ref) {
|
|
34
|
+
const target = schema.$ref.replace(/^#\//, "").split("/").reduce((o, k) => o?.[k], root);
|
|
35
|
+
if (!target) return [`${path}: schema reference ${schema.$ref} not found`];
|
|
36
|
+
return validateSchema(value, target, root, path);
|
|
37
|
+
}
|
|
38
|
+
if (schema.anyOf) {
|
|
39
|
+
return schema.anyOf.some((s) => validateSchema(value, s, root, path).length === 0) ? [] : [`${path}: matches none of the allowed forms`];
|
|
40
|
+
}
|
|
41
|
+
const out = [];
|
|
42
|
+
if ("const" in schema && value !== schema.const) return [`${path}: must be ${JSON.stringify(schema.const)}`];
|
|
43
|
+
if (schema.enum && !schema.enum.includes(value)) return [`${path}: must be one of ${schema.enum.join(", ")}`];
|
|
44
|
+
if (schema.type) {
|
|
45
|
+
const types = [].concat(schema.type);
|
|
46
|
+
if (!types.some((t) => isType(value, t))) return [`${path}: must be ${types.join(" or ")}, found ${typeOf(value)}`];
|
|
47
|
+
}
|
|
48
|
+
if (typeof value === "number") {
|
|
49
|
+
if (!Number.isFinite(value)) out.push(`${path}: must be a finite number`);
|
|
50
|
+
if (schema.minimum !== undefined && value < schema.minimum) out.push(`${path}: must be at least ${schema.minimum}`);
|
|
51
|
+
if (schema.maximum !== undefined && value > schema.maximum) out.push(`${path}: must be at most ${schema.maximum}`);
|
|
52
|
+
}
|
|
53
|
+
if (typeof value === "string") {
|
|
54
|
+
if (schema.minLength !== undefined && value.length < schema.minLength) out.push(`${path}: must be at least ${schema.minLength} characters`);
|
|
55
|
+
if (schema.maxLength !== undefined && value.length > schema.maxLength) out.push(`${path}: must be at most ${schema.maxLength} characters`);
|
|
56
|
+
if (schema.pattern && !new RegExp(schema.pattern).test(value)) out.push(`${path}: does not have the required form`);
|
|
57
|
+
}
|
|
58
|
+
if (Array.isArray(value)) {
|
|
59
|
+
if (schema.minItems !== undefined && value.length < schema.minItems) out.push(`${path}: needs at least ${schema.minItems} item(s)`);
|
|
60
|
+
if (schema.maxItems !== undefined && value.length > schema.maxItems) out.push(`${path}: allows at most ${schema.maxItems} items`);
|
|
61
|
+
if (schema.items) value.forEach((v, i) => out.push(...validateSchema(v, schema.items, root, `${path}[${i}]`)));
|
|
62
|
+
}
|
|
63
|
+
if (typeOf(value) === "object") {
|
|
64
|
+
const props = schema.properties ?? {};
|
|
65
|
+
for (const key of schema.required ?? []) if (!(key in value)) out.push(`${path}.${key}: is required`);
|
|
66
|
+
for (const [key, v] of Object.entries(value)) {
|
|
67
|
+
if (key in props) out.push(...validateSchema(v, props[key], root, `${path}.${key}`));
|
|
68
|
+
else if (schema.additionalProperties === false) out.push(`${path}.${key}: is not part of the format`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/* ── Digest ──────────────────────────────────────────────────────────── */
|
|
75
|
+
|
|
76
|
+
/** The receipt as bytes to hash: keys sorted, no whitespace, `digest` left out. */
|
|
77
|
+
export function canonical(value) {
|
|
78
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
79
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
|
80
|
+
return `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(",")}}`;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function digestOf(receipt) {
|
|
84
|
+
const { digest: _left_out, ...rest } = receipt;
|
|
85
|
+
return `sha256:${createHash("sha256").update(canonical(rest), "utf8").digest("hex")}`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/* ── Consistency: the rules of SPEC.md, by number ────────────────────── */
|
|
89
|
+
|
|
90
|
+
/** The grade the totals earn (SPEC rule G). */
|
|
91
|
+
export function gradeOf(t) {
|
|
92
|
+
if (t.failingNow > 0 || t.contradicted > 0) return "failed";
|
|
93
|
+
if (t.ran > 0) return t.verifiers >= t.reviewers ? "verified" : "partial";
|
|
94
|
+
if (t.filesChanged === 0) return "reviewed";
|
|
95
|
+
if (t.reviewers > 0 && (t.notVisible ?? 0) >= t.reviewers) return "not-visible";
|
|
96
|
+
return "unverified";
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Errors and warnings from a receipt that already passed the schema. */
|
|
100
|
+
export function checkConsistency(r) {
|
|
101
|
+
const errors = [], warnings = [];
|
|
102
|
+
const err = (rule, msg) => errors.push(`${rule}: ${msg}`);
|
|
103
|
+
const t = r.totals;
|
|
104
|
+
|
|
105
|
+
// S — seats.
|
|
106
|
+
const names = r.seats.map((s) => s.name);
|
|
107
|
+
if (new Set(names).size !== names.length) err("S1", "two seats share a name");
|
|
108
|
+
if (!r.seats.some((s) => s.role === "lead")) err("S2", "no seat is the lead");
|
|
109
|
+
const reviewers = r.seats.filter((s) => s.role === "reviewer");
|
|
110
|
+
for (const s of r.seats) {
|
|
111
|
+
if (s.timeouts > s.errors) err("S3", `${s.name}: more timeouts than errors`);
|
|
112
|
+
if (s.role === "reviewer" && !s.stance) err("S4", `${s.name}: a reviewer needs a stance`);
|
|
113
|
+
if (s.role === "lead" && s.stance) err("S4", `${s.name}: the lead has no stance`);
|
|
114
|
+
if (s.costBasis && s.costBasis !== "metered" && s.costUsd > 0) err("S5", `${s.name}: a ${s.costBasis} seat cannot carry a metered cost`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// V — verdict.
|
|
118
|
+
const agreed = reviewers.filter((s) => s.stance === "agreed");
|
|
119
|
+
const answered = reviewers.filter((s) => s.stance !== "unavailable");
|
|
120
|
+
const out = reviewers.filter((s) => s.stance && s.stance !== "agreed");
|
|
121
|
+
if (reviewers.length === 0) {
|
|
122
|
+
if (r.verdict !== null) err("V1", "nobody reviewed, so there can be no verdict");
|
|
123
|
+
} else if (r.verdict === null) {
|
|
124
|
+
err("V1", "reviewers took part, so a verdict is required");
|
|
125
|
+
} else {
|
|
126
|
+
const v = r.verdict;
|
|
127
|
+
if (v.agreed !== agreed.length) err("V2", `verdict counts ${v.agreed} agreeing; the seats show ${agreed.length}`);
|
|
128
|
+
const key = (h) => `${h.name}\u0000${h.stance}`;
|
|
129
|
+
const want = out.map(key).sort().join("\n"), got = v.holdouts.map(key).sort().join("\n");
|
|
130
|
+
if (want !== got) err("V3", "holdouts are not exactly the reviewers that did not agree");
|
|
131
|
+
if (v.outcome === "agreed") {
|
|
132
|
+
const allWhoAnsweredAgreed = answered.length > 0 && answered.every((s) => s.stance === "agreed");
|
|
133
|
+
const someoneMissing = answered.length < reviewers.length;
|
|
134
|
+
if (!allWhoAnsweredAgreed) err("V4", "outcome is agreed while a reviewer that answered did not agree");
|
|
135
|
+
else if (someoneMissing && t.failingNow > 0) err("V4", "outcome is agreed without a reviewer while a check is still failing");
|
|
136
|
+
if (v.reviewers !== answered.length) err("V5", `an agreed verdict counts the reviewers that answered (${answered.length}), not ${v.reviewers}`);
|
|
137
|
+
} else {
|
|
138
|
+
const expected = agreed.length * 2 > reviewers.length ? "majority" : "none";
|
|
139
|
+
if (v.outcome !== expected) err("V4", `outcome should be ${expected}: ${agreed.length} of ${reviewers.length} agree`);
|
|
140
|
+
if (v.reviewers !== reviewers.length) err("V5", `verdict counts ${v.reviewers} reviewers; the seats show ${reviewers.length}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// T — totals.
|
|
145
|
+
if (t.reviewers !== answered.length) err("T1", `totals count ${t.reviewers} reviewers; ${answered.length} answered`);
|
|
146
|
+
if (t.verifiers > t.reviewers) err("T2", "more verifiers than reviewers");
|
|
147
|
+
if ((t.notVisible ?? 0) > t.reviewers) err("T2", "more not-visible seats than reviewers");
|
|
148
|
+
if (t.contradicted > t.reviewers) err("T2", "more contradicted seats than reviewers");
|
|
149
|
+
const hidden = reviewers.filter((s) => s.commandsVisible === false).length;
|
|
150
|
+
if ((t.notVisible ?? 0) > hidden) err("T2", `totals count ${t.notVisible} not-visible seats; ${hidden} reviewer(s) are marked so`);
|
|
151
|
+
if (t.passed + t.failed + (t.expected ?? 0) > t.ran) err("T3", "passed, failed and expected add up to more than ran");
|
|
152
|
+
if (t.failingNow > t.failed) err("T4", "more checks failing now than ever failed");
|
|
153
|
+
if ((t.resolved ?? 0) > t.failed - t.failingNow) err("T4", "more checks resolved than stopped failing");
|
|
154
|
+
if (t.ran > 0 && t.verifiers === 0) err("T5", "commands were run, but no reviewer is counted as running any");
|
|
155
|
+
if (t.ran === 0 && t.verifiers > 0) err("T5", "a reviewer is counted as running commands, but none were run");
|
|
156
|
+
|
|
157
|
+
// G — grade.
|
|
158
|
+
const grade = gradeOf(t);
|
|
159
|
+
if (r.grade !== grade) err("G1", `grade is ${r.grade}; the totals earn ${grade}`);
|
|
160
|
+
|
|
161
|
+
// C — checks.
|
|
162
|
+
r.checks.forEach((c, i) => {
|
|
163
|
+
const at = `checks[${i}]`;
|
|
164
|
+
for (const who of [c.by, ...(c.alsoBy ?? [])]) if (who !== null && !names.includes(who)) err("C1", `${at}: ${who} is not a seat in this run`);
|
|
165
|
+
if (c.exitCode === 0 && c.state !== "checked") err("C2", `${at}: exited 0, so its state is checked, not ${c.state}`);
|
|
166
|
+
if ((c.state === "blocked") !== (c.missingSetup !== null)) err("C3", `${at}: blocked and missing setup go together`);
|
|
167
|
+
if ((c.state === "unknown" || c.state === "untested") && c.exitCode !== null) err("C4", `${at}: a state of ${c.state} has no exit code`);
|
|
168
|
+
});
|
|
169
|
+
const ranHere = r.checks.filter((c) => c.state !== "untested").length;
|
|
170
|
+
if (ranHere > t.ran) warnings.push(`C5: ${ranHere} checks are listed as run, but the totals count ${t.ran}`);
|
|
171
|
+
|
|
172
|
+
// M — money and time.
|
|
173
|
+
const seatCost = r.seats.reduce((n, s) => n + s.costUsd, 0);
|
|
174
|
+
if (Math.abs(seatCost - r.costUsd) > 0.01) warnings.push(`M1: seats add up to $${seatCost.toFixed(4)}; the run says $${r.costUsd.toFixed(4)}`);
|
|
175
|
+
if (r.firstReplyMs !== undefined && r.firstReplyMs > r.durationMs) err("M2", "the first reply came after the run ended");
|
|
176
|
+
if (Number.isNaN(Date.parse(r.endedAt))) err("M3", "endedAt is not a real date");
|
|
177
|
+
|
|
178
|
+
// D — digest.
|
|
179
|
+
if (r.digest && r.digest !== digestOf(r)) err("D1", "digest does not match the receipt's contents");
|
|
180
|
+
if (!r.digest) warnings.push("D2: no digest, so a copy cannot be told from an edited one");
|
|
181
|
+
|
|
182
|
+
return { errors, warnings };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Schema first; the consistency rules only run on a receipt with the right shape. */
|
|
186
|
+
export function verifyReceipt(receipt) {
|
|
187
|
+
const schemaErrors = validateSchema(receipt, loadSchema());
|
|
188
|
+
if (schemaErrors.length) return { valid: false, errors: schemaErrors.map((e) => `schema: ${e}`), warnings: [] };
|
|
189
|
+
const { errors, warnings } = checkConsistency(receipt);
|
|
190
|
+
return { valid: errors.length === 0, errors, warnings };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/* ── Command line ────────────────────────────────────────────────────── */
|
|
194
|
+
|
|
195
|
+
const USAGE = `Usage:
|
|
196
|
+
verify.mjs <receipt.json> check a receipt; exit 0 valid, 1 invalid
|
|
197
|
+
verify.mjs --json <receipt.json> the same, as JSON
|
|
198
|
+
verify.mjs --digest <receipt.json> print the digest the receipt should carry`;
|
|
199
|
+
|
|
200
|
+
function main(argv) {
|
|
201
|
+
const flags = argv.filter((a) => a.startsWith("--"));
|
|
202
|
+
const files = argv.filter((a) => !a.startsWith("--"));
|
|
203
|
+
if (files.length !== 1 || flags.some((f) => !["--json", "--digest"].includes(f))) { console.error(USAGE); return 2; }
|
|
204
|
+
let receipt;
|
|
205
|
+
try { receipt = JSON.parse(readFileSync(files[0], "utf8")); }
|
|
206
|
+
catch (e) { console.error(`Cannot read ${files[0]}: ${e.message}`); return 2; }
|
|
207
|
+
if (flags.includes("--digest")) {
|
|
208
|
+
if (typeOf(receipt) !== "object") { console.error("Not a receipt."); return 2; }
|
|
209
|
+
console.log(digestOf(receipt));
|
|
210
|
+
return 0;
|
|
211
|
+
}
|
|
212
|
+
const result = verifyReceipt(receipt);
|
|
213
|
+
if (flags.includes("--json")) console.log(JSON.stringify(result, null, 2));
|
|
214
|
+
else {
|
|
215
|
+
console.log(result.valid ? `VALID ${files[0]}` : `INVALID ${files[0]}`);
|
|
216
|
+
for (const e of result.errors) console.log(` error ${e}`);
|
|
217
|
+
for (const w of result.warnings) console.log(` warning ${w}`);
|
|
218
|
+
}
|
|
219
|
+
return result.valid ? 0 : 1;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Run as a program, and not imported: true through a symlink too, which is how an installed command arrives. */
|
|
223
|
+
function isMain() {
|
|
224
|
+
if (!process.argv[1]) return false;
|
|
225
|
+
try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
|
|
226
|
+
catch { return false; }
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (isMain()) process.exit(main(process.argv.slice(2)));
|