driftproof 0.11.1 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -14
- package/bin/driftproof +194 -8
- package/config/models.json +3 -2
- package/config.js +2 -2
- package/lib/counts.js +104 -0
- package/lib/decision.js +40 -7
- package/lib/diff.js +12 -3
- package/lib/export.js +4 -1
- package/lib/importers-anthropic.js +451 -0
- package/lib/importers.js +50 -13
- package/lib/receipt.js +72 -3
- package/lib/regrade.js +266 -0
- package/lib/reuse.js +144 -9
- package/lib/run.js +39 -4
- package/lib/skill.js +26 -9
- package/lib/stale.js +194 -0
- package/lib/verdict.js +18 -0
- package/package.json +2 -2
- package/spec/RECEIPT.md +103 -13
- package/spec/receipt.schema.json +330 -15
- package/spec/receipt.v0.7.schema.json +1563 -0
- package/spec/receipt.v0.8.schema.json +1677 -0
- package/spec/stale.v1.schema.json +353 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://driftproofhq.com/spec/stale.v1.schema.json",
|
|
4
|
+
"title": "driftproof stale, output v1",
|
|
5
|
+
"description": "What `driftproof stale --json` prints (spec 053): for each receipt, what it recorded against what would run today, per axis, the decision per arm (rerun, then regrade, then unknown, then reuse), the status, any newer-model advisory and the command to run next; or the reason the receipt could not be read. An axis nobody records or gives is unknown, never current. Fields are only ever added, never renamed or removed; a consumer ignores fields it does not know. An axis effect of advisory is a patch release of the same harness (spec 053 A-053-1): it reruns and regrades nothing and counts in summary.advisories with any newer-model advisory. exit_code is the process exit: 0 every receipt current with no advisory, 1 any stale, 3 none stale but something unknown or advised, 2 any error; with strict, 3 becomes 1.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"additionalProperties": true,
|
|
8
|
+
"required": [
|
|
9
|
+
"schema",
|
|
10
|
+
"runner_version",
|
|
11
|
+
"strict",
|
|
12
|
+
"receipts",
|
|
13
|
+
"summary",
|
|
14
|
+
"exit_code"
|
|
15
|
+
],
|
|
16
|
+
"properties": {
|
|
17
|
+
"schema": {
|
|
18
|
+
"const": "driftproof.stale/1"
|
|
19
|
+
},
|
|
20
|
+
"runner_version": {
|
|
21
|
+
"type": "string"
|
|
22
|
+
},
|
|
23
|
+
"strict": {
|
|
24
|
+
"type": "boolean"
|
|
25
|
+
},
|
|
26
|
+
"receipts": {
|
|
27
|
+
"type": "array",
|
|
28
|
+
"items": {
|
|
29
|
+
"oneOf": [
|
|
30
|
+
{
|
|
31
|
+
"type": "object",
|
|
32
|
+
"additionalProperties": true,
|
|
33
|
+
"required": [
|
|
34
|
+
"path",
|
|
35
|
+
"receipt_hash",
|
|
36
|
+
"skill",
|
|
37
|
+
"model_id",
|
|
38
|
+
"date_utc",
|
|
39
|
+
"verification_level",
|
|
40
|
+
"status",
|
|
41
|
+
"arms",
|
|
42
|
+
"axes",
|
|
43
|
+
"advisories",
|
|
44
|
+
"runner_version",
|
|
45
|
+
"next"
|
|
46
|
+
],
|
|
47
|
+
"properties": {
|
|
48
|
+
"path": {
|
|
49
|
+
"type": "string"
|
|
50
|
+
},
|
|
51
|
+
"receipt_hash": {
|
|
52
|
+
"type": [
|
|
53
|
+
"string",
|
|
54
|
+
"null"
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
"skill": {
|
|
58
|
+
"type": [
|
|
59
|
+
"string",
|
|
60
|
+
"null"
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
"model_id": {
|
|
64
|
+
"type": [
|
|
65
|
+
"string",
|
|
66
|
+
"null"
|
|
67
|
+
]
|
|
68
|
+
},
|
|
69
|
+
"date_utc": {
|
|
70
|
+
"type": [
|
|
71
|
+
"string",
|
|
72
|
+
"null"
|
|
73
|
+
]
|
|
74
|
+
},
|
|
75
|
+
"verification_level": {
|
|
76
|
+
"type": [
|
|
77
|
+
"string",
|
|
78
|
+
"null"
|
|
79
|
+
]
|
|
80
|
+
},
|
|
81
|
+
"status": {
|
|
82
|
+
"enum": [
|
|
83
|
+
"current",
|
|
84
|
+
"stale",
|
|
85
|
+
"unknown"
|
|
86
|
+
]
|
|
87
|
+
},
|
|
88
|
+
"arms": {
|
|
89
|
+
"type": "object",
|
|
90
|
+
"additionalProperties": false,
|
|
91
|
+
"required": [
|
|
92
|
+
"with_skill",
|
|
93
|
+
"baseline"
|
|
94
|
+
],
|
|
95
|
+
"properties": {
|
|
96
|
+
"with_skill": {
|
|
97
|
+
"type": "object",
|
|
98
|
+
"additionalProperties": true,
|
|
99
|
+
"required": [
|
|
100
|
+
"decision",
|
|
101
|
+
"axes"
|
|
102
|
+
],
|
|
103
|
+
"properties": {
|
|
104
|
+
"decision": {
|
|
105
|
+
"enum": [
|
|
106
|
+
"rerun",
|
|
107
|
+
"regrade",
|
|
108
|
+
"unknown",
|
|
109
|
+
"reuse"
|
|
110
|
+
]
|
|
111
|
+
},
|
|
112
|
+
"axes": {
|
|
113
|
+
"type": "array",
|
|
114
|
+
"items": {
|
|
115
|
+
"type": "string"
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
"cases": {
|
|
119
|
+
"type": "array",
|
|
120
|
+
"items": {
|
|
121
|
+
"type": "string"
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
"baseline": {
|
|
127
|
+
"type": "object",
|
|
128
|
+
"additionalProperties": true,
|
|
129
|
+
"required": [
|
|
130
|
+
"decision",
|
|
131
|
+
"axes"
|
|
132
|
+
],
|
|
133
|
+
"properties": {
|
|
134
|
+
"decision": {
|
|
135
|
+
"enum": [
|
|
136
|
+
"rerun",
|
|
137
|
+
"regrade",
|
|
138
|
+
"unknown",
|
|
139
|
+
"reuse"
|
|
140
|
+
]
|
|
141
|
+
},
|
|
142
|
+
"axes": {
|
|
143
|
+
"type": "array",
|
|
144
|
+
"items": {
|
|
145
|
+
"type": "string"
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
"cases": {
|
|
149
|
+
"type": "array",
|
|
150
|
+
"items": {
|
|
151
|
+
"type": "string"
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
"axes": {
|
|
159
|
+
"type": "array",
|
|
160
|
+
"items": {
|
|
161
|
+
"type": "object",
|
|
162
|
+
"additionalProperties": true,
|
|
163
|
+
"required": [
|
|
164
|
+
"axis",
|
|
165
|
+
"recorded",
|
|
166
|
+
"current",
|
|
167
|
+
"effect",
|
|
168
|
+
"arms",
|
|
169
|
+
"reason"
|
|
170
|
+
],
|
|
171
|
+
"properties": {
|
|
172
|
+
"axis": {
|
|
173
|
+
"enum": [
|
|
174
|
+
"model",
|
|
175
|
+
"harness",
|
|
176
|
+
"skill",
|
|
177
|
+
"suite",
|
|
178
|
+
"judge_model",
|
|
179
|
+
"judge_template",
|
|
180
|
+
"rubric"
|
|
181
|
+
]
|
|
182
|
+
},
|
|
183
|
+
"recorded": {
|
|
184
|
+
"type": [
|
|
185
|
+
"string",
|
|
186
|
+
"null"
|
|
187
|
+
]
|
|
188
|
+
},
|
|
189
|
+
"current": {
|
|
190
|
+
"type": [
|
|
191
|
+
"string",
|
|
192
|
+
"null"
|
|
193
|
+
]
|
|
194
|
+
},
|
|
195
|
+
"effect": {
|
|
196
|
+
"enum": [
|
|
197
|
+
"current",
|
|
198
|
+
"rerun",
|
|
199
|
+
"regrade",
|
|
200
|
+
"unknown",
|
|
201
|
+
"advisory"
|
|
202
|
+
]
|
|
203
|
+
},
|
|
204
|
+
"arms": {
|
|
205
|
+
"type": "array",
|
|
206
|
+
"items": {
|
|
207
|
+
"enum": [
|
|
208
|
+
"with_skill",
|
|
209
|
+
"baseline"
|
|
210
|
+
]
|
|
211
|
+
},
|
|
212
|
+
"minItems": 1
|
|
213
|
+
},
|
|
214
|
+
"reason": {
|
|
215
|
+
"type": "string",
|
|
216
|
+
"minLength": 1
|
|
217
|
+
},
|
|
218
|
+
"cases": {
|
|
219
|
+
"type": "array",
|
|
220
|
+
"items": {
|
|
221
|
+
"type": "string"
|
|
222
|
+
}
|
|
223
|
+
},
|
|
224
|
+
"cases_added": {
|
|
225
|
+
"type": "array",
|
|
226
|
+
"items": {
|
|
227
|
+
"type": "string"
|
|
228
|
+
}
|
|
229
|
+
},
|
|
230
|
+
"cases_removed": {
|
|
231
|
+
"type": "array",
|
|
232
|
+
"items": {
|
|
233
|
+
"type": "string"
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
"minItems": 1
|
|
239
|
+
},
|
|
240
|
+
"advisories": {
|
|
241
|
+
"type": "array",
|
|
242
|
+
"items": {
|
|
243
|
+
"type": "object",
|
|
244
|
+
"additionalProperties": true,
|
|
245
|
+
"required": [
|
|
246
|
+
"model_id",
|
|
247
|
+
"released"
|
|
248
|
+
],
|
|
249
|
+
"properties": {
|
|
250
|
+
"model_id": {
|
|
251
|
+
"type": "string"
|
|
252
|
+
},
|
|
253
|
+
"released": {
|
|
254
|
+
"type": "string"
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
},
|
|
259
|
+
"runner_version": {
|
|
260
|
+
"type": "object",
|
|
261
|
+
"additionalProperties": true,
|
|
262
|
+
"required": [
|
|
263
|
+
"recorded",
|
|
264
|
+
"current"
|
|
265
|
+
],
|
|
266
|
+
"properties": {
|
|
267
|
+
"recorded": {
|
|
268
|
+
"type": [
|
|
269
|
+
"string",
|
|
270
|
+
"null"
|
|
271
|
+
]
|
|
272
|
+
},
|
|
273
|
+
"current": {
|
|
274
|
+
"type": "string"
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
"next": {
|
|
279
|
+
"type": [
|
|
280
|
+
"string",
|
|
281
|
+
"null"
|
|
282
|
+
]
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
"type": "object",
|
|
288
|
+
"additionalProperties": false,
|
|
289
|
+
"required": [
|
|
290
|
+
"path",
|
|
291
|
+
"error"
|
|
292
|
+
],
|
|
293
|
+
"properties": {
|
|
294
|
+
"path": {
|
|
295
|
+
"type": "string"
|
|
296
|
+
},
|
|
297
|
+
"error": {
|
|
298
|
+
"type": "string",
|
|
299
|
+
"minLength": 1
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
]
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
"summary": {
|
|
307
|
+
"type": "object",
|
|
308
|
+
"additionalProperties": true,
|
|
309
|
+
"required": [
|
|
310
|
+
"receipts",
|
|
311
|
+
"current",
|
|
312
|
+
"stale",
|
|
313
|
+
"unknown",
|
|
314
|
+
"errors",
|
|
315
|
+
"advisories"
|
|
316
|
+
],
|
|
317
|
+
"properties": {
|
|
318
|
+
"receipts": {
|
|
319
|
+
"type": "integer",
|
|
320
|
+
"minimum": 0
|
|
321
|
+
},
|
|
322
|
+
"current": {
|
|
323
|
+
"type": "integer",
|
|
324
|
+
"minimum": 0
|
|
325
|
+
},
|
|
326
|
+
"stale": {
|
|
327
|
+
"type": "integer",
|
|
328
|
+
"minimum": 0
|
|
329
|
+
},
|
|
330
|
+
"unknown": {
|
|
331
|
+
"type": "integer",
|
|
332
|
+
"minimum": 0
|
|
333
|
+
},
|
|
334
|
+
"errors": {
|
|
335
|
+
"type": "integer",
|
|
336
|
+
"minimum": 0
|
|
337
|
+
},
|
|
338
|
+
"advisories": {
|
|
339
|
+
"type": "integer",
|
|
340
|
+
"minimum": 0
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
},
|
|
344
|
+
"exit_code": {
|
|
345
|
+
"enum": [
|
|
346
|
+
0,
|
|
347
|
+
1,
|
|
348
|
+
2,
|
|
349
|
+
3
|
|
350
|
+
]
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|