@openpond/evals 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,562 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "oneOf": [
4
+ {
5
+ "type": "object",
6
+ "properties": {
7
+ "id": {
8
+ "$ref": "#/$defs/__schema0"
9
+ },
10
+ "targetId": {
11
+ "$ref": "#/$defs/__schema0"
12
+ },
13
+ "baseRelease": {
14
+ "$ref": "#/$defs/__schema1"
15
+ },
16
+ "editorVersion": {
17
+ "$ref": "#/$defs/__schema4"
18
+ },
19
+ "targetKind": {
20
+ "type": "string",
21
+ "const": "definition"
22
+ },
23
+ "fields": {
24
+ "type": "object",
25
+ "properties": {
26
+ "name": {
27
+ "type": "string",
28
+ "maxLength": 500
29
+ },
30
+ "description": {
31
+ "type": "string",
32
+ "maxLength": 10000
33
+ },
34
+ "instructions": {
35
+ "type": "string",
36
+ "maxLength": 20000
37
+ },
38
+ "input": {
39
+ "type": "string"
40
+ },
41
+ "output": {
42
+ "type": "string"
43
+ },
44
+ "familyNamespace": {
45
+ "type": "string"
46
+ },
47
+ "sources": {
48
+ "maxItems": 100,
49
+ "type": "array",
50
+ "items": {
51
+ "$ref": "#/$defs/__schema5"
52
+ }
53
+ },
54
+ "recipeRef": {
55
+ "$ref": "#/$defs/__schema2"
56
+ }
57
+ },
58
+ "required": [
59
+ "name",
60
+ "description",
61
+ "instructions",
62
+ "input",
63
+ "output",
64
+ "familyNamespace",
65
+ "sources"
66
+ ],
67
+ "additionalProperties": false
68
+ },
69
+ "schemaVersion": {
70
+ "$ref": "#/$defs/__schema7"
71
+ },
72
+ "revision": {
73
+ "$ref": "#/$defs/__schema8"
74
+ },
75
+ "status": {
76
+ "$ref": "#/$defs/__schema9"
77
+ },
78
+ "publishedRelease": {
79
+ "$ref": "#/$defs/__schema10"
80
+ },
81
+ "createdAt": {
82
+ "$ref": "#/$defs/__schema11"
83
+ },
84
+ "updatedAt": {
85
+ "$ref": "#/$defs/__schema11"
86
+ },
87
+ "contentHash": {
88
+ "$ref": "#/$defs/__schema3"
89
+ }
90
+ },
91
+ "required": [
92
+ "id",
93
+ "targetId",
94
+ "baseRelease",
95
+ "editorVersion",
96
+ "targetKind",
97
+ "fields",
98
+ "schemaVersion",
99
+ "revision",
100
+ "status",
101
+ "publishedRelease",
102
+ "createdAt",
103
+ "updatedAt",
104
+ "contentHash"
105
+ ],
106
+ "additionalProperties": false
107
+ },
108
+ {
109
+ "type": "object",
110
+ "properties": {
111
+ "id": {
112
+ "$ref": "#/$defs/__schema0"
113
+ },
114
+ "targetId": {
115
+ "$ref": "#/$defs/__schema0"
116
+ },
117
+ "baseRelease": {
118
+ "$ref": "#/$defs/__schema1"
119
+ },
120
+ "editorVersion": {
121
+ "$ref": "#/$defs/__schema4"
122
+ },
123
+ "targetKind": {
124
+ "type": "string",
125
+ "const": "reward"
126
+ },
127
+ "fields": {
128
+ "type": "object",
129
+ "properties": {
130
+ "name": {
131
+ "type": "string",
132
+ "maxLength": 500
133
+ },
134
+ "description": {
135
+ "type": "string",
136
+ "maxLength": 10000
137
+ },
138
+ "kind": {
139
+ "type": "string",
140
+ "enum": [
141
+ "custom_verifier",
142
+ "state",
143
+ "content",
144
+ "schema",
145
+ "artifact",
146
+ "runtime_event",
147
+ "model_judge",
148
+ "learned_model",
149
+ "human"
150
+ ]
151
+ },
152
+ "fields": {
153
+ "type": "string"
154
+ },
155
+ "outputField": {
156
+ "type": "string"
157
+ },
158
+ "expectedField": {
159
+ "type": "string"
160
+ },
161
+ "expectedValue": {
162
+ "type": "string"
163
+ },
164
+ "schema": {
165
+ "type": "string"
166
+ },
167
+ "reference": {
168
+ "type": "string"
169
+ },
170
+ "events": {
171
+ "type": "string"
172
+ },
173
+ "code": {
174
+ "type": "string"
175
+ },
176
+ "exportName": {
177
+ "type": "string"
178
+ },
179
+ "timeout": {
180
+ "type": "string"
181
+ },
182
+ "rubric": {
183
+ "type": "string"
184
+ },
185
+ "providerId": {
186
+ "type": "string"
187
+ },
188
+ "modelId": {
189
+ "type": "string"
190
+ },
191
+ "modelRevision": {
192
+ "type": "string"
193
+ },
194
+ "temperature": {
195
+ "type": "string"
196
+ },
197
+ "reviewerRole": {
198
+ "type": "string"
199
+ },
200
+ "learnedId": {
201
+ "type": "string"
202
+ },
203
+ "learnedHash": {
204
+ "type": "string"
205
+ },
206
+ "inputContract": {
207
+ "type": "string"
208
+ },
209
+ "minimum": {
210
+ "type": "string"
211
+ },
212
+ "maximum": {
213
+ "type": "string"
214
+ }
215
+ },
216
+ "required": [
217
+ "name",
218
+ "description",
219
+ "kind",
220
+ "fields",
221
+ "outputField",
222
+ "expectedField",
223
+ "expectedValue",
224
+ "schema",
225
+ "reference",
226
+ "events",
227
+ "code",
228
+ "exportName",
229
+ "timeout",
230
+ "rubric",
231
+ "providerId",
232
+ "modelId",
233
+ "modelRevision",
234
+ "temperature",
235
+ "reviewerRole",
236
+ "learnedId",
237
+ "learnedHash",
238
+ "inputContract",
239
+ "minimum",
240
+ "maximum"
241
+ ],
242
+ "additionalProperties": false
243
+ },
244
+ "schemaVersion": {
245
+ "$ref": "#/$defs/__schema7"
246
+ },
247
+ "revision": {
248
+ "$ref": "#/$defs/__schema8"
249
+ },
250
+ "status": {
251
+ "$ref": "#/$defs/__schema9"
252
+ },
253
+ "publishedRelease": {
254
+ "$ref": "#/$defs/__schema10"
255
+ },
256
+ "createdAt": {
257
+ "$ref": "#/$defs/__schema11"
258
+ },
259
+ "updatedAt": {
260
+ "$ref": "#/$defs/__schema11"
261
+ },
262
+ "contentHash": {
263
+ "$ref": "#/$defs/__schema3"
264
+ }
265
+ },
266
+ "required": [
267
+ "id",
268
+ "targetId",
269
+ "baseRelease",
270
+ "editorVersion",
271
+ "targetKind",
272
+ "fields",
273
+ "schemaVersion",
274
+ "revision",
275
+ "status",
276
+ "publishedRelease",
277
+ "createdAt",
278
+ "updatedAt",
279
+ "contentHash"
280
+ ],
281
+ "additionalProperties": false
282
+ },
283
+ {
284
+ "type": "object",
285
+ "properties": {
286
+ "id": {
287
+ "$ref": "#/$defs/__schema0"
288
+ },
289
+ "targetId": {
290
+ "$ref": "#/$defs/__schema0"
291
+ },
292
+ "baseRelease": {
293
+ "$ref": "#/$defs/__schema1"
294
+ },
295
+ "editorVersion": {
296
+ "$ref": "#/$defs/__schema4"
297
+ },
298
+ "targetKind": {
299
+ "type": "string",
300
+ "const": "binding"
301
+ },
302
+ "fields": {
303
+ "type": "object",
304
+ "properties": {
305
+ "name": {
306
+ "type": "string",
307
+ "maxLength": 500
308
+ },
309
+ "description": {
310
+ "type": "string",
311
+ "maxLength": 10000
312
+ },
313
+ "sources": {
314
+ "maxItems": 100,
315
+ "type": "array",
316
+ "items": {
317
+ "$ref": "#/$defs/__schema5"
318
+ }
319
+ }
320
+ },
321
+ "required": [
322
+ "name",
323
+ "description",
324
+ "sources"
325
+ ],
326
+ "additionalProperties": false
327
+ },
328
+ "schemaVersion": {
329
+ "$ref": "#/$defs/__schema7"
330
+ },
331
+ "revision": {
332
+ "$ref": "#/$defs/__schema8"
333
+ },
334
+ "status": {
335
+ "$ref": "#/$defs/__schema9"
336
+ },
337
+ "publishedRelease": {
338
+ "$ref": "#/$defs/__schema10"
339
+ },
340
+ "createdAt": {
341
+ "$ref": "#/$defs/__schema11"
342
+ },
343
+ "updatedAt": {
344
+ "$ref": "#/$defs/__schema11"
345
+ },
346
+ "contentHash": {
347
+ "$ref": "#/$defs/__schema3"
348
+ }
349
+ },
350
+ "required": [
351
+ "id",
352
+ "targetId",
353
+ "baseRelease",
354
+ "editorVersion",
355
+ "targetKind",
356
+ "fields",
357
+ "schemaVersion",
358
+ "revision",
359
+ "status",
360
+ "publishedRelease",
361
+ "createdAt",
362
+ "updatedAt",
363
+ "contentHash"
364
+ ],
365
+ "additionalProperties": false
366
+ }
367
+ ],
368
+ "$defs": {
369
+ "__schema0": {
370
+ "type": "string",
371
+ "minLength": 1,
372
+ "maxLength": 240
373
+ },
374
+ "__schema1": {
375
+ "anyOf": [
376
+ {
377
+ "$ref": "#/$defs/__schema2"
378
+ },
379
+ {
380
+ "type": "null"
381
+ }
382
+ ]
383
+ },
384
+ "__schema2": {
385
+ "type": "object",
386
+ "properties": {
387
+ "id": {
388
+ "$ref": "#/$defs/__schema0"
389
+ },
390
+ "contentHash": {
391
+ "$ref": "#/$defs/__schema3"
392
+ },
393
+ "revision": {
394
+ "type": "integer",
395
+ "exclusiveMinimum": 0,
396
+ "maximum": 9007199254740991
397
+ }
398
+ },
399
+ "required": [
400
+ "id",
401
+ "contentHash",
402
+ "revision"
403
+ ],
404
+ "additionalProperties": false
405
+ },
406
+ "__schema3": {
407
+ "type": "string",
408
+ "pattern": "^[a-f0-9]{64}$"
409
+ },
410
+ "__schema4": {
411
+ "type": "string",
412
+ "const": "openpond.modelsEditor.v1"
413
+ },
414
+ "__schema5": {
415
+ "type": "object",
416
+ "properties": {
417
+ "graderId": {
418
+ "$ref": "#/$defs/__schema0"
419
+ },
420
+ "reward": {
421
+ "$ref": "#/$defs/__schema2"
422
+ },
423
+ "role": {
424
+ "type": "string",
425
+ "enum": [
426
+ "training",
427
+ "evaluation"
428
+ ]
429
+ },
430
+ "normalization": {
431
+ "oneOf": [
432
+ {
433
+ "type": "object",
434
+ "properties": {
435
+ "kind": {
436
+ "type": "string",
437
+ "const": "identity"
438
+ }
439
+ },
440
+ "required": [
441
+ "kind"
442
+ ],
443
+ "additionalProperties": false
444
+ },
445
+ {
446
+ "type": "object",
447
+ "properties": {
448
+ "kind": {
449
+ "type": "string",
450
+ "const": "linear"
451
+ },
452
+ "minimum": {
453
+ "type": "number"
454
+ },
455
+ "maximum": {
456
+ "type": "number"
457
+ },
458
+ "direction": {
459
+ "type": "string",
460
+ "enum": [
461
+ "higher",
462
+ "lower"
463
+ ]
464
+ }
465
+ },
466
+ "required": [
467
+ "kind",
468
+ "minimum",
469
+ "maximum",
470
+ "direction"
471
+ ],
472
+ "additionalProperties": false
473
+ }
474
+ ]
475
+ },
476
+ "weight": {
477
+ "type": "number",
478
+ "minimum": 0,
479
+ "maximum": 1000
480
+ },
481
+ "required": {
482
+ "type": "boolean"
483
+ },
484
+ "hardGate": {
485
+ "type": "boolean"
486
+ },
487
+ "privileged": {
488
+ "type": "boolean"
489
+ },
490
+ "fixtureRefs": {
491
+ "maxItems": 1000,
492
+ "type": "array",
493
+ "items": {
494
+ "$ref": "#/$defs/__schema6"
495
+ }
496
+ }
497
+ },
498
+ "required": [
499
+ "graderId",
500
+ "reward",
501
+ "role",
502
+ "normalization",
503
+ "weight",
504
+ "required",
505
+ "hardGate",
506
+ "privileged",
507
+ "fixtureRefs"
508
+ ],
509
+ "additionalProperties": false
510
+ },
511
+ "__schema6": {
512
+ "type": "object",
513
+ "properties": {
514
+ "id": {
515
+ "$ref": "#/$defs/__schema0"
516
+ },
517
+ "contentHash": {
518
+ "$ref": "#/$defs/__schema3"
519
+ }
520
+ },
521
+ "required": [
522
+ "id",
523
+ "contentHash"
524
+ ],
525
+ "additionalProperties": false
526
+ },
527
+ "__schema7": {
528
+ "type": "string",
529
+ "const": "openpond.authoringDraft.v1"
530
+ },
531
+ "__schema8": {
532
+ "type": "integer",
533
+ "exclusiveMinimum": 0,
534
+ "maximum": 9007199254740991
535
+ },
536
+ "__schema9": {
537
+ "type": "string",
538
+ "enum": [
539
+ "draft",
540
+ "archived",
541
+ "published"
542
+ ]
543
+ },
544
+ "__schema10": {
545
+ "anyOf": [
546
+ {
547
+ "$ref": "#/$defs/__schema2"
548
+ },
549
+ {
550
+ "type": "null"
551
+ }
552
+ ]
553
+ },
554
+ "__schema11": {
555
+ "type": "string",
556
+ "format": "date-time",
557
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
558
+ }
559
+ },
560
+ "$id": "urn:openpond:learning:v1:draft",
561
+ "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
562
+ }
@@ -121,6 +121,7 @@
121
121
  "__schema1": {
122
122
  "type": "string",
123
123
  "enum": [
124
+ "draft",
124
125
  "asset",
125
126
  "definition",
126
127
  "reward",