@openpond/evals 0.11.9 → 0.11.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LEARNING.md +35 -0
  2. package/dist/experiments.js +169 -0
  3. package/dist/index.js +1 -0
  4. package/dist/learning/acceptance-group.js +199 -0
  5. package/dist/learning/acceptance-plan.js +7 -1
  6. package/dist/learning/authoring.js +2 -1
  7. package/dist/learning/index.js +1 -0
  8. package/dist/learning/reward-authoring.js +3 -3
  9. package/dist/rewards.js +5 -0
  10. package/dist/types/experiments.d.ts +426 -0
  11. package/dist/types/experiments.d.ts.map +1 -0
  12. package/dist/types/index.d.ts +1 -0
  13. package/dist/types/index.d.ts.map +1 -1
  14. package/dist/types/learning/acceptance-group.d.ts +540 -0
  15. package/dist/types/learning/acceptance-group.d.ts.map +1 -0
  16. package/dist/types/learning/acceptance-plan.d.ts +39 -0
  17. package/dist/types/learning/acceptance-plan.d.ts.map +1 -1
  18. package/dist/types/learning/admission.d.ts +1 -0
  19. package/dist/types/learning/admission.d.ts.map +1 -1
  20. package/dist/types/learning/authoring-service.d.ts +1 -0
  21. package/dist/types/learning/authoring-service.d.ts.map +1 -1
  22. package/dist/types/learning/authoring.d.ts +4 -0
  23. package/dist/types/learning/authoring.d.ts.map +1 -1
  24. package/dist/types/learning/contracts.d.ts +4 -3
  25. package/dist/types/learning/contracts.d.ts.map +1 -1
  26. package/dist/types/learning/index.d.ts +1 -0
  27. package/dist/types/learning/index.d.ts.map +1 -1
  28. package/dist/types/learning/iteration-candidate-decision.d.ts +1 -1
  29. package/dist/types/learning/operations.d.ts +6 -0
  30. package/dist/types/learning/operations.d.ts.map +1 -1
  31. package/dist/types/learning/repository.d.ts +4 -2
  32. package/dist/types/learning/repository.d.ts.map +1 -1
  33. package/dist/types/learning/reward-authoring.d.ts.map +1 -1
  34. package/dist/types/learning/reward-calibration.d.ts +1 -0
  35. package/dist/types/learning/reward-calibration.d.ts.map +1 -1
  36. package/dist/types/learning/transport.d.ts +8 -3
  37. package/dist/types/learning/transport.d.ts.map +1 -1
  38. package/dist/types/rewards.d.ts +4 -0
  39. package/dist/types/rewards.d.ts.map +1 -1
  40. package/package.json +5 -1
  41. package/schemas/learning/v1/acceptance-group-manifest.schema.json +261 -0
  42. package/schemas/learning/v1/acceptance-group-snapshot.schema.json +585 -0
  43. package/schemas/learning/v1/acceptance-plan.schema.json +205 -0
  44. package/schemas/learning/v1/command-request.schema.json +248 -235
  45. package/schemas/learning/v1/draft.schema.json +4 -0
  46. package/schemas/learning/v1/reward.schema.json +4 -0
@@ -0,0 +1,585 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "type": "object",
4
+ "properties": {
5
+ "schemaVersion": {
6
+ "type": "string",
7
+ "const": "openpond.acceptanceGroupSnapshot.v1"
8
+ },
9
+ "manifest": {
10
+ "type": "object",
11
+ "properties": {
12
+ "schemaVersion": {
13
+ "type": "string",
14
+ "const": "openpond.acceptanceGroupManifest.v1"
15
+ },
16
+ "id": {
17
+ "$ref": "#/$defs/__schema0"
18
+ },
19
+ "teamId": {
20
+ "$ref": "#/$defs/__schema0"
21
+ },
22
+ "trainingJob": {
23
+ "$ref": "#/$defs/__schema1"
24
+ },
25
+ "baseline": {
26
+ "$ref": "#/$defs/__schema1"
27
+ },
28
+ "candidate": {
29
+ "$ref": "#/$defs/__schema1"
30
+ },
31
+ "plan": {
32
+ "type": "object",
33
+ "properties": {
34
+ "schemaVersion": {
35
+ "type": "string",
36
+ "const": "openpond.acceptancePlan.v1"
37
+ },
38
+ "id": {
39
+ "$ref": "#/$defs/__schema2"
40
+ },
41
+ "revision": {
42
+ "type": "integer",
43
+ "exclusiveMinimum": 0,
44
+ "maximum": 9007199254740991
45
+ },
46
+ "checks": {
47
+ "minItems": 1,
48
+ "maxItems": 100,
49
+ "type": "array",
50
+ "items": {
51
+ "$ref": "#/$defs/__schema3"
52
+ }
53
+ },
54
+ "maximumSpendUsd": {
55
+ "type": "number",
56
+ "exclusiveMinimum": 0
57
+ },
58
+ "contentHash": {
59
+ "$ref": "#/$defs/__schema9"
60
+ }
61
+ },
62
+ "required": [
63
+ "schemaVersion",
64
+ "id",
65
+ "revision",
66
+ "checks",
67
+ "maximumSpendUsd",
68
+ "contentHash"
69
+ ],
70
+ "additionalProperties": false
71
+ },
72
+ "maximumParallelAttempts": {
73
+ "type": "integer",
74
+ "minimum": 1,
75
+ "maximum": 8
76
+ },
77
+ "createdAt": {
78
+ "$ref": "#/$defs/__schema15"
79
+ },
80
+ "contentHash": {
81
+ "$ref": "#/$defs/__schema16"
82
+ }
83
+ },
84
+ "required": [
85
+ "schemaVersion",
86
+ "id",
87
+ "teamId",
88
+ "trainingJob",
89
+ "baseline",
90
+ "candidate",
91
+ "plan",
92
+ "maximumParallelAttempts",
93
+ "createdAt",
94
+ "contentHash"
95
+ ],
96
+ "additionalProperties": false
97
+ },
98
+ "revision": {
99
+ "type": "integer",
100
+ "exclusiveMinimum": 0,
101
+ "maximum": 9007199254740991
102
+ },
103
+ "state": {
104
+ "type": "string",
105
+ "enum": [
106
+ "queued",
107
+ "running",
108
+ "blocked",
109
+ "cancelling",
110
+ "cleaning",
111
+ "completed",
112
+ "failed",
113
+ "cancelled",
114
+ "budget_exhausted"
115
+ ]
116
+ },
117
+ "attempts": {
118
+ "maxItems": 2000,
119
+ "type": "array",
120
+ "items": {
121
+ "$ref": "#/$defs/__schema17"
122
+ }
123
+ },
124
+ "failureCode": {
125
+ "anyOf": [
126
+ {
127
+ "$ref": "#/$defs/__schema0"
128
+ },
129
+ {
130
+ "type": "null"
131
+ }
132
+ ]
133
+ },
134
+ "updatedAt": {
135
+ "$ref": "#/$defs/__schema15"
136
+ },
137
+ "contentHash": {
138
+ "$ref": "#/$defs/__schema16"
139
+ }
140
+ },
141
+ "required": [
142
+ "schemaVersion",
143
+ "manifest",
144
+ "revision",
145
+ "state",
146
+ "attempts",
147
+ "failureCode",
148
+ "updatedAt",
149
+ "contentHash"
150
+ ],
151
+ "additionalProperties": false,
152
+ "$defs": {
153
+ "__schema0": {
154
+ "type": "string",
155
+ "minLength": 1,
156
+ "maxLength": 240
157
+ },
158
+ "__schema1": {
159
+ "type": "object",
160
+ "properties": {
161
+ "id": {
162
+ "type": "string",
163
+ "minLength": 1,
164
+ "maxLength": 240
165
+ },
166
+ "contentHash": {
167
+ "type": "string",
168
+ "pattern": "^[a-f0-9]{64}$"
169
+ }
170
+ },
171
+ "required": [
172
+ "id",
173
+ "contentHash"
174
+ ],
175
+ "additionalProperties": false
176
+ },
177
+ "__schema2": {
178
+ "type": "string",
179
+ "minLength": 1,
180
+ "maxLength": 240
181
+ },
182
+ "__schema3": {
183
+ "type": "object",
184
+ "properties": {
185
+ "id": {
186
+ "$ref": "#/$defs/__schema2"
187
+ },
188
+ "name": {
189
+ "$ref": "#/$defs/__schema4"
190
+ },
191
+ "required": {
192
+ "$ref": "#/$defs/__schema5"
193
+ },
194
+ "role": {
195
+ "$ref": "#/$defs/__schema6"
196
+ },
197
+ "unit": {
198
+ "$ref": "#/$defs/__schema7"
199
+ },
200
+ "baselineReuse": {
201
+ "$ref": "#/$defs/__schema8"
202
+ },
203
+ "dataset": {
204
+ "$ref": "#/$defs/__schema1"
205
+ },
206
+ "evaluator": {
207
+ "$ref": "#/$defs/__schema1"
208
+ },
209
+ "executionHash": {
210
+ "$ref": "#/$defs/__schema9"
211
+ },
212
+ "populationHash": {
213
+ "$ref": "#/$defs/__schema9"
214
+ },
215
+ "metric": {
216
+ "$ref": "#/$defs/__schema2"
217
+ },
218
+ "direction": {
219
+ "$ref": "#/$defs/__schema10"
220
+ },
221
+ "minimumCoverage": {
222
+ "$ref": "#/$defs/__schema11"
223
+ },
224
+ "threshold": {
225
+ "$ref": "#/$defs/__schema12"
226
+ },
227
+ "maximumRegression": {
228
+ "$ref": "#/$defs/__schema13"
229
+ },
230
+ "maximumSpendUsd": {
231
+ "$ref": "#/$defs/__schema14"
232
+ }
233
+ },
234
+ "required": [
235
+ "id",
236
+ "name",
237
+ "required",
238
+ "dataset",
239
+ "evaluator",
240
+ "executionHash",
241
+ "populationHash",
242
+ "metric",
243
+ "direction",
244
+ "minimumCoverage",
245
+ "threshold",
246
+ "maximumRegression",
247
+ "maximumSpendUsd"
248
+ ],
249
+ "additionalProperties": false
250
+ },
251
+ "__schema4": {
252
+ "type": "string",
253
+ "minLength": 1,
254
+ "maxLength": 200
255
+ },
256
+ "__schema5": {
257
+ "type": "boolean"
258
+ },
259
+ "__schema6": {
260
+ "type": "string",
261
+ "enum": [
262
+ "quality",
263
+ "retention",
264
+ "diagnostic"
265
+ ]
266
+ },
267
+ "__schema7": {
268
+ "type": "string",
269
+ "minLength": 1,
270
+ "maxLength": 80
271
+ },
272
+ "__schema8": {
273
+ "type": "object",
274
+ "properties": {
275
+ "maximumAgeSeconds": {
276
+ "type": "integer",
277
+ "exclusiveMinimum": 0,
278
+ "maximum": 31536000
279
+ },
280
+ "qualification": {
281
+ "$ref": "#/$defs/__schema1"
282
+ }
283
+ },
284
+ "required": [
285
+ "maximumAgeSeconds",
286
+ "qualification"
287
+ ],
288
+ "additionalProperties": false
289
+ },
290
+ "__schema9": {
291
+ "type": "string",
292
+ "pattern": "^[a-f0-9]{64}$"
293
+ },
294
+ "__schema10": {
295
+ "type": "string",
296
+ "enum": [
297
+ "higher",
298
+ "lower"
299
+ ]
300
+ },
301
+ "__schema11": {
302
+ "type": "number",
303
+ "exclusiveMinimum": 0,
304
+ "maximum": 1
305
+ },
306
+ "__schema12": {
307
+ "type": "number"
308
+ },
309
+ "__schema13": {
310
+ "type": "number",
311
+ "minimum": 0
312
+ },
313
+ "__schema14": {
314
+ "type": "number",
315
+ "exclusiveMinimum": 0
316
+ },
317
+ "__schema15": {
318
+ "type": "string",
319
+ "format": "date-time",
320
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$"
321
+ },
322
+ "__schema16": {
323
+ "type": "string",
324
+ "pattern": "^[a-f0-9]{64}$"
325
+ },
326
+ "__schema17": {
327
+ "type": "object",
328
+ "properties": {
329
+ "id": {
330
+ "$ref": "#/$defs/__schema0"
331
+ },
332
+ "checkId": {
333
+ "$ref": "#/$defs/__schema0"
334
+ },
335
+ "subject": {
336
+ "$ref": "#/$defs/__schema18"
337
+ },
338
+ "ordinal": {
339
+ "$ref": "#/$defs/__schema19"
340
+ },
341
+ "dispatchId": {
342
+ "$ref": "#/$defs/__schema0"
343
+ },
344
+ "state": {
345
+ "$ref": "#/$defs/__schema20"
346
+ },
347
+ "artifact": {
348
+ "$ref": "#/$defs/__schema1"
349
+ },
350
+ "checkHash": {
351
+ "$ref": "#/$defs/__schema16"
352
+ },
353
+ "measurement": {
354
+ "$ref": "#/$defs/__schema21"
355
+ },
356
+ "executionReceipt": {
357
+ "$ref": "#/$defs/__schema26"
358
+ },
359
+ "reusedFrom": {
360
+ "$ref": "#/$defs/__schema27"
361
+ },
362
+ "spendUsd": {
363
+ "$ref": "#/$defs/__schema28"
364
+ },
365
+ "reservedUsd": {
366
+ "$ref": "#/$defs/__schema28"
367
+ },
368
+ "cleanupComplete": {
369
+ "$ref": "#/$defs/__schema29"
370
+ },
371
+ "accountingComplete": {
372
+ "$ref": "#/$defs/__schema30"
373
+ },
374
+ "failureCode": {
375
+ "$ref": "#/$defs/__schema31"
376
+ },
377
+ "createdAt": {
378
+ "$ref": "#/$defs/__schema15"
379
+ },
380
+ "updatedAt": {
381
+ "$ref": "#/$defs/__schema15"
382
+ }
383
+ },
384
+ "required": [
385
+ "id",
386
+ "checkId",
387
+ "subject",
388
+ "ordinal",
389
+ "dispatchId",
390
+ "state",
391
+ "artifact",
392
+ "checkHash",
393
+ "measurement",
394
+ "executionReceipt",
395
+ "reusedFrom",
396
+ "spendUsd",
397
+ "reservedUsd",
398
+ "cleanupComplete",
399
+ "accountingComplete",
400
+ "failureCode",
401
+ "createdAt",
402
+ "updatedAt"
403
+ ],
404
+ "additionalProperties": false
405
+ },
406
+ "__schema18": {
407
+ "type": "string",
408
+ "enum": [
409
+ "baseline",
410
+ "candidate"
411
+ ]
412
+ },
413
+ "__schema19": {
414
+ "type": "integer",
415
+ "exclusiveMinimum": 0,
416
+ "maximum": 9007199254740991
417
+ },
418
+ "__schema20": {
419
+ "type": "string",
420
+ "enum": [
421
+ "queued",
422
+ "dispatching",
423
+ "running",
424
+ "grading",
425
+ "cleaning",
426
+ "completed",
427
+ "failed",
428
+ "cancelled",
429
+ "budget_exhausted"
430
+ ]
431
+ },
432
+ "__schema21": {
433
+ "anyOf": [
434
+ {
435
+ "type": "object",
436
+ "properties": {
437
+ "runId": {
438
+ "$ref": "#/$defs/__schema2"
439
+ },
440
+ "artifact": {
441
+ "$ref": "#/$defs/__schema1"
442
+ },
443
+ "checkHash": {
444
+ "$ref": "#/$defs/__schema9"
445
+ },
446
+ "populationHash": {
447
+ "$ref": "#/$defs/__schema9"
448
+ },
449
+ "scoredPopulationHash": {
450
+ "$ref": "#/$defs/__schema9"
451
+ },
452
+ "executionHash": {
453
+ "$ref": "#/$defs/__schema9"
454
+ },
455
+ "evaluator": {
456
+ "$ref": "#/$defs/__schema1"
457
+ },
458
+ "metric": {
459
+ "$ref": "#/$defs/__schema2"
460
+ },
461
+ "expectedCount": {
462
+ "$ref": "#/$defs/__schema22"
463
+ },
464
+ "scoredCount": {
465
+ "$ref": "#/$defs/__schema23"
466
+ },
467
+ "score": {
468
+ "$ref": "#/$defs/__schema24"
469
+ },
470
+ "status": {
471
+ "$ref": "#/$defs/__schema25"
472
+ },
473
+ "evidenceHash": {
474
+ "$ref": "#/$defs/__schema9"
475
+ }
476
+ },
477
+ "required": [
478
+ "runId",
479
+ "artifact",
480
+ "checkHash",
481
+ "populationHash",
482
+ "scoredPopulationHash",
483
+ "executionHash",
484
+ "evaluator",
485
+ "metric",
486
+ "expectedCount",
487
+ "scoredCount",
488
+ "score",
489
+ "status",
490
+ "evidenceHash"
491
+ ],
492
+ "additionalProperties": false
493
+ },
494
+ {
495
+ "type": "null"
496
+ }
497
+ ]
498
+ },
499
+ "__schema22": {
500
+ "type": "integer",
501
+ "exclusiveMinimum": 0,
502
+ "maximum": 9007199254740991
503
+ },
504
+ "__schema23": {
505
+ "type": "integer",
506
+ "minimum": 0,
507
+ "maximum": 9007199254740991
508
+ },
509
+ "__schema24": {
510
+ "anyOf": [
511
+ {
512
+ "type": "number"
513
+ },
514
+ {
515
+ "type": "null"
516
+ }
517
+ ]
518
+ },
519
+ "__schema25": {
520
+ "type": "string",
521
+ "enum": [
522
+ "pending",
523
+ "running",
524
+ "completed",
525
+ "failed",
526
+ "cancelled"
527
+ ]
528
+ },
529
+ "__schema26": {
530
+ "anyOf": [
531
+ {
532
+ "$ref": "#/$defs/__schema1"
533
+ },
534
+ {
535
+ "type": "null"
536
+ }
537
+ ]
538
+ },
539
+ "__schema27": {
540
+ "anyOf": [
541
+ {
542
+ "type": "object",
543
+ "properties": {
544
+ "group": {
545
+ "$ref": "#/$defs/__schema1"
546
+ },
547
+ "attemptId": {
548
+ "$ref": "#/$defs/__schema0"
549
+ }
550
+ },
551
+ "required": [
552
+ "group",
553
+ "attemptId"
554
+ ],
555
+ "additionalProperties": false
556
+ },
557
+ {
558
+ "type": "null"
559
+ }
560
+ ]
561
+ },
562
+ "__schema28": {
563
+ "type": "number",
564
+ "minimum": 0
565
+ },
566
+ "__schema29": {
567
+ "type": "boolean"
568
+ },
569
+ "__schema30": {
570
+ "type": "boolean"
571
+ },
572
+ "__schema31": {
573
+ "anyOf": [
574
+ {
575
+ "$ref": "#/$defs/__schema0"
576
+ },
577
+ {
578
+ "type": "null"
579
+ }
580
+ ]
581
+ }
582
+ },
583
+ "$id": "urn:openpond:learning:v1:acceptance-group-snapshot",
584
+ "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
585
+ }