@openpond/evals 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/LEARNING.md +120 -0
  2. package/README.md +2 -0
  3. package/dist/graders.js +26 -5
  4. package/dist/index.js +3 -0
  5. package/dist/javascript-verifier-contract.js +7 -0
  6. package/dist/javascript-verifier-node.js +56 -0
  7. package/dist/javascript-verifier-worker-source.js +1 -0
  8. package/dist/javascript-verifier-worker.js +6 -0
  9. package/dist/javascript-verifier.js +64 -0
  10. package/dist/learning/admission.js +209 -0
  11. package/dist/learning/assets.js +37 -0
  12. package/dist/learning/contracts.js +267 -0
  13. package/dist/learning/errors.js +12 -0
  14. package/dist/learning/grade-worker.js +99 -0
  15. package/dist/learning/index.js +9 -0
  16. package/dist/learning/operations.js +33 -0
  17. package/dist/learning/repository.js +48 -0
  18. package/dist/learning/service.js +377 -0
  19. package/dist/learning/transport.js +35 -0
  20. package/dist/rewards.js +230 -0
  21. package/dist/task-schema-meta-validator.js +2 -0
  22. package/dist/task-schema-validation.js +113 -0
  23. package/dist/task-schema.js +4 -0
  24. package/dist/task-value-validator.js +37 -0
  25. package/dist/tasksets.js +3 -0
  26. package/dist/telemetry-catalog.js +1 -1
  27. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +7 -0
  28. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  29. package/dist/types/conformance.d.ts +14 -0
  30. package/dist/types/conformance.d.ts.map +1 -1
  31. package/dist/types/execution-contracts.d.ts +14 -0
  32. package/dist/types/execution-contracts.d.ts.map +1 -1
  33. package/dist/types/graders.d.ts.map +1 -1
  34. package/dist/types/index.d.ts +3 -0
  35. package/dist/types/index.d.ts.map +1 -1
  36. package/dist/types/javascript-verifier-contract.d.ts +9 -0
  37. package/dist/types/javascript-verifier-contract.d.ts.map +1 -0
  38. package/dist/types/javascript-verifier-node.d.ts +5 -0
  39. package/dist/types/javascript-verifier-node.d.ts.map +1 -0
  40. package/dist/types/javascript-verifier-worker-source.d.ts +2 -0
  41. package/dist/types/javascript-verifier-worker.d.ts +2 -0
  42. package/dist/types/javascript-verifier-worker.d.ts.map +1 -0
  43. package/dist/types/javascript-verifier.d.ts +15 -0
  44. package/dist/types/javascript-verifier.d.ts.map +1 -0
  45. package/dist/types/learning/admission.d.ts +335 -0
  46. package/dist/types/learning/admission.d.ts.map +1 -0
  47. package/dist/types/learning/assets.d.ts +49 -0
  48. package/dist/types/learning/assets.d.ts.map +1 -0
  49. package/dist/types/learning/contracts.d.ts +2149 -0
  50. package/dist/types/learning/contracts.d.ts.map +1 -0
  51. package/dist/types/learning/errors.d.ts +8 -0
  52. package/dist/types/learning/errors.d.ts.map +1 -0
  53. package/dist/types/learning/grade-worker.d.ts +31 -0
  54. package/dist/types/learning/grade-worker.d.ts.map +1 -0
  55. package/dist/types/learning/index.d.ts +10 -0
  56. package/dist/types/learning/index.d.ts.map +1 -0
  57. package/dist/types/learning/operations.d.ts +2061 -0
  58. package/dist/types/learning/operations.d.ts.map +1 -0
  59. package/dist/types/learning/repository.d.ts +1143 -0
  60. package/dist/types/learning/repository.d.ts.map +1 -0
  61. package/dist/types/learning/service.d.ts +36 -0
  62. package/dist/types/learning/service.d.ts.map +1 -0
  63. package/dist/types/learning/transport.d.ts +2173 -0
  64. package/dist/types/learning/transport.d.ts.map +1 -0
  65. package/dist/types/preferences.d.ts +3 -3
  66. package/dist/types/review-conformance.d.ts +5 -5
  67. package/dist/types/rewards.d.ts +611 -0
  68. package/dist/types/rewards.d.ts.map +1 -0
  69. package/dist/types/task-schema-meta-validator.d.ts +3 -0
  70. package/dist/types/task-schema-validation.d.ts +18 -0
  71. package/dist/types/task-schema-validation.d.ts.map +1 -0
  72. package/dist/types/task-schema.d.ts +3 -0
  73. package/dist/types/task-schema.d.ts.map +1 -0
  74. package/dist/types/task-value-validator.d.ts +3 -0
  75. package/dist/types/task-value-validator.d.ts.map +1 -0
  76. package/dist/types/tasksets.d.ts +28 -0
  77. package/dist/types/tasksets.d.ts.map +1 -1
  78. package/package.json +30 -3
  79. package/schemas/learning/v1/asset.schema.json +89 -0
  80. package/schemas/learning/v1/batch.schema.json +132 -0
  81. package/schemas/learning/v1/binding.schema.json +207 -0
  82. package/schemas/learning/v1/command-request.schema.json +2414 -0
  83. package/schemas/learning/v1/decision.schema.json +440 -0
  84. package/schemas/learning/v1/definition.schema.json +397 -0
  85. package/schemas/learning/v1/evidence-inspection.schema.json +106 -0
  86. package/schemas/learning/v1/evidence.schema.json +317 -0
  87. package/schemas/learning/v1/example-submission.schema.json +253 -0
  88. package/schemas/learning/v1/feedback-submission.schema.json +114 -0
  89. package/schemas/learning/v1/feedback.schema.json +233 -0
  90. package/schemas/learning/v1/grade.schema.json +412 -0
  91. package/schemas/learning/v1/iteration.schema.json +236 -0
  92. package/schemas/learning/v1/package.schema.json +776 -0
  93. package/schemas/learning/v1/policy.schema.json +373 -0
  94. package/schemas/learning/v1/read-request.schema.json +142 -0
  95. package/schemas/learning/v1/reward.schema.json +302 -0
  96. package/schemas/learning/v1/source.schema.json +198 -0
@@ -0,0 +1,236 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "type": "object",
4
+ "properties": {
5
+ "schemaVersion": {
6
+ "type": "string",
7
+ "const": "openpond.learningIteration.v1"
8
+ },
9
+ "id": {
10
+ "$ref": "#/$defs/__schema0"
11
+ },
12
+ "revision": {
13
+ "type": "integer",
14
+ "exclusiveMinimum": 0,
15
+ "maximum": 9007199254740991
16
+ },
17
+ "policy": {
18
+ "$ref": "#/$defs/__schema1"
19
+ },
20
+ "status": {
21
+ "type": "string",
22
+ "enum": [
23
+ "waiting_for_data",
24
+ "waiting_for_review",
25
+ "ready",
26
+ "training",
27
+ "evaluating",
28
+ "candidate_ready",
29
+ "accepted",
30
+ "rejected",
31
+ "paused",
32
+ "failed",
33
+ "cancelling",
34
+ "cancelled"
35
+ ]
36
+ },
37
+ "triggerIdentity": {
38
+ "$ref": "#/$defs/__schema0"
39
+ },
40
+ "batch": {
41
+ "anyOf": [
42
+ {
43
+ "$ref": "#/$defs/__schema1"
44
+ },
45
+ {
46
+ "type": "null"
47
+ }
48
+ ]
49
+ },
50
+ "sourceWatermarks": {
51
+ "type": "object",
52
+ "propertyNames": {
53
+ "$ref": "#/$defs/__schema0"
54
+ },
55
+ "additionalProperties": {
56
+ "type": "integer",
57
+ "minimum": 0,
58
+ "maximum": 9007199254740991
59
+ }
60
+ },
61
+ "trainingParent": {
62
+ "$ref": "#/$defs/__schema3"
63
+ },
64
+ "teacher": {
65
+ "anyOf": [
66
+ {
67
+ "$ref": "#/$defs/__schema3"
68
+ },
69
+ {
70
+ "type": "null"
71
+ }
72
+ ]
73
+ },
74
+ "upstreamEvent": {
75
+ "anyOf": [
76
+ {
77
+ "$ref": "#/$defs/__schema3"
78
+ },
79
+ {
80
+ "type": "null"
81
+ }
82
+ ]
83
+ },
84
+ "trainingJob": {
85
+ "anyOf": [
86
+ {
87
+ "$ref": "#/$defs/__schema3"
88
+ },
89
+ {
90
+ "type": "null"
91
+ }
92
+ ]
93
+ },
94
+ "evaluationJob": {
95
+ "anyOf": [
96
+ {
97
+ "$ref": "#/$defs/__schema3"
98
+ },
99
+ {
100
+ "type": "null"
101
+ }
102
+ ]
103
+ },
104
+ "candidateVersion": {
105
+ "anyOf": [
106
+ {
107
+ "$ref": "#/$defs/__schema3"
108
+ },
109
+ {
110
+ "type": "null"
111
+ }
112
+ ]
113
+ },
114
+ "dispatchId": {
115
+ "$ref": "#/$defs/__schema0"
116
+ },
117
+ "retryCount": {
118
+ "type": "integer",
119
+ "minimum": 0,
120
+ "maximum": 9007199254740991
121
+ },
122
+ "spendUsd": {
123
+ "type": "number",
124
+ "minimum": 0
125
+ },
126
+ "failure": {
127
+ "anyOf": [
128
+ {
129
+ "type": "object",
130
+ "properties": {
131
+ "code": {
132
+ "$ref": "#/$defs/__schema0"
133
+ },
134
+ "message": {
135
+ "type": "string",
136
+ "maxLength": 20000
137
+ }
138
+ },
139
+ "required": [
140
+ "code",
141
+ "message"
142
+ ],
143
+ "additionalProperties": false
144
+ },
145
+ {
146
+ "type": "null"
147
+ }
148
+ ]
149
+ },
150
+ "createdAt": {
151
+ "$ref": "#/$defs/__schema4"
152
+ },
153
+ "updatedAt": {
154
+ "$ref": "#/$defs/__schema4"
155
+ }
156
+ },
157
+ "required": [
158
+ "schemaVersion",
159
+ "id",
160
+ "revision",
161
+ "policy",
162
+ "status",
163
+ "triggerIdentity",
164
+ "batch",
165
+ "sourceWatermarks",
166
+ "trainingParent",
167
+ "teacher",
168
+ "upstreamEvent",
169
+ "trainingJob",
170
+ "evaluationJob",
171
+ "candidateVersion",
172
+ "dispatchId",
173
+ "retryCount",
174
+ "spendUsd",
175
+ "failure",
176
+ "createdAt",
177
+ "updatedAt"
178
+ ],
179
+ "additionalProperties": false,
180
+ "$defs": {
181
+ "__schema0": {
182
+ "type": "string",
183
+ "minLength": 1,
184
+ "maxLength": 240
185
+ },
186
+ "__schema1": {
187
+ "type": "object",
188
+ "properties": {
189
+ "id": {
190
+ "$ref": "#/$defs/__schema0"
191
+ },
192
+ "contentHash": {
193
+ "$ref": "#/$defs/__schema2"
194
+ },
195
+ "revision": {
196
+ "type": "integer",
197
+ "exclusiveMinimum": 0,
198
+ "maximum": 9007199254740991
199
+ }
200
+ },
201
+ "required": [
202
+ "id",
203
+ "contentHash",
204
+ "revision"
205
+ ],
206
+ "additionalProperties": false
207
+ },
208
+ "__schema2": {
209
+ "type": "string",
210
+ "pattern": "^[a-f0-9]{64}$"
211
+ },
212
+ "__schema3": {
213
+ "type": "object",
214
+ "properties": {
215
+ "id": {
216
+ "$ref": "#/$defs/__schema0"
217
+ },
218
+ "contentHash": {
219
+ "$ref": "#/$defs/__schema2"
220
+ }
221
+ },
222
+ "required": [
223
+ "id",
224
+ "contentHash"
225
+ ],
226
+ "additionalProperties": false
227
+ },
228
+ "__schema4": {
229
+ "type": "string",
230
+ "format": "date-time",
231
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
232
+ }
233
+ },
234
+ "$id": "urn:openpond:learning:v1:iteration",
235
+ "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
236
+ }