@openpond/evals 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CONTRACT.md +13 -1
  2. package/LEARNING.md +106 -0
  3. package/README.md +28 -2
  4. package/conformance/telemetry/v1/invalid-batch.json +17 -0
  5. package/conformance/telemetry/v1/valid-batch.json +58 -0
  6. package/dist/graders.js +26 -5
  7. package/dist/index.js +8 -0
  8. package/dist/javascript-verifier-contract.js +7 -0
  9. package/dist/javascript-verifier-node.js +56 -0
  10. package/dist/javascript-verifier-worker-source.js +1 -0
  11. package/dist/javascript-verifier-worker.js +6 -0
  12. package/dist/javascript-verifier.js +64 -0
  13. package/dist/learned-preference.js +334 -0
  14. package/dist/learning/admission.js +209 -0
  15. package/dist/learning/assets.js +37 -0
  16. package/dist/learning/contracts.js +267 -0
  17. package/dist/learning/errors.js +12 -0
  18. package/dist/learning/grade-worker.js +99 -0
  19. package/dist/learning/index.js +9 -0
  20. package/dist/learning/operations.js +33 -0
  21. package/dist/learning/repository.js +48 -0
  22. package/dist/learning/service.js +367 -0
  23. package/dist/learning/transport.js +25 -0
  24. package/dist/preferences.js +77 -11
  25. package/dist/rewards.js +230 -0
  26. package/dist/task-schema.js +142 -0
  27. package/dist/tasksets.js +3 -0
  28. package/dist/telemetry/index.js +4 -0
  29. package/dist/telemetry-analysis.js +183 -0
  30. package/dist/telemetry-bundle.js +60 -0
  31. package/dist/telemetry-catalog.js +50 -0
  32. package/dist/telemetry.js +112 -0
  33. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +7 -0
  34. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  35. package/dist/types/conformance.d.ts +14 -0
  36. package/dist/types/conformance.d.ts.map +1 -1
  37. package/dist/types/execution-contracts.d.ts +14 -0
  38. package/dist/types/execution-contracts.d.ts.map +1 -1
  39. package/dist/types/graders.d.ts.map +1 -1
  40. package/dist/types/index.d.ts +8 -0
  41. package/dist/types/index.d.ts.map +1 -1
  42. package/dist/types/javascript-verifier-contract.d.ts +9 -0
  43. package/dist/types/javascript-verifier-contract.d.ts.map +1 -0
  44. package/dist/types/javascript-verifier-node.d.ts +5 -0
  45. package/dist/types/javascript-verifier-node.d.ts.map +1 -0
  46. package/dist/types/javascript-verifier-worker-source.d.ts +2 -0
  47. package/dist/types/javascript-verifier-worker.d.ts +2 -0
  48. package/dist/types/javascript-verifier-worker.d.ts.map +1 -0
  49. package/dist/types/javascript-verifier.d.ts +15 -0
  50. package/dist/types/javascript-verifier.d.ts.map +1 -0
  51. package/dist/types/learned-preference.d.ts +282 -0
  52. package/dist/types/learned-preference.d.ts.map +1 -0
  53. package/dist/types/learning/admission.d.ts +335 -0
  54. package/dist/types/learning/admission.d.ts.map +1 -0
  55. package/dist/types/learning/assets.d.ts +49 -0
  56. package/dist/types/learning/assets.d.ts.map +1 -0
  57. package/dist/types/learning/contracts.d.ts +2149 -0
  58. package/dist/types/learning/contracts.d.ts.map +1 -0
  59. package/dist/types/learning/errors.d.ts +8 -0
  60. package/dist/types/learning/errors.d.ts.map +1 -0
  61. package/dist/types/learning/grade-worker.d.ts +31 -0
  62. package/dist/types/learning/grade-worker.d.ts.map +1 -0
  63. package/dist/types/learning/index.d.ts +10 -0
  64. package/dist/types/learning/index.d.ts.map +1 -0
  65. package/dist/types/learning/operations.d.ts +2061 -0
  66. package/dist/types/learning/operations.d.ts.map +1 -0
  67. package/dist/types/learning/repository.d.ts +1143 -0
  68. package/dist/types/learning/repository.d.ts.map +1 -0
  69. package/dist/types/learning/service.d.ts +22 -0
  70. package/dist/types/learning/service.d.ts.map +1 -0
  71. package/dist/types/learning/transport.d.ts +2139 -0
  72. package/dist/types/learning/transport.d.ts.map +1 -0
  73. package/dist/types/preferences.d.ts +55 -7
  74. package/dist/types/preferences.d.ts.map +1 -1
  75. package/dist/types/review-conformance.d.ts +5 -5
  76. package/dist/types/rewards.d.ts +611 -0
  77. package/dist/types/rewards.d.ts.map +1 -0
  78. package/dist/types/task-schema.d.ts +18 -0
  79. package/dist/types/task-schema.d.ts.map +1 -0
  80. package/dist/types/tasksets.d.ts +28 -0
  81. package/dist/types/tasksets.d.ts.map +1 -1
  82. package/dist/types/telemetry/index.d.ts +5 -0
  83. package/dist/types/telemetry/index.d.ts.map +1 -0
  84. package/dist/types/telemetry-analysis.d.ts +116 -0
  85. package/dist/types/telemetry-analysis.d.ts.map +1 -0
  86. package/dist/types/telemetry-bundle.d.ts +309 -0
  87. package/dist/types/telemetry-bundle.d.ts.map +1 -0
  88. package/dist/types/telemetry-catalog.d.ts +269 -0
  89. package/dist/types/telemetry-catalog.d.ts.map +1 -0
  90. package/dist/types/telemetry.d.ts +266 -0
  91. package/dist/types/telemetry.d.ts.map +1 -0
  92. package/package.json +40 -3
  93. package/schemas/learning/v1/asset.schema.json +89 -0
  94. package/schemas/learning/v1/batch.schema.json +132 -0
  95. package/schemas/learning/v1/binding.schema.json +207 -0
  96. package/schemas/learning/v1/command-request.schema.json +2414 -0
  97. package/schemas/learning/v1/decision.schema.json +440 -0
  98. package/schemas/learning/v1/definition.schema.json +397 -0
  99. package/schemas/learning/v1/evidence.schema.json +317 -0
  100. package/schemas/learning/v1/example-submission.schema.json +253 -0
  101. package/schemas/learning/v1/feedback-submission.schema.json +114 -0
  102. package/schemas/learning/v1/feedback.schema.json +233 -0
  103. package/schemas/learning/v1/grade.schema.json +412 -0
  104. package/schemas/learning/v1/iteration.schema.json +236 -0
  105. package/schemas/learning/v1/package.schema.json +776 -0
  106. package/schemas/learning/v1/policy.schema.json +373 -0
  107. package/schemas/learning/v1/read-request.schema.json +101 -0
  108. package/schemas/learning/v1/reward.schema.json +302 -0
  109. package/schemas/learning/v1/source.schema.json +198 -0
  110. package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
  111. package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
  112. package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
  113. package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
  114. package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
  115. package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
  116. package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
  117. package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
  118. package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0
@@ -0,0 +1,611 @@
1
+ import { z } from "zod";
2
+ import { type AttemptEvidence, type CustomVerifierRunner, type ModelJudgeRunner } from "./graders.js";
3
+ import { type GraderSpec, type TaskRecord } from "./tasksets.js";
4
+ export declare const RewardImplementationSchema: z.ZodUnion<readonly [z.ZodObject<{
5
+ kind: z.ZodEnum<{
6
+ content: "content";
7
+ schema: "schema";
8
+ artifact: "artifact";
9
+ runtime_event: "runtime_event";
10
+ state: "state";
11
+ }>;
12
+ config: z.ZodRecord<z.ZodString, z.ZodUnknown>;
13
+ }, z.core.$strict>, z.ZodObject<{
14
+ kind: z.ZodLiteral<"model_judge">;
15
+ rubricRef: z.ZodObject<{
16
+ id: z.ZodString;
17
+ path: z.ZodString;
18
+ contentHash: z.ZodString;
19
+ sizeBytes: z.ZodNumber;
20
+ mediaType: z.ZodString;
21
+ visibility: z.ZodEnum<{
22
+ policy: "policy";
23
+ verifier: "verifier";
24
+ host_private: "host_private";
25
+ }>;
26
+ }, z.core.$strict>;
27
+ calibrationStatus: z.ZodEnum<{
28
+ pending: "pending";
29
+ passed: "passed";
30
+ failed: "failed";
31
+ }>;
32
+ model: z.ZodOptional<z.ZodObject<{
33
+ providerId: z.ZodString;
34
+ modelId: z.ZodString;
35
+ revision: z.ZodNullable<z.ZodString>;
36
+ }, z.core.$strict>>;
37
+ temperature: z.ZodOptional<z.ZodNumber>;
38
+ }, z.core.$strict>, z.ZodObject<{
39
+ kind: z.ZodLiteral<"custom_verifier">;
40
+ networkPolicy: z.ZodLiteral<"none">;
41
+ verifierRef: z.ZodObject<{
42
+ id: z.ZodString;
43
+ path: z.ZodString;
44
+ contentHash: z.ZodString;
45
+ sizeBytes: z.ZodNumber;
46
+ mediaType: z.ZodString;
47
+ visibility: z.ZodEnum<{
48
+ policy: "policy";
49
+ verifier: "verifier";
50
+ host_private: "host_private";
51
+ }>;
52
+ }, z.core.$strict>;
53
+ exportName: z.ZodOptional<z.ZodString>;
54
+ timeoutMs: z.ZodNumber;
55
+ }, z.core.$strict>, z.ZodObject<{
56
+ kind: z.ZodLiteral<"human">;
57
+ rubricRef: z.ZodObject<{
58
+ id: z.ZodString;
59
+ path: z.ZodString;
60
+ contentHash: z.ZodString;
61
+ sizeBytes: z.ZodNumber;
62
+ mediaType: z.ZodString;
63
+ visibility: z.ZodEnum<{
64
+ policy: "policy";
65
+ verifier: "verifier";
66
+ host_private: "host_private";
67
+ }>;
68
+ }, z.core.$strict>;
69
+ reviewerRole: z.ZodString;
70
+ }, z.core.$strict>, z.ZodObject<{
71
+ kind: z.ZodLiteral<"learned_model">;
72
+ modelVersion: z.ZodObject<{
73
+ id: z.ZodString;
74
+ contentHash: z.ZodString;
75
+ }, z.core.$strict>;
76
+ inputContract: z.ZodObject<{
77
+ id: z.ZodString;
78
+ path: z.ZodString;
79
+ contentHash: z.ZodString;
80
+ sizeBytes: z.ZodNumber;
81
+ mediaType: z.ZodString;
82
+ visibility: z.ZodEnum<{
83
+ policy: "policy";
84
+ verifier: "verifier";
85
+ host_private: "host_private";
86
+ }>;
87
+ }, z.core.$strict>;
88
+ }, z.core.$strict>]>;
89
+ export declare const RewardReleaseContentSchema: z.ZodObject<{
90
+ schemaVersion: z.ZodLiteral<"openpond.rewardRelease.v1">;
91
+ id: z.ZodString;
92
+ revision: z.ZodNumber;
93
+ name: z.ZodString;
94
+ description: z.ZodString;
95
+ implementation: z.ZodUnion<readonly [z.ZodObject<{
96
+ kind: z.ZodEnum<{
97
+ content: "content";
98
+ schema: "schema";
99
+ artifact: "artifact";
100
+ runtime_event: "runtime_event";
101
+ state: "state";
102
+ }>;
103
+ config: z.ZodRecord<z.ZodString, z.ZodUnknown>;
104
+ }, z.core.$strict>, z.ZodObject<{
105
+ kind: z.ZodLiteral<"model_judge">;
106
+ rubricRef: z.ZodObject<{
107
+ id: z.ZodString;
108
+ path: z.ZodString;
109
+ contentHash: z.ZodString;
110
+ sizeBytes: z.ZodNumber;
111
+ mediaType: z.ZodString;
112
+ visibility: z.ZodEnum<{
113
+ policy: "policy";
114
+ verifier: "verifier";
115
+ host_private: "host_private";
116
+ }>;
117
+ }, z.core.$strict>;
118
+ calibrationStatus: z.ZodEnum<{
119
+ pending: "pending";
120
+ passed: "passed";
121
+ failed: "failed";
122
+ }>;
123
+ model: z.ZodOptional<z.ZodObject<{
124
+ providerId: z.ZodString;
125
+ modelId: z.ZodString;
126
+ revision: z.ZodNullable<z.ZodString>;
127
+ }, z.core.$strict>>;
128
+ temperature: z.ZodOptional<z.ZodNumber>;
129
+ }, z.core.$strict>, z.ZodObject<{
130
+ kind: z.ZodLiteral<"custom_verifier">;
131
+ networkPolicy: z.ZodLiteral<"none">;
132
+ verifierRef: z.ZodObject<{
133
+ id: z.ZodString;
134
+ path: z.ZodString;
135
+ contentHash: z.ZodString;
136
+ sizeBytes: z.ZodNumber;
137
+ mediaType: z.ZodString;
138
+ visibility: z.ZodEnum<{
139
+ policy: "policy";
140
+ verifier: "verifier";
141
+ host_private: "host_private";
142
+ }>;
143
+ }, z.core.$strict>;
144
+ exportName: z.ZodOptional<z.ZodString>;
145
+ timeoutMs: z.ZodNumber;
146
+ }, z.core.$strict>, z.ZodObject<{
147
+ kind: z.ZodLiteral<"human">;
148
+ rubricRef: z.ZodObject<{
149
+ id: z.ZodString;
150
+ path: z.ZodString;
151
+ contentHash: z.ZodString;
152
+ sizeBytes: z.ZodNumber;
153
+ mediaType: z.ZodString;
154
+ visibility: z.ZodEnum<{
155
+ policy: "policy";
156
+ verifier: "verifier";
157
+ host_private: "host_private";
158
+ }>;
159
+ }, z.core.$strict>;
160
+ reviewerRole: z.ZodString;
161
+ }, z.core.$strict>, z.ZodObject<{
162
+ kind: z.ZodLiteral<"learned_model">;
163
+ modelVersion: z.ZodObject<{
164
+ id: z.ZodString;
165
+ contentHash: z.ZodString;
166
+ }, z.core.$strict>;
167
+ inputContract: z.ZodObject<{
168
+ id: z.ZodString;
169
+ path: z.ZodString;
170
+ contentHash: z.ZodString;
171
+ sizeBytes: z.ZodNumber;
172
+ mediaType: z.ZodString;
173
+ visibility: z.ZodEnum<{
174
+ policy: "policy";
175
+ verifier: "verifier";
176
+ host_private: "host_private";
177
+ }>;
178
+ }, z.core.$strict>;
179
+ }, z.core.$strict>]>;
180
+ rawScore: z.ZodObject<{
181
+ minimum: z.ZodNumber;
182
+ maximum: z.ZodNumber;
183
+ }, z.core.$strict>;
184
+ assets: z.ZodArray<z.ZodObject<{
185
+ id: z.ZodString;
186
+ path: z.ZodString;
187
+ contentHash: z.ZodString;
188
+ sizeBytes: z.ZodNumber;
189
+ mediaType: z.ZodString;
190
+ visibility: z.ZodEnum<{
191
+ policy: "policy";
192
+ verifier: "verifier";
193
+ host_private: "host_private";
194
+ }>;
195
+ }, z.core.$strict>>;
196
+ }, z.core.$strict>;
197
+ export declare const RewardReleaseSchema: z.ZodObject<{
198
+ schemaVersion: z.ZodLiteral<"openpond.rewardRelease.v1">;
199
+ id: z.ZodString;
200
+ revision: z.ZodNumber;
201
+ name: z.ZodString;
202
+ description: z.ZodString;
203
+ implementation: z.ZodUnion<readonly [z.ZodObject<{
204
+ kind: z.ZodEnum<{
205
+ content: "content";
206
+ schema: "schema";
207
+ artifact: "artifact";
208
+ runtime_event: "runtime_event";
209
+ state: "state";
210
+ }>;
211
+ config: z.ZodRecord<z.ZodString, z.ZodUnknown>;
212
+ }, z.core.$strict>, z.ZodObject<{
213
+ kind: z.ZodLiteral<"model_judge">;
214
+ rubricRef: z.ZodObject<{
215
+ id: z.ZodString;
216
+ path: z.ZodString;
217
+ contentHash: z.ZodString;
218
+ sizeBytes: z.ZodNumber;
219
+ mediaType: z.ZodString;
220
+ visibility: z.ZodEnum<{
221
+ policy: "policy";
222
+ verifier: "verifier";
223
+ host_private: "host_private";
224
+ }>;
225
+ }, z.core.$strict>;
226
+ calibrationStatus: z.ZodEnum<{
227
+ pending: "pending";
228
+ passed: "passed";
229
+ failed: "failed";
230
+ }>;
231
+ model: z.ZodOptional<z.ZodObject<{
232
+ providerId: z.ZodString;
233
+ modelId: z.ZodString;
234
+ revision: z.ZodNullable<z.ZodString>;
235
+ }, z.core.$strict>>;
236
+ temperature: z.ZodOptional<z.ZodNumber>;
237
+ }, z.core.$strict>, z.ZodObject<{
238
+ kind: z.ZodLiteral<"custom_verifier">;
239
+ networkPolicy: z.ZodLiteral<"none">;
240
+ verifierRef: z.ZodObject<{
241
+ id: z.ZodString;
242
+ path: z.ZodString;
243
+ contentHash: z.ZodString;
244
+ sizeBytes: z.ZodNumber;
245
+ mediaType: z.ZodString;
246
+ visibility: z.ZodEnum<{
247
+ policy: "policy";
248
+ verifier: "verifier";
249
+ host_private: "host_private";
250
+ }>;
251
+ }, z.core.$strict>;
252
+ exportName: z.ZodOptional<z.ZodString>;
253
+ timeoutMs: z.ZodNumber;
254
+ }, z.core.$strict>, z.ZodObject<{
255
+ kind: z.ZodLiteral<"human">;
256
+ rubricRef: z.ZodObject<{
257
+ id: z.ZodString;
258
+ path: z.ZodString;
259
+ contentHash: z.ZodString;
260
+ sizeBytes: z.ZodNumber;
261
+ mediaType: z.ZodString;
262
+ visibility: z.ZodEnum<{
263
+ policy: "policy";
264
+ verifier: "verifier";
265
+ host_private: "host_private";
266
+ }>;
267
+ }, z.core.$strict>;
268
+ reviewerRole: z.ZodString;
269
+ }, z.core.$strict>, z.ZodObject<{
270
+ kind: z.ZodLiteral<"learned_model">;
271
+ modelVersion: z.ZodObject<{
272
+ id: z.ZodString;
273
+ contentHash: z.ZodString;
274
+ }, z.core.$strict>;
275
+ inputContract: z.ZodObject<{
276
+ id: z.ZodString;
277
+ path: z.ZodString;
278
+ contentHash: z.ZodString;
279
+ sizeBytes: z.ZodNumber;
280
+ mediaType: z.ZodString;
281
+ visibility: z.ZodEnum<{
282
+ policy: "policy";
283
+ verifier: "verifier";
284
+ host_private: "host_private";
285
+ }>;
286
+ }, z.core.$strict>;
287
+ }, z.core.$strict>]>;
288
+ rawScore: z.ZodObject<{
289
+ minimum: z.ZodNumber;
290
+ maximum: z.ZodNumber;
291
+ }, z.core.$strict>;
292
+ assets: z.ZodArray<z.ZodObject<{
293
+ id: z.ZodString;
294
+ path: z.ZodString;
295
+ contentHash: z.ZodString;
296
+ sizeBytes: z.ZodNumber;
297
+ mediaType: z.ZodString;
298
+ visibility: z.ZodEnum<{
299
+ policy: "policy";
300
+ verifier: "verifier";
301
+ host_private: "host_private";
302
+ }>;
303
+ }, z.core.$strict>>;
304
+ contentHash: z.ZodString;
305
+ }, z.core.$strict>;
306
+ export declare const RewardReleaseRefSchema: z.ZodObject<{
307
+ id: z.ZodString;
308
+ contentHash: z.ZodString;
309
+ revision: z.ZodNumber;
310
+ }, z.core.$strict>;
311
+ export declare const RewardNormalizationSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
312
+ kind: z.ZodLiteral<"identity">;
313
+ }, z.core.$strict>, z.ZodObject<{
314
+ kind: z.ZodLiteral<"linear">;
315
+ minimum: z.ZodNumber;
316
+ maximum: z.ZodNumber;
317
+ direction: z.ZodEnum<{
318
+ higher: "higher";
319
+ lower: "lower";
320
+ }>;
321
+ }, z.core.$strict>], "kind">;
322
+ export declare const RewardBindingSourceSchema: z.ZodObject<{
323
+ graderId: z.ZodString;
324
+ reward: z.ZodObject<{
325
+ id: z.ZodString;
326
+ contentHash: z.ZodString;
327
+ revision: z.ZodNumber;
328
+ }, z.core.$strict>;
329
+ role: z.ZodEnum<{
330
+ evaluation: "evaluation";
331
+ training: "training";
332
+ }>;
333
+ normalization: z.ZodDiscriminatedUnion<[z.ZodObject<{
334
+ kind: z.ZodLiteral<"identity">;
335
+ }, z.core.$strict>, z.ZodObject<{
336
+ kind: z.ZodLiteral<"linear">;
337
+ minimum: z.ZodNumber;
338
+ maximum: z.ZodNumber;
339
+ direction: z.ZodEnum<{
340
+ higher: "higher";
341
+ lower: "lower";
342
+ }>;
343
+ }, z.core.$strict>], "kind">;
344
+ weight: z.ZodNumber;
345
+ required: z.ZodBoolean;
346
+ hardGate: z.ZodBoolean;
347
+ privileged: z.ZodBoolean;
348
+ fixtureRefs: z.ZodArray<z.ZodObject<{
349
+ id: z.ZodString;
350
+ contentHash: z.ZodString;
351
+ }, z.core.$strict>>;
352
+ }, z.core.$strict>;
353
+ export declare const RewardBindingContentSchema: z.ZodObject<{
354
+ schemaVersion: z.ZodLiteral<"openpond.rewardBinding.v1">;
355
+ id: z.ZodString;
356
+ revision: z.ZodNumber;
357
+ name: z.ZodOptional<z.ZodString>;
358
+ description: z.ZodOptional<z.ZodString>;
359
+ recipeRef: z.ZodOptional<z.ZodObject<{
360
+ id: z.ZodString;
361
+ contentHash: z.ZodString;
362
+ revision: z.ZodNumber;
363
+ }, z.core.$strict>>;
364
+ sources: z.ZodArray<z.ZodObject<{
365
+ graderId: z.ZodString;
366
+ reward: z.ZodObject<{
367
+ id: z.ZodString;
368
+ contentHash: z.ZodString;
369
+ revision: z.ZodNumber;
370
+ }, z.core.$strict>;
371
+ role: z.ZodEnum<{
372
+ evaluation: "evaluation";
373
+ training: "training";
374
+ }>;
375
+ normalization: z.ZodDiscriminatedUnion<[z.ZodObject<{
376
+ kind: z.ZodLiteral<"identity">;
377
+ }, z.core.$strict>, z.ZodObject<{
378
+ kind: z.ZodLiteral<"linear">;
379
+ minimum: z.ZodNumber;
380
+ maximum: z.ZodNumber;
381
+ direction: z.ZodEnum<{
382
+ higher: "higher";
383
+ lower: "lower";
384
+ }>;
385
+ }, z.core.$strict>], "kind">;
386
+ weight: z.ZodNumber;
387
+ required: z.ZodBoolean;
388
+ hardGate: z.ZodBoolean;
389
+ privileged: z.ZodBoolean;
390
+ fixtureRefs: z.ZodArray<z.ZodObject<{
391
+ id: z.ZodString;
392
+ contentHash: z.ZodString;
393
+ }, z.core.$strict>>;
394
+ }, z.core.$strict>>;
395
+ aggregation: z.ZodLiteral<"weighted_mean">;
396
+ unscorable: z.ZodLiteral<"exclude_optional_require_all_required">;
397
+ }, z.core.$strict>;
398
+ export declare const RewardBindingSchema: z.ZodObject<{
399
+ schemaVersion: z.ZodLiteral<"openpond.rewardBinding.v1">;
400
+ id: z.ZodString;
401
+ revision: z.ZodNumber;
402
+ name: z.ZodOptional<z.ZodString>;
403
+ description: z.ZodOptional<z.ZodString>;
404
+ recipeRef: z.ZodOptional<z.ZodObject<{
405
+ id: z.ZodString;
406
+ contentHash: z.ZodString;
407
+ revision: z.ZodNumber;
408
+ }, z.core.$strict>>;
409
+ sources: z.ZodArray<z.ZodObject<{
410
+ graderId: z.ZodString;
411
+ reward: z.ZodObject<{
412
+ id: z.ZodString;
413
+ contentHash: z.ZodString;
414
+ revision: z.ZodNumber;
415
+ }, z.core.$strict>;
416
+ role: z.ZodEnum<{
417
+ evaluation: "evaluation";
418
+ training: "training";
419
+ }>;
420
+ normalization: z.ZodDiscriminatedUnion<[z.ZodObject<{
421
+ kind: z.ZodLiteral<"identity">;
422
+ }, z.core.$strict>, z.ZodObject<{
423
+ kind: z.ZodLiteral<"linear">;
424
+ minimum: z.ZodNumber;
425
+ maximum: z.ZodNumber;
426
+ direction: z.ZodEnum<{
427
+ higher: "higher";
428
+ lower: "lower";
429
+ }>;
430
+ }, z.core.$strict>], "kind">;
431
+ weight: z.ZodNumber;
432
+ required: z.ZodBoolean;
433
+ hardGate: z.ZodBoolean;
434
+ privileged: z.ZodBoolean;
435
+ fixtureRefs: z.ZodArray<z.ZodObject<{
436
+ id: z.ZodString;
437
+ contentHash: z.ZodString;
438
+ }, z.core.$strict>>;
439
+ }, z.core.$strict>>;
440
+ aggregation: z.ZodLiteral<"weighted_mean">;
441
+ unscorable: z.ZodLiteral<"exclude_optional_require_all_required">;
442
+ contentHash: z.ZodString;
443
+ }, z.core.$strict>;
444
+ export declare const BoundRewardResultSchema: z.ZodObject<{
445
+ graderId: z.ZodString;
446
+ reward: z.ZodObject<{
447
+ id: z.ZodString;
448
+ contentHash: z.ZodString;
449
+ revision: z.ZodNumber;
450
+ }, z.core.$strict>;
451
+ role: z.ZodEnum<{
452
+ evaluation: "evaluation";
453
+ training: "training";
454
+ }>;
455
+ status: z.ZodEnum<{
456
+ pending: "pending";
457
+ failed: "failed";
458
+ scored: "scored";
459
+ unavailable: "unavailable";
460
+ }>;
461
+ rawScore: z.ZodNullable<z.ZodNumber>;
462
+ normalizedScore: z.ZodNullable<z.ZodNumber>;
463
+ passed: z.ZodNullable<z.ZodBoolean>;
464
+ evidenceHashes: z.ZodArray<z.ZodString>;
465
+ message: z.ZodNullable<z.ZodString>;
466
+ }, z.core.$strict>;
467
+ export declare const RewardCompositionContentSchema: z.ZodObject<{
468
+ schemaVersion: z.ZodLiteral<"openpond.rewardComposition.v1">;
469
+ binding: z.ZodObject<{
470
+ id: z.ZodString;
471
+ contentHash: z.ZodString;
472
+ revision: z.ZodNumber;
473
+ }, z.core.$strict>;
474
+ taskHash: z.ZodString;
475
+ outputHash: z.ZodString;
476
+ results: z.ZodArray<z.ZodObject<{
477
+ graderId: z.ZodString;
478
+ reward: z.ZodObject<{
479
+ id: z.ZodString;
480
+ contentHash: z.ZodString;
481
+ revision: z.ZodNumber;
482
+ }, z.core.$strict>;
483
+ role: z.ZodEnum<{
484
+ evaluation: "evaluation";
485
+ training: "training";
486
+ }>;
487
+ status: z.ZodEnum<{
488
+ pending: "pending";
489
+ failed: "failed";
490
+ scored: "scored";
491
+ unavailable: "unavailable";
492
+ }>;
493
+ rawScore: z.ZodNullable<z.ZodNumber>;
494
+ normalizedScore: z.ZodNullable<z.ZodNumber>;
495
+ passed: z.ZodNullable<z.ZodBoolean>;
496
+ evidenceHashes: z.ZodArray<z.ZodString>;
497
+ message: z.ZodNullable<z.ZodString>;
498
+ }, z.core.$strict>>;
499
+ training: z.ZodObject<{
500
+ status: z.ZodEnum<{
501
+ scored: "scored";
502
+ unscorable: "unscorable";
503
+ not_configured: "not_configured";
504
+ }>;
505
+ score: z.ZodNullable<z.ZodNumber>;
506
+ passed: z.ZodNullable<z.ZodBoolean>;
507
+ }, z.core.$strict>;
508
+ evaluation: z.ZodObject<{
509
+ status: z.ZodEnum<{
510
+ scored: "scored";
511
+ unscorable: "unscorable";
512
+ not_configured: "not_configured";
513
+ }>;
514
+ score: z.ZodNullable<z.ZodNumber>;
515
+ passed: z.ZodNullable<z.ZodBoolean>;
516
+ }, z.core.$strict>;
517
+ }, z.core.$strict>;
518
+ export declare const RewardCompositionSchema: z.ZodObject<{
519
+ schemaVersion: z.ZodLiteral<"openpond.rewardComposition.v1">;
520
+ binding: z.ZodObject<{
521
+ id: z.ZodString;
522
+ contentHash: z.ZodString;
523
+ revision: z.ZodNumber;
524
+ }, z.core.$strict>;
525
+ taskHash: z.ZodString;
526
+ outputHash: z.ZodString;
527
+ results: z.ZodArray<z.ZodObject<{
528
+ graderId: z.ZodString;
529
+ reward: z.ZodObject<{
530
+ id: z.ZodString;
531
+ contentHash: z.ZodString;
532
+ revision: z.ZodNumber;
533
+ }, z.core.$strict>;
534
+ role: z.ZodEnum<{
535
+ evaluation: "evaluation";
536
+ training: "training";
537
+ }>;
538
+ status: z.ZodEnum<{
539
+ pending: "pending";
540
+ failed: "failed";
541
+ scored: "scored";
542
+ unavailable: "unavailable";
543
+ }>;
544
+ rawScore: z.ZodNullable<z.ZodNumber>;
545
+ normalizedScore: z.ZodNullable<z.ZodNumber>;
546
+ passed: z.ZodNullable<z.ZodBoolean>;
547
+ evidenceHashes: z.ZodArray<z.ZodString>;
548
+ message: z.ZodNullable<z.ZodString>;
549
+ }, z.core.$strict>>;
550
+ training: z.ZodObject<{
551
+ status: z.ZodEnum<{
552
+ scored: "scored";
553
+ unscorable: "unscorable";
554
+ not_configured: "not_configured";
555
+ }>;
556
+ score: z.ZodNullable<z.ZodNumber>;
557
+ passed: z.ZodNullable<z.ZodBoolean>;
558
+ }, z.core.$strict>;
559
+ evaluation: z.ZodObject<{
560
+ status: z.ZodEnum<{
561
+ scored: "scored";
562
+ unscorable: "unscorable";
563
+ not_configured: "not_configured";
564
+ }>;
565
+ score: z.ZodNullable<z.ZodNumber>;
566
+ passed: z.ZodNullable<z.ZodBoolean>;
567
+ }, z.core.$strict>;
568
+ contentHash: z.ZodString;
569
+ }, z.core.$strict>;
570
+ export type RewardRelease = z.infer<typeof RewardReleaseSchema>;
571
+ export type RewardBinding = z.infer<typeof RewardBindingSchema>;
572
+ export type RewardComposition = z.infer<typeof RewardCompositionSchema>;
573
+ export type BoundRewardResult = z.infer<typeof BoundRewardResultSchema>;
574
+ export type RewardBindingSource = z.infer<typeof RewardBindingSourceSchema>;
575
+ export type LearnedRewardRunner = (input: {
576
+ reward: RewardRelease;
577
+ task: TaskRecord;
578
+ evidence: AttemptEvidence;
579
+ signal?: AbortSignal;
580
+ }) => Promise<{
581
+ rawScore: number;
582
+ passed: boolean;
583
+ evidenceHashes: string[];
584
+ }>;
585
+ export declare function createRewardRelease(input: z.input<typeof RewardReleaseContentSchema>): RewardRelease;
586
+ export declare function createRewardBinding(input: z.input<typeof RewardBindingContentSchema>, rewards: RewardRelease[]): RewardBinding;
587
+ export declare function resolveBoundRewards(input: RewardBinding, rewards: RewardRelease[]): Array<{
588
+ source: RewardBindingSource;
589
+ reward: RewardRelease;
590
+ }>;
591
+ export declare function compileBoundGraders(binding: RewardBinding, rewards: RewardRelease[]): GraderSpec[];
592
+ export declare function executeRewardBinding(input: {
593
+ binding: RewardBinding;
594
+ rewards: RewardRelease[];
595
+ task: TaskRecord;
596
+ evidence: AttemptEvidence;
597
+ modelJudge?: ModelJudgeRunner;
598
+ customVerifier?: CustomVerifierRunner;
599
+ learnedReward?: LearnedRewardRunner;
600
+ signal?: AbortSignal;
601
+ }): Promise<RewardComposition>;
602
+ export declare function composeBoundRewards(input: {
603
+ binding: RewardBinding;
604
+ taskHash: string;
605
+ outputHash: string;
606
+ results: BoundRewardResult[];
607
+ }): RewardComposition;
608
+ export declare function assertRewardHash(value: {
609
+ contentHash: string;
610
+ }): void;
611
+ //# sourceMappingURL=rewards.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rewards.d.ts","sourceRoot":"","sources":["../../../../src/rewards.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAiB,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrH,OAAO,EAAsI,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AAIrM,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAMrC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgBrC,CAAC;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAqF,CAAC;AACtH,eAAO,MAAM,sBAAsB;;;;kBAAuF,CAAC;AAE3H,eAAO,MAAM,yBAAyB;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU3B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiBrC,CAAC;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAqF,CAAC;AAEtH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAQhC,CAAC;AACZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAqF,CAAC;AAE1H,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,KAAK,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC;AAEpN,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,GAAG,aAAa,CAIpG;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,aAAa,CAK9H;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;IAAE,MAAM,EAAE,mBAAmB,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,CAAC,CAajJ;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,UAAU,EAAE,CAKlG;AAED,wBAAsB,oBAAoB,CAAC,KAAK,EAAE;IAChD,OAAO,EAAE,aAAa,CAAC;IACvB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkC7B;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,OAAO,EAAE,aAAa,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAAE,GAAG,iBAAiB,CAsC5J;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAGrE"}
@@ -0,0 +1,18 @@
1
+ export declare const TASK_JSON_SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema";
2
+ export declare const TASK_VALUE_MAX_BYTES = 1048576;
3
+ export declare const TASK_SCHEMA_MAX_BYTES = 32768;
4
+ export type TaskSchemaIssue = {
5
+ path: string;
6
+ code: string;
7
+ message: string;
8
+ };
9
+ export type TaskSchemaResult = {
10
+ valid: boolean;
11
+ issues: TaskSchemaIssue[];
12
+ };
13
+ /** Bound producer-controlled JSON before hashing, schema compilation or validation. */
14
+ export declare function assertBoundedTaskJson(value: unknown, maxBytes?: number): void;
15
+ /** No remote resolution, custom code, data mutation, regex or format plugins. */
16
+ export declare function validateTaskSchema(schema: unknown): TaskSchemaResult;
17
+ export declare function validateTaskValue(schema: unknown, value: unknown): TaskSchemaResult;
18
+ //# sourceMappingURL=task-schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task-schema.d.ts","sourceRoot":"","sources":["../../../../src/task-schema.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,wBAAwB,iDAAiD,CAAC;AACvF,eAAO,MAAM,oBAAoB,UAAY,CAAC;AAC9C,eAAO,MAAM,qBAAqB,QAAS,CAAC;AAE5C,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAC9E,MAAM,MAAM,gBAAgB,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,eAAe,EAAE,CAAA;CAAE,CAAC;AAQ7E,uFAAuF;AACvF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,SAAuB,GAAG,IAAI,CA0B3F;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAOpE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAgBnF"}