@openpond/evals 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CONTRACT.md +27 -1
  2. package/README.md +31 -2
  3. package/conformance/telemetry/v1/invalid-batch.json +17 -0
  4. package/conformance/telemetry/v1/valid-batch.json +58 -0
  5. package/dist/artifact-verification.js +172 -0
  6. package/dist/builtin-benchmarks/harness-refiner.js +469 -24
  7. package/dist/execution-contracts.js +153 -0
  8. package/dist/execution-receipts.js +185 -0
  9. package/dist/graders.js +8 -2
  10. package/dist/index.js +10 -0
  11. package/dist/learned-preference.js +334 -0
  12. package/dist/preferences.js +803 -0
  13. package/dist/rollouts.js +241 -0
  14. package/dist/tasksets.js +24 -0
  15. package/dist/telemetry/index.js +4 -0
  16. package/dist/telemetry-analysis.js +183 -0
  17. package/dist/telemetry-bundle.js +60 -0
  18. package/dist/telemetry-catalog.js +50 -0
  19. package/dist/telemetry.js +112 -0
  20. package/dist/types/artifact-verification.d.ts +31 -0
  21. package/dist/types/artifact-verification.d.ts.map +1 -0
  22. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +22 -0
  23. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  24. package/dist/types/conformance.d.ts +44 -0
  25. package/dist/types/conformance.d.ts.map +1 -1
  26. package/dist/types/evidence/conformance.d.ts +24 -24
  27. package/dist/types/evidence/contracts.d.ts +42 -42
  28. package/dist/types/execution-contracts.d.ts +806 -0
  29. package/dist/types/execution-contracts.d.ts.map +1 -0
  30. package/dist/types/execution-receipts.d.ts +55 -0
  31. package/dist/types/execution-receipts.d.ts.map +1 -0
  32. package/dist/types/graders.d.ts.map +1 -1
  33. package/dist/types/index.d.ts +10 -0
  34. package/dist/types/index.d.ts.map +1 -1
  35. package/dist/types/learned-preference.d.ts +282 -0
  36. package/dist/types/learned-preference.d.ts.map +1 -0
  37. package/dist/types/preferences.d.ts +583 -0
  38. package/dist/types/preferences.d.ts.map +1 -0
  39. package/dist/types/review-conformance.d.ts +20 -15
  40. package/dist/types/review-conformance.d.ts.map +1 -1
  41. package/dist/types/rollouts.d.ts +221 -0
  42. package/dist/types/rollouts.d.ts.map +1 -0
  43. package/dist/types/tasksets.d.ts +89 -0
  44. package/dist/types/tasksets.d.ts.map +1 -1
  45. package/dist/types/telemetry/index.d.ts +5 -0
  46. package/dist/types/telemetry/index.d.ts.map +1 -0
  47. package/dist/types/telemetry-analysis.d.ts +116 -0
  48. package/dist/types/telemetry-analysis.d.ts.map +1 -0
  49. package/dist/types/telemetry-bundle.d.ts +309 -0
  50. package/dist/types/telemetry-bundle.d.ts.map +1 -0
  51. package/dist/types/telemetry-catalog.d.ts +269 -0
  52. package/dist/types/telemetry-catalog.d.ts.map +1 -0
  53. package/dist/types/telemetry.d.ts +266 -0
  54. package/dist/types/telemetry.d.ts.map +1 -0
  55. package/package.json +20 -2
  56. package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
  57. package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
  58. package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
  59. package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
  60. package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
  61. package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
  62. package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
  63. package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
  64. package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0
package/schemas/telemetry/v1/run-telemetry-event.schema.json ADDED
@@ -0,0 +1,201 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "properties": {
5
+ "schemaVersion": {
6
+ "type": "string",
7
+ "const": "openpond.runTelemetryEvent.v1"
8
+ },
9
+ "eventId": {
10
+ "type": "string",
11
+ "minLength": 1,
12
+ "maxLength": 240
13
+ },
14
+ "sequence": {
15
+ "type": "integer",
16
+ "minimum": 0,
17
+ "maximum": 9007199254740991
18
+ },
19
+ "occurredAt": {
20
+ "type": "string",
21
+ "format": "date-time",
22
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
23
+ },
24
+ "source": {
25
+ "type": "string",
26
+ "enum": [
27
+ "runtime",
28
+ "environment",
29
+ "grader",
30
+ "optimizer",
31
+ "control_plane",
32
+ "evaluation"
33
+ ]
34
+ },
35
+ "type": {
36
+ "type": "string",
37
+ "enum": [
38
+ "run_started",
39
+ "run_state_changed",
40
+ "rollout_group_started",
41
+ "attempt_completed",
42
+ "grader_completed",
43
+ "reward_composed",
44
+ "optimizer_step_completed",
45
+ "checkpoint_committed",
46
+ "evaluation_completed",
47
+ "run_completed",
48
+ "run_failed",
49
+ "cleanup_completed"
50
+ ]
51
+ },
52
+ "visibility": {
53
+ "type": "string",
54
+ "enum": [
55
+ "policy_visible",
56
+ "team_visible",
57
+ "host_private"
58
+ ]
59
+ },
60
+ "lineage": {
61
+ "type": "object",
62
+ "properties": {
63
+ "modelProjectId": {
64
+ "type": "string",
65
+ "minLength": 1,
66
+ "maxLength": 240
67
+ },
68
+ "runId": {
69
+ "type": "string",
70
+ "minLength": 1,
71
+ "maxLength": 240
72
+ },
73
+ "modelVersionId": {
74
+ "anyOf": [
75
+ {
76
+ "type": "string",
77
+ "minLength": 1,
78
+ "maxLength": 240
79
+ },
80
+ {
81
+ "type": "null"
82
+ }
83
+ ]
84
+ },
85
+ "harnessReleaseHash": {
86
+ "type": "string",
87
+ "pattern": "^[a-f0-9]{64}$"
88
+ },
89
+ "tasksetReleaseHash": {
90
+ "type": "string",
91
+ "pattern": "^[a-f0-9]{64}$"
92
+ },
93
+ "environmentReleaseHash": {
94
+ "anyOf": [
95
+ {
96
+ "type": "string",
97
+ "pattern": "^[a-f0-9]{64}$"
98
+ },
99
+ {
100
+ "type": "null"
101
+ }
102
+ ]
103
+ },
104
+ "checkpointId": {
105
+ "anyOf": [
106
+ {
107
+ "type": "string",
108
+ "minLength": 1,
109
+ "maxLength": 240
110
+ },
111
+ {
112
+ "type": "null"
113
+ }
114
+ ]
115
+ },
116
+ "step": {
117
+ "anyOf": [
118
+ {
119
+ "type": "integer",
120
+ "minimum": 0,
121
+ "maximum": 9007199254740991
122
+ },
123
+ {
124
+ "type": "null"
125
+ }
126
+ ]
127
+ },
128
+ "rolloutGroupId": {
129
+ "anyOf": [
130
+ {
131
+ "type": "string",
132
+ "minLength": 1,
133
+ "maxLength": 240
134
+ },
135
+ {
136
+ "type": "null"
137
+ }
138
+ ]
139
+ },
140
+ "attemptId": {
141
+ "anyOf": [
142
+ {
143
+ "type": "string",
144
+ "minLength": 1,
145
+ "maxLength": 240
146
+ },
147
+ {
148
+ "type": "null"
149
+ }
150
+ ]
151
+ },
152
+ "scenarioId": {
153
+ "anyOf": [
154
+ {
155
+ "type": "string",
156
+ "minLength": 1,
157
+ "maxLength": 240
158
+ },
159
+ {
160
+ "type": "null"
161
+ }
162
+ ]
163
+ }
164
+ },
165
+ "required": [
166
+ "modelProjectId",
167
+ "runId",
168
+ "modelVersionId",
169
+ "harnessReleaseHash",
170
+ "tasksetReleaseHash",
171
+ "environmentReleaseHash",
172
+ "checkpointId",
173
+ "step",
174
+ "rolloutGroupId",
175
+ "attemptId",
176
+ "scenarioId"
177
+ ],
178
+ "additionalProperties": false
179
+ },
180
+ "attributes": {
181
+ "default": {},
182
+ "type": "object",
183
+ "propertyNames": {
184
+ "type": "string"
185
+ },
186
+ "additionalProperties": {}
187
+ }
188
+ },
189
+ "required": [
190
+ "schemaVersion",
191
+ "eventId",
192
+ "sequence",
193
+ "occurredAt",
194
+ "source",
195
+ "type",
196
+ "visibility",
197
+ "lineage",
198
+ "attributes"
199
+ ],
200
+ "additionalProperties": false
201
+ }
package/schemas/telemetry/v1/telemetry-cohort.schema.json ADDED
@@ -0,0 +1,100 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "properties": {
5
+ "checkpointIds": {
6
+ "maxItems": 1000,
7
+ "type": "array",
8
+ "items": {
9
+ "type": "string",
10
+ "minLength": 1,
11
+ "maxLength": 240
12
+ }
13
+ },
14
+ "steps": {
15
+ "maxItems": 10000,
16
+ "type": "array",
17
+ "items": {
18
+ "type": "integer",
19
+ "minimum": 0,
20
+ "maximum": 9007199254740991
21
+ }
22
+ },
23
+ "scenarioIds": {
24
+ "maxItems": 100000,
25
+ "type": "array",
26
+ "items": {
27
+ "type": "string",
28
+ "minLength": 1,
29
+ "maxLength": 240
30
+ }
31
+ },
32
+ "rolloutGroupIds": {
33
+ "maxItems": 100000,
34
+ "type": "array",
35
+ "items": {
36
+ "type": "string",
37
+ "minLength": 1,
38
+ "maxLength": 240
39
+ }
40
+ },
41
+ "attemptIds": {
42
+ "maxItems": 1000000,
43
+ "type": "array",
44
+ "items": {
45
+ "type": "string",
46
+ "minLength": 1,
47
+ "maxLength": 240
48
+ }
49
+ },
50
+ "splits": {
51
+ "maxItems": 32,
52
+ "type": "array",
53
+ "items": {
54
+ "type": "string",
55
+ "minLength": 1,
56
+ "maxLength": 100
57
+ }
58
+ },
59
+ "failureOwners": {
60
+ "maxItems": 32,
61
+ "type": "array",
62
+ "items": {
63
+ "type": "string",
64
+ "minLength": 1,
65
+ "maxLength": 100
66
+ }
67
+ },
68
+ "graders": {
69
+ "maxItems": 1000,
70
+ "type": "array",
71
+ "items": {
72
+ "type": "string",
73
+ "minLength": 1,
74
+ "maxLength": 200
75
+ }
76
+ },
77
+ "rewardEligible": {
78
+ "anyOf": [
79
+ {
80
+ "type": "boolean"
81
+ },
82
+ {
83
+ "type": "null"
84
+ }
85
+ ]
86
+ }
87
+ },
88
+ "required": [
89
+ "checkpointIds",
90
+ "steps",
91
+ "scenarioIds",
92
+ "rolloutGroupIds",
93
+ "attemptIds",
94
+ "splits",
95
+ "failureOwners",
96
+ "graders",
97
+ "rewardEligible"
98
+ ],
99
+ "additionalProperties": false
100
+ }