@openpond/evals 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +36 -0
  2. package/dist/index.js +1 -0
  3. package/dist/metric-policy.js +50 -0
  4. package/dist/metrics-node.js +16 -0
  5. package/dist/metrics.js +105 -0
  6. package/dist/runs.js +12 -0
  7. package/dist/tasksets.js +5 -0
  8. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +14 -0
  9. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  10. package/dist/types/conformance.d.ts +28 -0
  11. package/dist/types/conformance.d.ts.map +1 -1
  12. package/dist/types/evidence/conformance.d.ts +4 -4
  13. package/dist/types/evidence/contracts.d.ts +8 -8
  14. package/dist/types/index.d.ts +1 -0
  15. package/dist/types/index.d.ts.map +1 -1
  16. package/dist/types/javascript-isolate.d.ts +1 -1
  17. package/dist/types/javascript-isolate.d.ts.map +1 -1
  18. package/dist/types/learning/admission.d.ts +3 -3
  19. package/dist/types/learning/contracts.d.ts +3 -3
  20. package/dist/types/learning/operations.d.ts +12 -12
  21. package/dist/types/learning/repository.d.ts +25 -3
  22. package/dist/types/learning/repository.d.ts.map +1 -1
  23. package/dist/types/learning/transport.d.ts +31 -9
  24. package/dist/types/learning/transport.d.ts.map +1 -1
  25. package/dist/types/metric-policy.d.ts +112 -0
  26. package/dist/types/metric-policy.d.ts.map +1 -0
  27. package/dist/types/metrics-node.d.ts +8 -0
  28. package/dist/types/metrics-node.d.ts.map +1 -0
  29. package/dist/types/metrics.d.ts +37 -0
  30. package/dist/types/metrics.d.ts.map +1 -0
  31. package/dist/types/preferences.d.ts +3 -3
  32. package/dist/types/rewards.d.ts +9 -9
  33. package/dist/types/runs.d.ts +88 -0
  34. package/dist/types/runs.d.ts.map +1 -1
  35. package/dist/types/tasksets.d.ts +44 -0
  36. package/dist/types/tasksets.d.ts.map +1 -1
  37. package/package.json +9 -1
  38. package/schemas/learning/v1/package.schema.json +108 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openpond/evals",
3
- "version": "0.8.1",
3
+ "version": "0.9.0",
4
4
  "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -19,6 +19,14 @@
19
19
  "LICENSE"
20
20
  ],
21
21
  "exports": {
22
+ "./metrics/node": {
23
+ "types": "./dist/types/metrics-node.d.ts",
24
+ "import": "./dist/metrics-node.js"
25
+ },
26
+ "./metrics": {
27
+ "types": "./dist/types/metrics.d.ts",
28
+ "import": "./dist/metrics.js"
29
+ },
22
30
  "./sql-execution/node": {
23
31
  "types": "./dist/types/sql-execution-node.d.ts",
24
32
  "import": "./dist/sql-execution-node.js"
package/schemas/learning/v1/package.schema.json CHANGED
@@ -181,6 +181,37 @@
181
181
  ],
182
182
  "additionalProperties": false
183
183
  },
184
+ "metrics": {
185
+ "type": "object",
186
+ "properties": {
187
+ "schemaVersion": {
188
+ "$ref": "#/$defs/__schema16"
189
+ },
190
+ "primaryMetric": {
191
+ "$ref": "#/$defs/__schema17"
192
+ },
193
+ "aggregation": {
194
+ "$ref": "#/$defs/__schema18"
195
+ },
196
+ "missingReward": {
197
+ "$ref": "#/$defs/__schema19"
198
+ },
199
+ "taskWeights": {
200
+ "$ref": "#/$defs/__schema20"
201
+ },
202
+ "customAggregator": {
203
+ "$ref": "#/$defs/__schema21"
204
+ }
205
+ },
206
+ "required": [
207
+ "schemaVersion",
208
+ "primaryMetric",
209
+ "aggregation",
210
+ "missingReward",
211
+ "customAggregator"
212
+ ],
213
+ "additionalProperties": false
214
+ },
184
215
  "metadata": {
185
216
  "$ref": "#/$defs/__schema9"
186
217
  },
@@ -769,6 +800,83 @@
769
800
  "__schema15": {
770
801
  "default": false,
771
802
  "type": "boolean"
803
+ },
804
+ "__schema16": {
805
+ "type": "string",
806
+ "const": "openpond.tasksetMetricPolicy.v1"
807
+ },
808
+ "__schema17": {
809
+ "type": "string",
810
+ "minLength": 1,
811
+ "maxLength": 240
812
+ },
813
+ "__schema18": {
814
+ "type": "string",
815
+ "enum": [
816
+ "mean_score",
817
+ "pass_rate",
818
+ "weighted_mean",
819
+ "custom"
820
+ ]
821
+ },
822
+ "__schema19": {
823
+ "type": "string",
824
+ "enum": [
825
+ "zero",
826
+ "exclude"
827
+ ]
828
+ },
829
+ "__schema20": {
830
+ "type": "object",
831
+ "propertyNames": {
832
+ "$ref": "#/$defs/__schema17"
833
+ },
834
+ "additionalProperties": {
835
+ "type": "number",
836
+ "exclusiveMinimum": 0,
837
+ "maximum": 1000000
838
+ }
839
+ },
840
+ "__schema21": {
841
+ "anyOf": [
842
+ {
843
+ "type": "object",
844
+ "properties": {
845
+ "module": {
846
+ "type": "string",
847
+ "minLength": 1,
848
+ "maxLength": 1000
849
+ },
850
+ "exportName": {
851
+ "type": "string",
852
+ "pattern": "^[A-Za-z_$][A-Za-z0-9_$]*$"
853
+ },
854
+ "contentHash": {
855
+ "$ref": "#/$defs/__schema2"
856
+ },
857
+ "timeoutMs": {
858
+ "type": "integer",
859
+ "exclusiveMinimum": 0,
860
+ "maximum": 300000
861
+ },
862
+ "networkPolicy": {
863
+ "type": "string",
864
+ "const": "none"
865
+ }
866
+ },
867
+ "required": [
868
+ "module",
869
+ "exportName",
870
+ "contentHash",
871
+ "timeoutMs",
872
+ "networkPolicy"
873
+ ],
874
+ "additionalProperties": false
875
+ },
876
+ {
877
+ "type": "null"
878
+ }
879
+ ]
772
880
  }
773
881
  },
774
882
  "$id": "urn:openpond:learning:v1:package",