npm - @openpond/evals - Versions diffs - 0.5.0 → 0.7.0 - Mend

@openpond/evals 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (118) hide show

package/CONTRACT.md +13 -1
package/LEARNING.md +106 -0
package/README.md +28 -2
package/conformance/telemetry/v1/invalid-batch.json +17 -0
package/conformance/telemetry/v1/valid-batch.json +58 -0
package/dist/graders.js +26 -5
package/dist/index.js +8 -0
package/dist/javascript-verifier-contract.js +7 -0
package/dist/javascript-verifier-node.js +56 -0
package/dist/javascript-verifier-worker-source.js +1 -0
package/dist/javascript-verifier-worker.js +6 -0
package/dist/javascript-verifier.js +64 -0
package/dist/learned-preference.js +334 -0
package/dist/learning/admission.js +209 -0
package/dist/learning/assets.js +37 -0
package/dist/learning/contracts.js +267 -0
package/dist/learning/errors.js +12 -0
package/dist/learning/grade-worker.js +99 -0
package/dist/learning/index.js +9 -0
package/dist/learning/operations.js +33 -0
package/dist/learning/repository.js +48 -0
package/dist/learning/service.js +367 -0
package/dist/learning/transport.js +25 -0
package/dist/preferences.js +77 -11
package/dist/rewards.js +230 -0
package/dist/task-schema.js +142 -0
package/dist/tasksets.js +3 -0
package/dist/telemetry/index.js +4 -0
package/dist/telemetry-analysis.js +183 -0
package/dist/telemetry-bundle.js +60 -0
package/dist/telemetry-catalog.js +50 -0
package/dist/telemetry.js +112 -0
package/dist/types/builtin-benchmarks/harness-refiner.d.ts +7 -0
package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
package/dist/types/conformance.d.ts +14 -0
package/dist/types/conformance.d.ts.map +1 -1
package/dist/types/execution-contracts.d.ts +14 -0
package/dist/types/execution-contracts.d.ts.map +1 -1
package/dist/types/graders.d.ts.map +1 -1
package/dist/types/index.d.ts +8 -0
package/dist/types/index.d.ts.map +1 -1
package/dist/types/javascript-verifier-contract.d.ts +9 -0
package/dist/types/javascript-verifier-contract.d.ts.map +1 -0
package/dist/types/javascript-verifier-node.d.ts +5 -0
package/dist/types/javascript-verifier-node.d.ts.map +1 -0
package/dist/types/javascript-verifier-worker-source.d.ts +2 -0
package/dist/types/javascript-verifier-worker.d.ts +2 -0
package/dist/types/javascript-verifier-worker.d.ts.map +1 -0
package/dist/types/javascript-verifier.d.ts +15 -0
package/dist/types/javascript-verifier.d.ts.map +1 -0
package/dist/types/learned-preference.d.ts +282 -0
package/dist/types/learned-preference.d.ts.map +1 -0
package/dist/types/learning/admission.d.ts +335 -0
package/dist/types/learning/admission.d.ts.map +1 -0
package/dist/types/learning/assets.d.ts +49 -0
package/dist/types/learning/assets.d.ts.map +1 -0
package/dist/types/learning/contracts.d.ts +2149 -0
package/dist/types/learning/contracts.d.ts.map +1 -0
package/dist/types/learning/errors.d.ts +8 -0
package/dist/types/learning/errors.d.ts.map +1 -0
package/dist/types/learning/grade-worker.d.ts +31 -0
package/dist/types/learning/grade-worker.d.ts.map +1 -0
package/dist/types/learning/index.d.ts +10 -0
package/dist/types/learning/index.d.ts.map +1 -0
package/dist/types/learning/operations.d.ts +2061 -0
package/dist/types/learning/operations.d.ts.map +1 -0
package/dist/types/learning/repository.d.ts +1143 -0
package/dist/types/learning/repository.d.ts.map +1 -0
package/dist/types/learning/service.d.ts +22 -0
package/dist/types/learning/service.d.ts.map +1 -0
package/dist/types/learning/transport.d.ts +2139 -0
package/dist/types/learning/transport.d.ts.map +1 -0
package/dist/types/preferences.d.ts +55 -7
package/dist/types/preferences.d.ts.map +1 -1
package/dist/types/review-conformance.d.ts +5 -5
package/dist/types/rewards.d.ts +611 -0
package/dist/types/rewards.d.ts.map +1 -0
package/dist/types/task-schema.d.ts +18 -0
package/dist/types/task-schema.d.ts.map +1 -0
package/dist/types/tasksets.d.ts +28 -0
package/dist/types/tasksets.d.ts.map +1 -1
package/dist/types/telemetry/index.d.ts +5 -0
package/dist/types/telemetry/index.d.ts.map +1 -0
package/dist/types/telemetry-analysis.d.ts +116 -0
package/dist/types/telemetry-analysis.d.ts.map +1 -0
package/dist/types/telemetry-bundle.d.ts +309 -0
package/dist/types/telemetry-bundle.d.ts.map +1 -0
package/dist/types/telemetry-catalog.d.ts +269 -0
package/dist/types/telemetry-catalog.d.ts.map +1 -0
package/dist/types/telemetry.d.ts +266 -0
package/dist/types/telemetry.d.ts.map +1 -0
package/package.json +40 -3
package/schemas/learning/v1/asset.schema.json +89 -0
package/schemas/learning/v1/batch.schema.json +132 -0
package/schemas/learning/v1/binding.schema.json +207 -0
package/schemas/learning/v1/command-request.schema.json +2414 -0
package/schemas/learning/v1/decision.schema.json +440 -0
package/schemas/learning/v1/definition.schema.json +397 -0
package/schemas/learning/v1/evidence.schema.json +317 -0
package/schemas/learning/v1/example-submission.schema.json +253 -0
package/schemas/learning/v1/feedback-submission.schema.json +114 -0
package/schemas/learning/v1/feedback.schema.json +233 -0
package/schemas/learning/v1/grade.schema.json +412 -0
package/schemas/learning/v1/iteration.schema.json +236 -0
package/schemas/learning/v1/package.schema.json +776 -0
package/schemas/learning/v1/policy.schema.json +373 -0
package/schemas/learning/v1/read-request.schema.json +101 -0
package/schemas/learning/v1/reward.schema.json +302 -0
package/schemas/learning/v1/source.schema.json +198 -0
package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0

package/schemas/telemetry/v1/metric-definition.schema.json ADDED Viewed

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.metricDefinition.v1"
+    },
+    "id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "displayName": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 200
+    },
+    "description": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 2000
+    },
+    "valueType": {
+      "type": "string",
+      "enum": [
+        "gauge",
+        "counter",
+        "distribution"
+      ]
+    },
+    "unit": {
+      "type": "string",
+      "enum": [
+        "ratio",
+        "count",
+        "seconds",
+        "milliseconds",
+        "tokens",
+        "bytes",
+        "usd",
+        "scalar"
+      ]
+    },
+    "direction": {
+      "type": "string",
+      "enum": [
+        "higher",
+        "lower",
+        "neutral"
+      ]
+    },
+    "aggregation": {
+      "type": "string",
+      "enum": [
+        "last",
+        "sum",
+        "mean",
+        "min",
+        "max",
+        "p50",
+        "p95"
+      ]
+    },
+    "visibility": {
+      "type": "string",
+      "enum": [
+        "policy_visible",
+        "team_visible",
+        "host_private"
+      ]
+    },
+    "boundedDimensions": {
+      "maxItems": 32,
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "displayName",
+    "description",
+    "valueType",
+    "unit",
+    "direction",
+    "aggregation",
+    "visibility",
+    "boundedDimensions"
+  ],
+  "additionalProperties": false
+}

package/schemas/telemetry/v1/metric-observation.schema.json ADDED Viewed

@@ -0,0 +1,182 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.metricObservation.v1"
+    },
+    "observationId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "metricId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "eventId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "sequence": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "observedAt": {
+      "type": "string",
+      "format": "date-time",
+      "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+    },
+    "value": {
+      "type": "number"
+    },
+    "lineage": {
+      "type": "object",
+      "properties": {
+        "modelProjectId": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 240
+        },
+        "runId": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 240
+        },
+        "modelVersionId": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "harnessReleaseHash": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "tasksetReleaseHash": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "environmentReleaseHash": {
+          "anyOf": [
+            {
+              "type": "string",
+              "pattern": "^[a-f0-9]{64}$"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "checkpointId": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "step": {
+          "anyOf": [
+            {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 9007199254740991
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "rolloutGroupId": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "attemptId": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "scenarioId": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            {
+              "type": "null"
+            }
+          ]
+        }
+      },
+      "required": [
+        "modelProjectId",
+        "runId",
+        "modelVersionId",
+        "harnessReleaseHash",
+        "tasksetReleaseHash",
+        "environmentReleaseHash",
+        "checkpointId",
+        "step",
+        "rolloutGroupId",
+        "attemptId",
+        "scenarioId"
+      ],
+      "additionalProperties": false
+    },
+    "dimensions": {
+      "type": "object",
+      "propertyNames": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      },
+      "additionalProperties": {
+        "type": "string",
+        "maxLength": 500
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "observationId",
+    "metricId",
+    "eventId",
+    "sequence",
+    "observedAt",
+    "value",
+    "lineage",
+    "dimensions"
+  ],
+  "additionalProperties": false
+}

package/schemas/telemetry/v1/run-metric-summary.schema.json ADDED Viewed

@@ -0,0 +1,98 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.runMetricSummary.v1"
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "metricId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "aggregation": {
+      "type": "string",
+      "enum": [
+        "last",
+        "sum",
+        "mean",
+        "min",
+        "max",
+        "p50",
+        "p95"
+      ]
+    },
+    "value": {
+      "anyOf": [
+        {
+          "type": "number"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "sampleCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "series": {
+      "maxItems": 100000,
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "step": {
+            "anyOf": [
+              {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "observedAt": {
+            "type": "string",
+            "format": "date-time",
+            "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+          },
+          "value": {
+            "type": "number"
+          },
+          "sampleCount": {
+            "type": "integer",
+            "exclusiveMinimum": 0,
+            "maximum": 9007199254740991
+          }
+        },
+        "required": [
+          "step",
+          "observedAt",
+          "value",
+          "sampleCount"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "runId",
+    "metricId",
+    "aggregation",
+    "value",
+    "sampleCount",
+    "series"
+  ],
+  "additionalProperties": false
+}