@openpond/evals 0.11.21 → 0.12.1

package/dist/types/telemetry.d.ts

@@ -8,26 +8,26 @@ export declare const TelemetryVisibilitySchema: z.ZodEnum<{
     team_visible: "team_visible";
 }>;
 export declare const TelemetrySourceSchema: z.ZodEnum<{
-    grader: "grader";
-    runtime: "runtime";
+    control_plane: "control_plane";
     environment: "environment";
     evaluation: "evaluation";
+    grader: "grader";
     optimizer: "optimizer";
-    control_plane: "control_plane";
+    runtime: "runtime";
 }>;
 export declare const TelemetryEventTypeSchema: z.ZodEnum<{
-    cleanup_completed: "cleanup_completed";
-    run_started: "run_started";
-    run_state_changed: "run_state_changed";
-    rollout_group_started: "rollout_group_started";
     attempt_completed: "attempt_completed";
-    grader_completed: "grader_completed";
-    reward_composed: "reward_composed";
-    optimizer_step_completed: "optimizer_step_completed";
     checkpoint_committed: "checkpoint_committed";
+    cleanup_completed: "cleanup_completed";
     evaluation_completed: "evaluation_completed";
+    grader_completed: "grader_completed";
+    optimizer_step_completed: "optimizer_step_completed";
+    reward_composed: "reward_composed";
+    rollout_group_started: "rollout_group_started";
     run_completed: "run_completed";
     run_failed: "run_failed";
+    run_started: "run_started";
+    run_state_changed: "run_state_changed";
 }>;
 export declare const RunTelemetryLineageSchema: z.ZodObject<{
     modelProjectId: z.ZodString;
@@ -48,26 +48,26 @@ export declare const RunTelemetryEventSchema: z.ZodObject<{
     sequence: z.ZodNumber;
     occurredAt: z.ZodString;
     source: z.ZodEnum<{
-        grader: "grader";
-        runtime: "runtime";
+        control_plane: "control_plane";
         environment: "environment";
         evaluation: "evaluation";
+        grader: "grader";
         optimizer: "optimizer";
-        control_plane: "control_plane";
+        runtime: "runtime";
     }>;
     type: z.ZodEnum<{
-        cleanup_completed: "cleanup_completed";
-        run_started: "run_started";
-        run_state_changed: "run_state_changed";
-        rollout_group_started: "rollout_group_started";
         attempt_completed: "attempt_completed";
-        grader_completed: "grader_completed";
-        reward_composed: "reward_composed";
-        optimizer_step_completed: "optimizer_step_completed";
         checkpoint_committed: "checkpoint_committed";
+        cleanup_completed: "cleanup_completed";
         evaluation_completed: "evaluation_completed";
+        grader_completed: "grader_completed";
+        optimizer_step_completed: "optimizer_step_completed";
+        reward_composed: "reward_composed";
+        rollout_group_started: "rollout_group_started";
         run_completed: "run_completed";
         run_failed: "run_failed";
+        run_started: "run_started";
+        run_state_changed: "run_state_changed";
     }>;
     visibility: z.ZodEnum<{
         host_private: "host_private";
@@ -90,19 +90,19 @@ export declare const RunTelemetryEventSchema: z.ZodObject<{
     attributes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
 }, z.core.$strict>;
 export declare const MetricValueTypeSchema: z.ZodEnum<{
-    gauge: "gauge";
     counter: "counter";
     distribution: "distribution";
+    gauge: "gauge";
 }>;
 export declare const MetricUnitSchema: z.ZodEnum<{
-    ratio: "ratio";
+    bytes: "bytes";
     count: "count";
-    seconds: "seconds";
     milliseconds: "milliseconds";
+    ratio: "ratio";
+    scalar: "scalar";
+    seconds: "seconds";
     tokens: "tokens";
-    bytes: "bytes";
     usd: "usd";
-    scalar: "scalar";
 }>;
 export declare const MetricDirectionSchema: z.ZodEnum<{
     higher: "higher";
@@ -110,13 +110,13 @@ export declare const MetricDirectionSchema: z.ZodEnum<{
     neutral: "neutral";
 }>;
 export declare const MetricAggregationSchema: z.ZodEnum<{
-    max: "max";
     last: "last";
-    sum: "sum";
+    max: "max";
     mean: "mean";
     min: "min";
     p50: "p50";
     p95: "p95";
+    sum: "sum";
 }>;
 export declare const MetricDefinitionSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.metricDefinition.v1">;
@@ -124,19 +124,19 @@ export declare const MetricDefinitionSchema: z.ZodObject<{
     displayName: z.ZodString;
     description: z.ZodString;
     valueType: z.ZodEnum<{
-        gauge: "gauge";
         counter: "counter";
         distribution: "distribution";
+        gauge: "gauge";
     }>;
     unit: z.ZodEnum<{
-        ratio: "ratio";
+        bytes: "bytes";
         count: "count";
-        seconds: "seconds";
         milliseconds: "milliseconds";
+        ratio: "ratio";
+        scalar: "scalar";
+        seconds: "seconds";
         tokens: "tokens";
-        bytes: "bytes";
         usd: "usd";
-        scalar: "scalar";
     }>;
     direction: z.ZodEnum<{
         higher: "higher";
@@ -144,13 +144,13 @@ export declare const MetricDefinitionSchema: z.ZodObject<{
         neutral: "neutral";
     }>;
     aggregation: z.ZodEnum<{
-        max: "max";
         last: "last";
-        sum: "sum";
+        max: "max";
         mean: "mean";
         min: "min";
         p50: "p50";
         p95: "p95";
+        sum: "sum";
     }>;
     visibility: z.ZodEnum<{
         host_private: "host_private";
@@ -190,26 +190,26 @@ export declare const RunTelemetryBatchSchema: z.ZodObject<{
         sequence: z.ZodNumber;
         occurredAt: z.ZodString;
         source: z.ZodEnum<{
-            grader: "grader";
-            runtime: "runtime";
+            control_plane: "control_plane";
             environment: "environment";
             evaluation: "evaluation";
+            grader: "grader";
             optimizer: "optimizer";
-            control_plane: "control_plane";
+            runtime: "runtime";
         }>;
         type: z.ZodEnum<{
-            cleanup_completed: "cleanup_completed";
-            run_started: "run_started";
-            run_state_changed: "run_state_changed";
-            rollout_group_started: "rollout_group_started";
             attempt_completed: "attempt_completed";
-            grader_completed: "grader_completed";
-            reward_composed: "reward_composed";
-            optimizer_step_completed: "optimizer_step_completed";
             checkpoint_committed: "checkpoint_committed";
+            cleanup_completed: "cleanup_completed";
             evaluation_completed: "evaluation_completed";
+            grader_completed: "grader_completed";
+            optimizer_step_completed: "optimizer_step_completed";
+            reward_composed: "reward_composed";
+            rollout_group_started: "rollout_group_started";
             run_completed: "run_completed";
             run_failed: "run_failed";
+            run_started: "run_started";
+            run_state_changed: "run_state_changed";
         }>;
         visibility: z.ZodEnum<{
             host_private: "host_private";

package/dist/types/telemetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../../../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,4BAA4B,EAAG,+BAAwC,CAAC;AACrF,eAAO,MAAM,gCAAgC,EAAG,8BAAuC,CAAC;AACxF,eAAO,MAAM,iCAAiC,EAAG,+BAAwC,CAAC;AAE1F,eAAO,MAAM,yBAAyB;;;;EAIpC,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;;;EAOhC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;EAanC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;kBAY3B,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;EAA+C,CAAC;AAClF,eAAO,MAAM,gBAAgB;;;;;;;;;EAS3B,CAAC;AACH,eAAO,MAAM,qBAAqB;;;;EAAyC,CAAC;AAC5E,eAAO,MAAM,uBAAuB;;;;;;;;EAA8D,CAAC;AAEnG,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWxB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,4BAA4B,EAAG,+BAAwC,CAAC;AACrF,eAAO,MAAM,gCAAgC,EAAG,8BAAuC,CAAC;AACxF,eAAO,MAAM,iCAAiC,EAAG,+BAAwC,CAAC;AAE1F,eAAO,MAAM,yBAAyB;;;;EAIpC,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;;;EAOhC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;EAanC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;kBAY3B,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;EAA+C,CAAC;AAClF,eAAO,MAAM,gBAAgB;;;;;;;;;EAS3B,CAAC;AACH,eAAO,MAAM,qBAAqB;;;;EAAyC,CAAC;AAC5E,eAAO,MAAM,uBAAuB;;;;;;;;EAA8D,CAAC;AAEnG,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWxB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.21",
+  "version": "0.12.1",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -19,6 +19,10 @@
     "LICENSE"
   ],
   "exports": {
+    "./connected-evidence/git": {
+      "types": "./dist/types/connected-evidence/git.d.ts",
+      "import": "./dist/connected-evidence/git.js"
+    },
     "./native-conversations": {
       "types": "./dist/types/native-conversations/index.d.ts",
       "import": "./dist/native-conversations/index.js"
@@ -177,6 +181,7 @@
   "dependencies": {
     "@jitl/quickjs-singlefile-cjs-release-sync": "0.32.0",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
+    "acorn": "8.15.0",
     "ajv": "^8.20.0",
     "cron-parser": "^5.6.2",
     "quickjs-emscripten-core": "0.32.0",

package/schemas/learning/v1/binding.schema.json

@@ -93,6 +93,14 @@
         "graderId": {
           "$ref": "#/$defs/__schema0"
         },
+        "taskIds": {
+          "minItems": 1,
+          "maxItems": 10000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
         "reward": {
           "$ref": "#/$defs/__schema1"
         },

package/schemas/learning/v1/command-request.schema.json

@@ -1482,6 +1482,14 @@
         "graderId": {
           "$ref": "#/$defs/__schema0"
         },
+        "taskIds": {
+          "minItems": 1,
+          "maxItems": 10000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
         "reward": {
           "$ref": "#/$defs/__schema1"
         },
@@ -1985,6 +1993,10 @@
             }
           ]
         },
+        "submissionMode": {
+          "type": "string",
+          "const": "structured_json"
+        },
         "maxBytes": {
           "default": null,
           "anyOf": [

package/schemas/learning/v1/decision.schema.json

@@ -236,6 +236,9 @@
         "taskHash": {
           "$ref": "#/$defs/__schema2"
         },
+        "taskId": {
+          "$ref": "#/$defs/__schema0"
+        },
         "outputHash": {
           "$ref": "#/$defs/__schema2"
         },
@@ -368,7 +371,8 @@
             "scored",
             "pending",
             "unavailable",
-            "failed"
+            "failed",
+            "not_applicable"
           ]
         },
         "rawScore": {

package/schemas/learning/v1/definition.schema.json

@@ -368,6 +368,10 @@
             }
           ]
         },
+        "submissionMode": {
+          "type": "string",
+          "const": "structured_json"
+        },
         "maxBytes": {
           "default": null,
           "anyOf": [

package/schemas/learning/v1/draft.schema.json

@@ -439,6 +439,14 @@
         "graderId": {
           "$ref": "#/$defs/__schema0"
         },
+        "taskIds": {
+          "minItems": 1,
+          "maxItems": 10000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
         "reward": {
           "$ref": "#/$defs/__schema2"
         },

package/schemas/learning/v1/evidence.schema.json

@@ -342,6 +342,10 @@
             }
           ]
         },
+        "submissionMode": {
+          "type": "string",
+          "const": "structured_json"
+        },
         "maxBytes": {
           "default": null,
           "anyOf": [

package/schemas/learning/v1/example-submission.schema.json

@@ -278,6 +278,10 @@
             }
           ]
         },
+        "submissionMode": {
+          "type": "string",
+          "const": "structured_json"
+        },
         "maxBytes": {
           "default": null,
           "anyOf": [

package/schemas/learning/v1/grade.schema.json

@@ -62,6 +62,9 @@
             "taskHash": {
               "$ref": "#/$defs/__schema2"
             },
+            "taskId": {
+              "$ref": "#/$defs/__schema0"
+            },
             "outputHash": {
               "$ref": "#/$defs/__schema2"
             },
@@ -345,7 +348,8 @@
             "scored",
             "pending",
             "unavailable",
-            "failed"
+            "failed",
+            "not_applicable"
           ]
         },
         "rawScore": {