@openpond/evals 0.9.0 → 0.9.1

package/README.md

@@ -44,6 +44,17 @@ Subpath exports are available at `/harness`, `/tasksets`, `/benchmarks`, `/grade
 not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
 resolve credentials, or persist artifacts.
 
+## Ordinary graders
+
+`evaluateDeterministicGrader` and `portableDeterministicCheck` from `/graders`
+provide the shared local/portable interpretation of ordinary checks. They retain
+explicit exact-text normalization, final-answer extraction, structured JSON,
+artifact and runtime-event requirements, and structural state comparisons.
+Native test/diff checks declare their expected output field when exported.
+Empty, unsupported or unavailable checks remain unscorable rather than earning
+a zero or a passing reward. Desktop grade components preserve null scores and
+display them as not scored.
+
 ## Authored Taskset metrics
 
 `@openpond/evals/metrics` exports the shared `TasksetMetricPolicySchema` and

package/dist/deterministic-graders.js

@@ -0,0 +1,110 @@
+import { contentHash } from "@openpond/harness";
+import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export function evaluateDeterministicGrader(input) {
+    const { grader, task, evidence } = input;
+    const config = grader.config;
+    if (grader.kind === "content") {
+        const outputField = text(config.outputField) ?? "text";
+        const actual = text(evidence.output[outputField]);
+        if (config.operator === "exact_equals") {
+            const expected = text(config.expectedValue);
+            if (expected === null)
+                return unavailable("Exact content grading requires an expected value.");
+            const normalize = (value) => {
+                const unicode = config.normalizeUnicode === true ? value.normalize("NFC") : value;
+                return config.trimWhitespace === true ? unicode.trim() : unicode;
+            };
+            return outcome(actual !== null && normalize(actual) === normalize(expected));
+        }
+        if (config.operator !== undefined && config.operator !== "final_answer_equals_expected")
+            return unavailable("Unsupported content grader operator.");
+        if (config.operator === undefined && (config.includes !== undefined || config.excludes !== undefined)) {
+            const includes = strings(config.includes), excludes = strings(config.excludes);
+            if (!includes.length && !excludes.length)
+                return unavailable("Content grading requires a nonempty check.");
+            const value = typeof evidence.output.text === "string" ? evidence.output.text : JSON.stringify(evidence.output);
+            return outcome(includes.every(item => value.includes(item)) && excludes.every(item => !value.includes(item)));
+        }
+        const expected = text(config.expectedValue) ?? text(task.expectedOutput?.[text(config.expectedField) ?? "text"]);
+        const normalize = config.operator === "final_answer_equals_expected" ? normalizeFinalAnswer : normalizeContent;
+        if (expected === null || !normalize(expected))
+            return unavailable("Content grading requires a nonempty expected value.");
+        return outcome(actual !== null && normalize(actual) === normalize(expected));
+    }
+    if (grader.kind === "schema") {
+        if (config.operator !== undefined && config.operator !== "json_schema_subset")
+            return unavailable("Unsupported schema grader operator.");
+        const schema = config.operator === "json_schema_subset" ? config.schema : config.jsonSchema;
+        if (schema !== undefined) {
+            const checked = validateTaskSchema(schema);
+            if (!checked.valid)
+                return unavailable(checked.issues[0].message);
+            let value = evidence.output;
+            if (config.operator === "json_schema_subset") {
+                const encoded = evidence.output[text(config.jsonField) ?? "text"];
+                if (typeof encoded === "string") {
+                    try {
+                        value = JSON.parse(encoded.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""));
+                    }
+                    catch {
+                        return outcome(false, "The model output was not valid JSON.");
+                    }
+                }
+            }
+            const checkedValue = validateTaskValue(schema, value);
+            return outcome(checkedValue.valid, checkedValue.valid ? undefined : checkedValue.issues.map(issue => issue.message).join(" "));
+        }
+        const required = strings(config.requiredKeys);
+        if (!required.length)
+            return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
+        return outcome(required.every(key => Object.hasOwn(evidence.output, key)));
+    }
+    if (grader.kind === "artifact") {
+        const contains = text(config.refIncludes);
+        if (!contains)
+            return unavailable("Artifact grading requires a declared artifact reference.");
+        return outcome(evidence.artifactRefs.some(ref => ref.includes(contains)));
+    }
+    if (grader.kind === "runtime_event") {
+        const required = strings(config.requiredEvents);
+        if (!required.length)
+            return unavailable("Runtime event grading requires declared events.");
+        return outcome(required.every(event => evidence.runtimeEventRefs.some(ref => ref.includes(event))));
+    }
+    if (config.operator === "output_field_equals") {
+        const field = text(config.outputField);
+        if (!field || !Object.hasOwn(config, "expectedValue"))
+            return unavailable("Output comparison requires a field and expected value.");
+        return outcome(Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(config.expectedValue));
+    }
+    if (config.operator !== undefined)
+        return unavailable("Unsupported state grader operator.");
+    const fields = strings(config.fields);
+    const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
+    if (!compared.length || compared.some(field => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
+        return unavailable("State grading requires expected values for every compared field.");
+    return outcome(compared.every(field => Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(task.expectedOutput[field])));
+}
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export function portableDeterministicCheck(grader) {
+    if (grader.kind === "file") {
+        const { pathIncludes, ...config } = grader.config;
+        return { kind: "artifact", config: { ...config, ...(pathIncludes === undefined ? {} : { refIncludes: pathIncludes }) } };
+    }
+    if (grader.kind === "diff" || grader.kind === "test")
+        return { kind: "state", config: { operator: "output_field_equals", outputField: grader.kind === "test" ? "testsPassed" : "diffAccepted", expectedValue: true } };
+    return { kind: grader.kind, config: grader.config };
+}
+function outcome(passed, feedback) { return { score: Number(passed), passed, feedback: feedback ?? (passed ? "Deterministic grader passed." : "Deterministic grader failed.") }; }
+function unavailable(feedback) { return { score: null, passed: false, feedback }; }
+function text(value) { return typeof value === "string" ? value : null; }
+function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
+function normalizeContent(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }
+function normalizeFinalAnswer(value) {
+    const boxed = [...value.matchAll(/\\boxed\{([^{}]+)\}/g)].at(-1)?.[1];
+    const hashAnswer = value.match(/####\s*([^\n\r]+)/)?.[1];
+    const label = value.match(/(?:final\s+answer|answer)\s*(?::|is|=)\s*([^\n\r]+)/i)?.[1];
+    return normalizeContent((boxed ?? hashAnswer ?? label ?? value).normalize("NFKC").trim().replace(/^\$+|\$+$/g, "").replace(/^\\\(|\\\)$/g, ""));
+}

package/dist/graders.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
-import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+import { evaluateDeterministicGrader } from "./deterministic-graders.js";
+export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
 export const GraderEvidenceContentSchema = z.object({
     schemaVersion: z.literal("openpond.graderEvidence.v1"),
     graderId: ReleaseIdSchema,
@@ -43,55 +44,15 @@ export async function gradeEvidence(input) {
     }));
 }
 function gradeDeterministic(grader, task, attempt) {
-    let passed = false;
-    const config = grader.config;
-    if (grader.kind === "content") {
-        const outputField = string(config.outputField) ?? "text";
-        const actual = string(attempt.output[outputField]);
-        const expected = string(config.expectedValue) ?? string(task.expectedOutput?.[string(config.expectedField) ?? "text"]);
-        if (expected === null || !normalize(expected))
-            return unavailable("Content grading requires a nonempty expected value.");
-        passed = actual !== null && expected !== null && normalize(actual) === normalize(expected);
-    }
-    else if (grader.kind === "schema") {
-        if (config.jsonSchema !== undefined) {
-            const schema = validateTaskSchema(config.jsonSchema);
-            if (!schema.valid)
-                return unavailable(schema.issues[0].message);
-            passed = validateTaskValue(config.jsonSchema, attempt.output).valid;
-        }
-        else {
-            const required = strings(config.requiredKeys);
-            if (!required.length)
-                return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
-            passed = required.every((key) => Object.hasOwn(attempt.output, key));
-        }
-    }
-    else if (grader.kind === "artifact") {
-        const contains = string(config.refIncludes) ?? "";
-        if (!contains)
-            return unavailable("Artifact grading requires a declared artifact reference.");
-        passed = attempt.artifactRefs.some((ref) => ref.includes(contains));
-    }
-    else if (grader.kind === "runtime_event") {
-        const requiredEvents = strings(config.requiredEvents);
-        if (!requiredEvents.length)
-            return unavailable("Runtime event grading requires declared events.");
-        passed = requiredEvents.every((required) => attempt.runtimeEventRefs.some((ref) => ref.includes(required)));
-    }
-    else {
-        const fields = strings(config.fields);
-        const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
-        if (!compared.length || compared.some((field) => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
-            return unavailable("State grading requires expected values for every compared field.");
-        passed = compared.every((field) => Object.hasOwn(attempt.output, field) && contentHash(attempt.output[field]) === contentHash(task.expectedOutput[field]));
-    }
+    const result = evaluateDeterministicGrader({ grader, task, evidence: attempt });
+    if (result.score === null)
+        return unavailable(result.feedback);
     return {
-        score: passed ? 1 : 0,
-        passed,
+        score: result.score,
+        passed: result.passed,
         rewardEligible: grader.rewardEligible,
-        failureClass: passed ? null : "policy_failure",
-        feedback: [passed ? "Deterministic grader passed." : "Deterministic grader failed."],
+        failureClass: result.passed ? null : "policy_failure",
+        feedback: [result.feedback],
         visibleEvidenceRefs: [...attempt.runtimeEventRefs, ...attempt.artifactRefs],
         privilegedEvidenceRefs: grader.privileged ? [task.privilegedContextRef].filter((ref) => ref !== null) : [],
     };
@@ -109,6 +70,3 @@ function evidence(grader, result) {
 function unavailable(message) {
     return { score: null, passed: false, rewardEligible: false, failureClass: "grader_failure", feedback: [message], visibleEvidenceRefs: [], privilegedEvidenceRefs: [] };
 }
-function string(value) { return typeof value === "string" ? value : null; }
-function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
-function normalize(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }

package/dist/types/deterministic-graders.d.ts

@@ -0,0 +1,23 @@
+import type { DeterministicGraderSpec, TaskRecord } from "./tasksets.js";
+export type DeterministicGrade = {
+    score: number | null;
+    passed: boolean;
+    feedback: string;
+};
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export declare function evaluateDeterministicGrader(input: {
+    grader: Pick<DeterministicGraderSpec, "kind" | "config">;
+    task: Pick<TaskRecord, "expectedOutput">;
+    evidence: {
+        output: Record<string, unknown>;
+        artifactRefs: string[];
+        runtimeEventRefs: string[];
+    };
+}): DeterministicGrade;
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export declare function portableDeterministicCheck(grader: {
+    kind: DeterministicGraderSpec["kind"] | "file" | "diff" | "test";
+    config: Record<string, unknown>;
+}): Pick<DeterministicGraderSpec, "kind" | "config">;
+//# sourceMappingURL=deterministic-graders.d.ts.map

package/dist/types/deterministic-graders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic-graders.d.ts","sourceRoot":"","sources":["../../../../src/deterministic-graders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGzE,MAAM,MAAM,kBAAkB,GAAG;IAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7F;6EAC6E;AAC7E,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;IACzD,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;IACzC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACnG,GAAG,kBAAkB,CAoErB;AAED,oFAAoF;AACpF,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,uBAAuB,CAAC,MAAM,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAO1M"}

package/dist/types/graders.d.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import type { GraderSpec, TaskRecord } from "./tasksets.js";
+export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
 export declare const GraderEvidenceContentSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
     graderId: z.ZodString;

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGrF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA+DD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}

package/dist/types/learning/repository.d.ts

@@ -1421,10 +1421,10 @@ export declare const LearningResourceKindSchema: z.ZodEnum<{
     policy: "policy";
     feedback: "feedback";
     reward: "reward";
+    evidence: "evidence";
     source: "source";
     asset: "asset";
     decision: "decision";
-    evidence: "evidence";
     binding: "binding";
     batch: "batch";
     grade: "grade";

package/dist/types/learning/transport.d.ts

@@ -1422,10 +1422,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
         policy: "policy";
         feedback: "feedback";
         reward: "reward";
+        evidence: "evidence";
         source: "source";
         asset: "asset";
         decision: "decision";
-        evidence: "evidence";
         binding: "binding";
         batch: "batch";
         grade: "grade";
@@ -1444,10 +1444,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
         policy: "policy";
         feedback: "feedback";
         reward: "reward";
+        evidence: "evidence";
         source: "source";
         asset: "asset";
         decision: "decision";
-        evidence: "evidence";
         binding: "binding";
         batch: "batch";
         grade: "grade";

package/dist/types/telemetry-analysis.d.ts

@@ -18,8 +18,8 @@ export declare const EvidenceReferenceSchema: z.ZodObject<{
     kind: z.ZodEnum<{
         artifact: "artifact";
         attempt: "attempt";
-        trace: "trace";
         grader: "grader";
+        trace: "trace";
         rollout: "rollout";
         checkpoint: "checkpoint";
     }>;

package/dist/types/telemetry-bundle.d.ts

@@ -54,8 +54,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;
@@ -122,8 +122,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
         kind: z.ZodEnum<{
             artifact: "artifact";
             attempt: "attempt";
-            trace: "trace";
             grader: "grader";
+            trace: "trace";
             rollout: "rollout";
             checkpoint: "checkpoint";
         }>;
@@ -202,8 +202,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;
@@ -270,8 +270,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
         kind: z.ZodEnum<{
             artifact: "artifact";
             attempt: "attempt";
-            trace: "trace";
             grader: "grader";
+            trace: "trace";
             rollout: "rollout";
             checkpoint: "checkpoint";
         }>;

package/dist/types/telemetry.d.ts

@@ -10,8 +10,8 @@ export declare const TelemetryVisibilitySchema: z.ZodEnum<{
 export declare const TelemetrySourceSchema: z.ZodEnum<{
     environment: "environment";
     evaluation: "evaluation";
-    runtime: "runtime";
     grader: "grader";
+    runtime: "runtime";
     optimizer: "optimizer";
     control_plane: "control_plane";
 }>;
@@ -50,8 +50,8 @@ export declare const RunTelemetryEventSchema: z.ZodObject<{
     source: z.ZodEnum<{
         environment: "environment";
         evaluation: "evaluation";
-        runtime: "runtime";
         grader: "grader";
+        runtime: "runtime";
         optimizer: "optimizer";
         control_plane: "control_plane";
     }>;
@@ -192,8 +192,8 @@ export declare const RunTelemetryBatchSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",