@openpond/evals 0.9.0 → 0.9.2

package/README.md

@@ -44,6 +44,24 @@ Subpath exports are available at `/harness`, `/tasksets`, `/benchmarks`, `/grade
 not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
 resolve credentials, or persist artifacts.
 
+## Ordinary graders
+
+`gradeTaskEvidence` returns a sealed ordinary Taskset grade with the complete
+grader population, weighted score, hard gates and explicit unavailable status.
+`verifyTaskGrade` checks retained results against the exact admitted task,
+attempt evidence and graders, including each component hash and aggregation.
+`aggregateGraderScores` provides the same policy to native grading adapters.
+Ordinary grades do not require or manufacture a Reward binding.
+
+`evaluateDeterministicGrader` and `portableDeterministicCheck` from `/graders`
+provide the shared local/portable interpretation of ordinary checks. They retain
+explicit exact-text normalization, final-answer extraction, structured JSON,
+artifact and runtime-event requirements, and structural state comparisons.
+Native test/diff checks declare their expected output field when exported.
+Empty, unsupported or unavailable checks remain unscorable rather than earning
+a zero or a passing reward. Desktop grade components preserve null scores and
+display them as not scored.
+
 ## Authored Taskset metrics
 
 `@openpond/evals/metrics` exports the shared `TasksetMetricPolicySchema` and

package/dist/deterministic-graders.js

@@ -0,0 +1,110 @@
+import { contentHash } from "@openpond/harness";
+import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export function evaluateDeterministicGrader(input) {
+    const { grader, task, evidence } = input;
+    const config = grader.config;
+    if (grader.kind === "content") {
+        const outputField = text(config.outputField) ?? "text";
+        const actual = text(evidence.output[outputField]);
+        if (config.operator === "exact_equals") {
+            const expected = text(config.expectedValue);
+            if (expected === null)
+                return unavailable("Exact content grading requires an expected value.");
+            const normalize = (value) => {
+                const unicode = config.normalizeUnicode === true ? value.normalize("NFC") : value;
+                return config.trimWhitespace === true ? unicode.trim() : unicode;
+            };
+            return outcome(actual !== null && normalize(actual) === normalize(expected));
+        }
+        if (config.operator !== undefined && config.operator !== "final_answer_equals_expected")
+            return unavailable("Unsupported content grader operator.");
+        if (config.operator === undefined && (config.includes !== undefined || config.excludes !== undefined)) {
+            const includes = strings(config.includes), excludes = strings(config.excludes);
+            if (!includes.length && !excludes.length)
+                return unavailable("Content grading requires a nonempty check.");
+            const value = typeof evidence.output.text === "string" ? evidence.output.text : JSON.stringify(evidence.output);
+            return outcome(includes.every(item => value.includes(item)) && excludes.every(item => !value.includes(item)));
+        }
+        const expected = text(config.expectedValue) ?? text(task.expectedOutput?.[text(config.expectedField) ?? "text"]);
+        const normalize = config.operator === "final_answer_equals_expected" ? normalizeFinalAnswer : normalizeContent;
+        if (expected === null || !normalize(expected))
+            return unavailable("Content grading requires a nonempty expected value.");
+        return outcome(actual !== null && normalize(actual) === normalize(expected));
+    }
+    if (grader.kind === "schema") {
+        if (config.operator !== undefined && config.operator !== "json_schema_subset")
+            return unavailable("Unsupported schema grader operator.");
+        const schema = config.operator === "json_schema_subset" ? config.schema : config.jsonSchema;
+        if (schema !== undefined) {
+            const checked = validateTaskSchema(schema);
+            if (!checked.valid)
+                return unavailable(checked.issues[0].message);
+            let value = evidence.output;
+            if (config.operator === "json_schema_subset") {
+                const encoded = evidence.output[text(config.jsonField) ?? "text"];
+                if (typeof encoded === "string") {
+                    try {
+                        value = JSON.parse(encoded.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""));
+                    }
+                    catch {
+                        return outcome(false, "The model output was not valid JSON.");
+                    }
+                }
+            }
+            const checkedValue = validateTaskValue(schema, value);
+            return outcome(checkedValue.valid, checkedValue.valid ? undefined : checkedValue.issues.map(issue => issue.message).join(" "));
+        }
+        const required = strings(config.requiredKeys);
+        if (!required.length)
+            return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
+        return outcome(required.every(key => Object.hasOwn(evidence.output, key)));
+    }
+    if (grader.kind === "artifact") {
+        const contains = text(config.refIncludes);
+        if (!contains)
+            return unavailable("Artifact grading requires a declared artifact reference.");
+        return outcome(evidence.artifactRefs.some(ref => ref.includes(contains)));
+    }
+    if (grader.kind === "runtime_event") {
+        const required = strings(config.requiredEvents);
+        if (!required.length)
+            return unavailable("Runtime event grading requires declared events.");
+        return outcome(required.every(event => evidence.runtimeEventRefs.some(ref => ref.includes(event))));
+    }
+    if (config.operator === "output_field_equals") {
+        const field = text(config.outputField);
+        if (!field || !Object.hasOwn(config, "expectedValue"))
+            return unavailable("Output comparison requires a field and expected value.");
+        return outcome(Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(config.expectedValue));
+    }
+    if (config.operator !== undefined)
+        return unavailable("Unsupported state grader operator.");
+    const fields = strings(config.fields);
+    const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
+    if (!compared.length || compared.some(field => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
+        return unavailable("State grading requires expected values for every compared field.");
+    return outcome(compared.every(field => Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(task.expectedOutput[field])));
+}
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export function portableDeterministicCheck(grader) {
+    if (grader.kind === "file") {
+        const { pathIncludes, ...config } = grader.config;
+        return { kind: "artifact", config: { ...config, ...(pathIncludes === undefined ? {} : { refIncludes: pathIncludes }) } };
+    }
+    if (grader.kind === "diff" || grader.kind === "test")
+        return { kind: "state", config: { operator: "output_field_equals", outputField: grader.kind === "test" ? "testsPassed" : "diffAccepted", expectedValue: true } };
+    return { kind: grader.kind, config: grader.config };
+}
+function outcome(passed, feedback) { return { score: Number(passed), passed, feedback: feedback ?? (passed ? "Deterministic grader passed." : "Deterministic grader failed.") }; }
+function unavailable(feedback) { return { score: null, passed: false, feedback }; }
+function text(value) { return typeof value === "string" ? value : null; }
+function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
+function normalizeContent(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }
+function normalizeFinalAnswer(value) {
+    const boxed = [...value.matchAll(/\\boxed\{([^{}]+)\}/g)].at(-1)?.[1];
+    const hashAnswer = value.match(/####\s*([^\n\r]+)/)?.[1];
+    const label = value.match(/(?:final\s+answer|answer)\s*(?::|is|=)\s*([^\n\r]+)/i)?.[1];
+    return normalizeContent((boxed ?? hashAnswer ?? label ?? value).normalize("NFKC").trim().replace(/^\$+|\$+$/g, "").replace(/^\\\(|\\\)$/g, ""));
+}

package/dist/graders.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
-import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+import { evaluateDeterministicGrader } from "./deterministic-graders.js";
+export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
 export const GraderEvidenceContentSchema = z.object({
     schemaVersion: z.literal("openpond.graderEvidence.v1"),
     graderId: ReleaseIdSchema,
@@ -43,55 +44,15 @@ export async function gradeEvidence(input) {
     }));
 }
 function gradeDeterministic(grader, task, attempt) {
-    let passed = false;
-    const config = grader.config;
-    if (grader.kind === "content") {
-        const outputField = string(config.outputField) ?? "text";
-        const actual = string(attempt.output[outputField]);
-        const expected = string(config.expectedValue) ?? string(task.expectedOutput?.[string(config.expectedField) ?? "text"]);
-        if (expected === null || !normalize(expected))
-            return unavailable("Content grading requires a nonempty expected value.");
-        passed = actual !== null && expected !== null && normalize(actual) === normalize(expected);
-    }
-    else if (grader.kind === "schema") {
-        if (config.jsonSchema !== undefined) {
-            const schema = validateTaskSchema(config.jsonSchema);
-            if (!schema.valid)
-                return unavailable(schema.issues[0].message);
-            passed = validateTaskValue(config.jsonSchema, attempt.output).valid;
-        }
-        else {
-            const required = strings(config.requiredKeys);
-            if (!required.length)
-                return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
-            passed = required.every((key) => Object.hasOwn(attempt.output, key));
-        }
-    }
-    else if (grader.kind === "artifact") {
-        const contains = string(config.refIncludes) ?? "";
-        if (!contains)
-            return unavailable("Artifact grading requires a declared artifact reference.");
-        passed = attempt.artifactRefs.some((ref) => ref.includes(contains));
-    }
-    else if (grader.kind === "runtime_event") {
-        const requiredEvents = strings(config.requiredEvents);
-        if (!requiredEvents.length)
-            return unavailable("Runtime event grading requires declared events.");
-        passed = requiredEvents.every((required) => attempt.runtimeEventRefs.some((ref) => ref.includes(required)));
-    }
-    else {
-        const fields = strings(config.fields);
-        const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
-        if (!compared.length || compared.some((field) => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
-            return unavailable("State grading requires expected values for every compared field.");
-        passed = compared.every((field) => Object.hasOwn(attempt.output, field) && contentHash(attempt.output[field]) === contentHash(task.expectedOutput[field]));
-    }
+    const result = evaluateDeterministicGrader({ grader, task, evidence: attempt });
+    if (result.score === null)
+        return unavailable(result.feedback);
     return {
-        score: passed ? 1 : 0,
-        passed,
+        score: result.score,
+        passed: result.passed,
         rewardEligible: grader.rewardEligible,
-        failureClass: passed ? null : "policy_failure",
-        feedback: [passed ? "Deterministic grader passed." : "Deterministic grader failed."],
+        failureClass: result.passed ? null : "policy_failure",
+        feedback: [result.feedback],
         visibleEvidenceRefs: [...attempt.runtimeEventRefs, ...attempt.artifactRefs],
         privilegedEvidenceRefs: grader.privileged ? [task.privilegedContextRef].filter((ref) => ref !== null) : [],
     };
@@ -109,6 +70,73 @@ function evidence(grader, result) {
 function unavailable(message) {
     return { score: null, passed: false, rewardEligible: false, failureClass: "grader_failure", feedback: [message], visibleEvidenceRefs: [], privilegedEvidenceRefs: [] };
 }
-function string(value) { return typeof value === "string" ? value : null; }
-function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
-function normalize(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }
+const GradeSummarySchema = z.object({
+    score: z.number().min(0).max(1).nullable(), passed: z.boolean(),
+    rewardEligible: z.boolean(), failureClass: FailureClassSchema.nullable(),
+    gradingStatus: z.enum(["scored", "unscorable", "not_configured"]),
+}).strict();
+/** Shared ordinary grading policy. Results must match the declared population;
+ * missing checks cannot disappear from the denominator or the hard gates. */
+export function aggregateGraderScores(input) {
+    const ids = new Set(input.graders.map(grader => grader.id));
+    if (ids.size !== input.graders.length || input.components.length !== input.graders.length)
+        throw new Error("Grade population differs from its declared graders.");
+    const components = new Map(input.components.map(item => [item.graderId, item]));
+    if (components.size !== input.components.length)
+        throw new Error("Grade population contains duplicate graders.");
+    let weighted = 0;
+    let totalWeight = 0;
+    let hardGateFailed = false;
+    let passed = true;
+    let eligible = false;
+    let unavailable = false;
+    let infrastructure = !!input.infrastructureError;
+    for (const grader of input.graders) {
+        const item = components.get(grader.id);
+        if (!item || item.graderVersion !== grader.version)
+            throw new Error("Grade identity differs from its declared grader.");
+        if (!Number.isFinite(grader.weight) || grader.weight < 0 || (item.score !== null && (!Number.isFinite(item.score) || item.score < 0 || item.score > 1)))
+            throw new Error("Grade scores and weights must be finite and within their declared range.");
+        infrastructure ||= item.failureClass === "infrastructure_failure";
+        unavailable ||= item.score === null || (item.failureClass !== null && item.failureClass !== "policy_failure");
+        hardGateFailed ||= grader.hardGate && !item.passed;
+        passed &&= item.passed;
+        eligible ||= grader.rewardEligible && item.rewardEligible;
+        totalWeight += grader.weight;
+        weighted += (item.score ?? 0) * grader.weight;
+    }
+    const score = infrastructure || unavailable || totalWeight <= 0 ? null : hardGateFailed ? 0 : weighted / totalWeight;
+    passed = score !== null && !hardGateFailed && passed;
+    return GradeSummarySchema.parse({ score, passed, rewardEligible: score !== null && eligible,
+        failureClass: infrastructure ? "infrastructure_failure" : score === null ? "grader_failure" : passed ? null : "policy_failure",
+        gradingStatus: !input.graders.length && !infrastructure ? "not_configured" : score === null ? "unscorable" : "scored" });
+}
+export const TaskGradeSchema = GradeSummarySchema.extend({
+    schemaVersion: z.literal("openpond.taskGrade.v1"),
+    taskHash: ReleaseHashSchema, evidenceHash: ReleaseHashSchema, graderSetHash: ReleaseHashSchema,
+    components: z.array(GraderEvidenceSchema).max(1_000), contentHash: ReleaseHashSchema,
+}).strict();
+export async function gradeTaskEvidence(input) {
+    const components = await gradeEvidence(input);
+    const content = { schemaVersion: "openpond.taskGrade.v1",
+        taskHash: contentHash(input.task), evidenceHash: contentHash(input.evidence), graderSetHash: contentHash(input.graders), components,
+        ...aggregateGraderScores({ graders: input.graders, components, infrastructureError: input.evidence.infrastructureError }) };
+    return TaskGradeSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+/** Verify retained results against the exact admitted inputs without rerunning
+ * graders. The storage owner supplies the trusted artifact, not the caller. */
+export function verifyTaskGrade(value, input) {
+    const result = TaskGradeSchema.parse(value);
+    const { contentHash: hash, ...content } = result;
+    if (hash !== contentHash(content) || result.taskHash !== contentHash(input.task) || result.evidenceHash !== contentHash(input.evidence) || result.graderSetHash !== contentHash(input.graders))
+        throw new Error("Task grade differs from its admitted task, evidence or graders.");
+    for (const component of result.components) {
+        const { contentHash: componentHash, ...body } = component;
+        if (componentHash !== contentHash(body))
+            throw new Error("Grader evidence integrity failed.");
+    }
+    const summary = aggregateGraderScores({ graders: input.graders, components: result.components, infrastructureError: input.evidence.infrastructureError });
+    if (contentHash(GradeSummarySchema.parse({ score: result.score, passed: result.passed, rewardEligible: result.rewardEligible, failureClass: result.failureClass, gradingStatus: result.gradingStatus })) !== contentHash(summary))
+        throw new Error("Task grade summary differs from its grader evidence.");
+    return result;
+}

package/dist/types/deterministic-graders.d.ts

@@ -0,0 +1,23 @@
+import type { DeterministicGraderSpec, TaskRecord } from "./tasksets.js";
+export type DeterministicGrade = {
+    score: number | null;
+    passed: boolean;
+    feedback: string;
+};
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export declare function evaluateDeterministicGrader(input: {
+    grader: Pick<DeterministicGraderSpec, "kind" | "config">;
+    task: Pick<TaskRecord, "expectedOutput">;
+    evidence: {
+        output: Record<string, unknown>;
+        artifactRefs: string[];
+        runtimeEventRefs: string[];
+    };
+}): DeterministicGrade;
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export declare function portableDeterministicCheck(grader: {
+    kind: DeterministicGraderSpec["kind"] | "file" | "diff" | "test";
+    config: Record<string, unknown>;
+}): Pick<DeterministicGraderSpec, "kind" | "config">;
+//# sourceMappingURL=deterministic-graders.d.ts.map

package/dist/types/deterministic-graders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic-graders.d.ts","sourceRoot":"","sources":["../../../../src/deterministic-graders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGzE,MAAM,MAAM,kBAAkB,GAAG;IAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7F;6EAC6E;AAC7E,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;IACzD,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;IACzC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACnG,GAAG,kBAAkB,CAoErB;AAED,oFAAoF;AACpF,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,uBAAuB,CAAC,MAAM,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAO1M"}

package/dist/types/graders.d.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import type { GraderSpec, TaskRecord } from "./tasksets.js";
+export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
 export declare const GraderEvidenceContentSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
     graderId: z.ZodString;
@@ -67,4 +68,77 @@ export declare function gradeEvidence(input: {
     customVerifier?: CustomVerifierRunner;
 }): Promise<GraderEvidence[]>;
 export type GraderEvidence = z.infer<typeof GraderEvidenceSchema>;
+declare const GradeSummarySchema: z.ZodObject<{
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    gradingStatus: z.ZodEnum<{
+        scored: "scored";
+        unscorable: "unscorable";
+        not_configured: "not_configured";
+    }>;
+}, z.core.$strict>;
+/** Shared ordinary grading policy. Results must match the declared population;
+ * missing checks cannot disappear from the denominator or the hard gates. */
+export declare function aggregateGraderScores(input: {
+    graders: ReadonlyArray<Pick<GraderSpec, "id" | "version" | "weight" | "hardGate" | "rewardEligible">>;
+    components: ReadonlyArray<Pick<GraderEvidence, "graderId" | "graderVersion" | "score" | "passed" | "rewardEligible" | "failureClass">>;
+    infrastructureError?: string | null;
+}): z.infer<typeof GradeSummarySchema>;
+export declare const TaskGradeSchema: z.ZodObject<{
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    gradingStatus: z.ZodEnum<{
+        scored: "scored";
+        unscorable: "unscorable";
+        not_configured: "not_configured";
+    }>;
+    schemaVersion: z.ZodLiteral<"openpond.taskGrade.v1">;
+    taskHash: z.ZodString;
+    evidenceHash: z.ZodString;
+    graderSetHash: z.ZodString;
+    components: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
+        graderId: z.ZodString;
+        graderVersion: z.ZodString;
+        score: z.ZodNullable<z.ZodNumber>;
+        passed: z.ZodBoolean;
+        rewardEligible: z.ZodBoolean;
+        failureClass: z.ZodNullable<z.ZodEnum<{
+            policy_failure: "policy_failure";
+            grader_failure: "grader_failure";
+            environment_failure: "environment_failure";
+            infrastructure_failure: "infrastructure_failure";
+            timeout: "timeout";
+            cancelled: "cancelled";
+        }>>;
+        feedback: z.ZodArray<z.ZodString>;
+        visibleEvidenceRefs: z.ZodArray<z.ZodString>;
+        privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type TaskGrade = z.infer<typeof TaskGradeSchema>;
+export declare function gradeTaskEvidence(input: Parameters<typeof gradeEvidence>[0]): Promise<TaskGrade>;
+/** Verify retained results against the exact admitted inputs without rerunning
+ * graders. The storage owner supplies the trusted artifact, not the caller. */
+export declare function verifyTaskGrade(value: unknown, input: Pick<Parameters<typeof gradeEvidence>[0], "task" | "evidence" | "graders">): TaskGrade;
 //# sourceMappingURL=graders.d.ts.map

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGrF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA+DD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;kBAIb,CAAC;AAEZ;6EAC6E;AAC7E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,CAAC,CAAC,CAAC;IACtG,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,GAAG,eAAe,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,GAAG,cAAc,CAAC,CAAC,CAAC;IACvI,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAwBrC;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAIjB,CAAC;AACZ,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAMtG;AAED;+EAC+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC,GAAG,SAAS,CAW5I"}

package/dist/types/learning/repository.d.ts

@@ -1421,10 +1421,10 @@ export declare const LearningResourceKindSchema: z.ZodEnum<{
     policy: "policy";
     feedback: "feedback";
     reward: "reward";
+    evidence: "evidence";
     source: "source";
     asset: "asset";
     decision: "decision";
-    evidence: "evidence";
     binding: "binding";
     batch: "batch";
     grade: "grade";

package/dist/types/learning/transport.d.ts

@@ -1422,10 +1422,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
         policy: "policy";
         feedback: "feedback";
         reward: "reward";
+        evidence: "evidence";
         source: "source";
         asset: "asset";
         decision: "decision";
-        evidence: "evidence";
         binding: "binding";
         batch: "batch";
         grade: "grade";
@@ -1444,10 +1444,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
         policy: "policy";
         feedback: "feedback";
         reward: "reward";
+        evidence: "evidence";
         source: "source";
         asset: "asset";
         decision: "decision";
-        evidence: "evidence";
         binding: "binding";
         batch: "batch";
         grade: "grade";

package/dist/types/telemetry-analysis.d.ts

@@ -18,8 +18,8 @@ export declare const EvidenceReferenceSchema: z.ZodObject<{
     kind: z.ZodEnum<{
         artifact: "artifact";
         attempt: "attempt";
-        trace: "trace";
         grader: "grader";
+        trace: "trace";
         rollout: "rollout";
         checkpoint: "checkpoint";
     }>;

package/dist/types/telemetry-bundle.d.ts

@@ -54,8 +54,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;
@@ -122,8 +122,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
         kind: z.ZodEnum<{
             artifact: "artifact";
             attempt: "attempt";
-            trace: "trace";
             grader: "grader";
+            trace: "trace";
             rollout: "rollout";
             checkpoint: "checkpoint";
         }>;
@@ -202,8 +202,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;
@@ -270,8 +270,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
         kind: z.ZodEnum<{
             artifact: "artifact";
             attempt: "attempt";
-            trace: "trace";
             grader: "grader";
+            trace: "trace";
             rollout: "rollout";
             checkpoint: "checkpoint";
         }>;

package/dist/types/telemetry.d.ts

@@ -10,8 +10,8 @@ export declare const TelemetryVisibilitySchema: z.ZodEnum<{
 export declare const TelemetrySourceSchema: z.ZodEnum<{
     environment: "environment";
     evaluation: "evaluation";
-    runtime: "runtime";
     grader: "grader";
+    runtime: "runtime";
     optimizer: "optimizer";
     control_plane: "control_plane";
 }>;
@@ -50,8 +50,8 @@ export declare const RunTelemetryEventSchema: z.ZodObject<{
     source: z.ZodEnum<{
         environment: "environment";
         evaluation: "evaluation";
-        runtime: "runtime";
         grader: "grader";
+        runtime: "runtime";
         optimizer: "optimizer";
         control_plane: "control_plane";
     }>;
@@ -192,8 +192,8 @@ export declare const RunTelemetryBatchSchema: z.ZodObject<{
         source: z.ZodEnum<{
             environment: "environment";
             evaluation: "evaluation";
-            runtime: "runtime";
             grader: "grader";
+            runtime: "runtime";
             optimizer: "optimizer";
             control_plane: "control_plane";
         }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",