@openpond/evals 0.9.1 → 0.9.3

package/README.md

@@ -46,6 +46,13 @@ resolve credentials, or persist artifacts.
 
 ## Ordinary graders
 
+`gradeTaskEvidence` returns a sealed ordinary Taskset grade with the complete
+grader population, weighted score, hard gates and explicit unavailable status.
+`verifyTaskGrade` checks retained results against the exact admitted task,
+attempt evidence and graders, including each component hash and aggregation.
+`aggregateGraderScores` provides the same policy to native grading adapters.
+Ordinary grades do not require or manufacture a Reward binding.
+
 `evaluateDeterministicGrader` and `portableDeterministicCheck` from `/graders`
 provide the shared local/portable interpretation of ordinary checks. They retain
 explicit exact-text normalization, final-answer extraction, structured JSON,

package/dist/graders.js

@@ -70,3 +70,73 @@ function evidence(grader, result) {
 function unavailable(message) {
     return { score: null, passed: false, rewardEligible: false, failureClass: "grader_failure", feedback: [message], visibleEvidenceRefs: [], privilegedEvidenceRefs: [] };
 }
+const GradeSummarySchema = z.object({
+    score: z.number().min(0).max(1).nullable(), passed: z.boolean(),
+    rewardEligible: z.boolean(), failureClass: FailureClassSchema.nullable(),
+    gradingStatus: z.enum(["scored", "unscorable", "not_configured"]),
+}).strict();
+/** Shared ordinary grading policy. Results must match the declared population;
+ * missing checks cannot disappear from the denominator or the hard gates. */
+export function aggregateGraderScores(input) {
+    const ids = new Set(input.graders.map(grader => grader.id));
+    if (ids.size !== input.graders.length || input.components.length !== input.graders.length)
+        throw new Error("Grade population differs from its declared graders.");
+    const components = new Map(input.components.map(item => [item.graderId, item]));
+    if (components.size !== input.components.length)
+        throw new Error("Grade population contains duplicate graders.");
+    let weighted = 0;
+    let totalWeight = 0;
+    let hardGateFailed = false;
+    let passed = true;
+    let eligible = false;
+    let unavailable = false;
+    let infrastructure = !!input.infrastructureError;
+    for (const grader of input.graders) {
+        const item = components.get(grader.id);
+        if (!item || item.graderVersion !== grader.version)
+            throw new Error("Grade identity differs from its declared grader.");
+        if (!Number.isFinite(grader.weight) || grader.weight < 0 || (item.score !== null && (!Number.isFinite(item.score) || item.score < 0 || item.score > 1)))
+            throw new Error("Grade scores and weights must be finite and within their declared range.");
+        infrastructure ||= item.failureClass === "infrastructure_failure";
+        unavailable ||= item.score === null || (item.failureClass !== null && item.failureClass !== "policy_failure");
+        hardGateFailed ||= grader.hardGate && !item.passed;
+        passed &&= item.passed;
+        eligible ||= grader.rewardEligible && item.rewardEligible;
+        totalWeight += grader.weight;
+        weighted += (item.score ?? 0) * grader.weight;
+    }
+    const score = infrastructure || unavailable || totalWeight <= 0 ? null : hardGateFailed ? 0 : weighted / totalWeight;
+    passed = score !== null && !hardGateFailed && passed;
+    return GradeSummarySchema.parse({ score, passed, rewardEligible: score !== null && eligible,
+        failureClass: infrastructure ? "infrastructure_failure" : score === null ? "grader_failure" : passed ? null : "policy_failure",
+        gradingStatus: !input.graders.length && !infrastructure ? "not_configured" : score === null ? "unscorable" : "scored" });
+}
+export const TaskGradeSchema = GradeSummarySchema.extend({
+    schemaVersion: z.literal("openpond.taskGrade.v1"),
+    taskHash: ReleaseHashSchema, evidenceHash: ReleaseHashSchema, graderSetHash: ReleaseHashSchema,
+    components: z.array(GraderEvidenceSchema).max(1_000), contentHash: ReleaseHashSchema,
+}).strict();
+export async function gradeTaskEvidence(input) {
+    const components = await gradeEvidence(input);
+    const content = { schemaVersion: "openpond.taskGrade.v1",
+        taskHash: contentHash(input.task), evidenceHash: contentHash(input.evidence), graderSetHash: contentHash(input.graders), components,
+        ...aggregateGraderScores({ graders: input.graders, components, infrastructureError: input.evidence.infrastructureError }) };
+    return TaskGradeSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+/** Verify retained results against the exact admitted inputs without rerunning
+ * graders. The storage owner supplies the trusted artifact, not the caller. */
+export function verifyTaskGrade(value, input) {
+    const result = TaskGradeSchema.parse(value);
+    const { contentHash: hash, ...content } = result;
+    if (hash !== contentHash(content) || result.taskHash !== contentHash(input.task) || result.evidenceHash !== contentHash(input.evidence) || result.graderSetHash !== contentHash(input.graders))
+        throw new Error("Task grade differs from its admitted task, evidence or graders.");
+    for (const component of result.components) {
+        const { contentHash: componentHash, ...body } = component;
+        if (componentHash !== contentHash(body))
+            throw new Error("Grader evidence integrity failed.");
+    }
+    const summary = aggregateGraderScores({ graders: input.graders, components: result.components, infrastructureError: input.evidence.infrastructureError });
+    if (contentHash(GradeSummarySchema.parse({ score: result.score, passed: result.passed, rewardEligible: result.rewardEligible, failureClass: result.failureClass, gradingStatus: result.gradingStatus })) !== contentHash(summary))
+        throw new Error("Task grade summary differs from its grader evidence.");
+    return result;
+}

package/dist/types/graders.d.ts

@@ -68,4 +68,77 @@ export declare function gradeEvidence(input: {
     customVerifier?: CustomVerifierRunner;
 }): Promise<GraderEvidence[]>;
 export type GraderEvidence = z.infer<typeof GraderEvidenceSchema>;
+declare const GradeSummarySchema: z.ZodObject<{
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    gradingStatus: z.ZodEnum<{
+        scored: "scored";
+        unscorable: "unscorable";
+        not_configured: "not_configured";
+    }>;
+}, z.core.$strict>;
+/** Shared ordinary grading policy. Results must match the declared population;
+ * missing checks cannot disappear from the denominator or the hard gates. */
+export declare function aggregateGraderScores(input: {
+    graders: ReadonlyArray<Pick<GraderSpec, "id" | "version" | "weight" | "hardGate" | "rewardEligible">>;
+    components: ReadonlyArray<Pick<GraderEvidence, "graderId" | "graderVersion" | "score" | "passed" | "rewardEligible" | "failureClass">>;
+    infrastructureError?: string | null;
+}): z.infer<typeof GradeSummarySchema>;
+export declare const TaskGradeSchema: z.ZodObject<{
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    gradingStatus: z.ZodEnum<{
+        scored: "scored";
+        unscorable: "unscorable";
+        not_configured: "not_configured";
+    }>;
+    schemaVersion: z.ZodLiteral<"openpond.taskGrade.v1">;
+    taskHash: z.ZodString;
+    evidenceHash: z.ZodString;
+    graderSetHash: z.ZodString;
+    components: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
+        graderId: z.ZodString;
+        graderVersion: z.ZodString;
+        score: z.ZodNullable<z.ZodNumber>;
+        passed: z.ZodBoolean;
+        rewardEligible: z.ZodBoolean;
+        failureClass: z.ZodNullable<z.ZodEnum<{
+            policy_failure: "policy_failure";
+            grader_failure: "grader_failure";
+            environment_failure: "environment_failure";
+            infrastructure_failure: "infrastructure_failure";
+            timeout: "timeout";
+            cancelled: "cancelled";
+        }>>;
+        feedback: z.ZodArray<z.ZodString>;
+        visibleEvidenceRefs: z.ZodArray<z.ZodString>;
+        privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type TaskGrade = z.infer<typeof TaskGradeSchema>;
+export declare function gradeTaskEvidence(input: Parameters<typeof gradeEvidence>[0]): Promise<TaskGrade>;
+/** Verify retained results against the exact admitted inputs without rerunning
+ * graders. The storage owner supplies the trusted artifact, not the caller. */
+export declare function verifyTaskGrade(value: unknown, input: Pick<Parameters<typeof gradeEvidence>[0], "task" | "evidence" | "graders">): TaskGrade;
 //# sourceMappingURL=graders.d.ts.map

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAErG,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA8BD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;kBAIb,CAAC;AAEZ;6EAC6E;AAC7E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,CAAC,CAAC,CAAC;IACtG,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,GAAG,eAAe,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,GAAG,cAAc,CAAC,CAAC,CAAC;IACvI,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAwBrC;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAIjB,CAAC;AACZ,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAMtG;AAED;+EAC+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC,GAAG,SAAS,CAW5I"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -158,7 +158,7 @@
     "zod": "^4.1.13"
   },
   "peerDependencies": {
-    "@openpond/harness": "^0.2.0"
+    "@openpond/harness": "^0.2.0 || ^0.3.0"
   },
   "devDependencies": {
     "@openpond/harness": "workspace:*",