@openpond/evals 0.12.7 → 0.12.8

package/RELEASING.md

@@ -36,3 +36,9 @@ leave the lockfile unchanged. The helper reports the selected path before commit
 A dry run requires clean, current master and prints the commands without changing
 Git or package versions. Existing trusted publishing, registry verification,
 package tags, and retry behavior are unchanged.
+
+Version `0.12.8` gives newly authored Reward support files content-addressed
+paths so training and evaluation Rewards, including different revisions of one
+Reward, can coexist in one private export. Existing published assets and checked
+Reward revisions remain immutable; edit, recheck and publish a new revision to
+use the new paths. Asset visibility and reference-integrity checks are unchanged.

package/dist/learning/reward-authoring.js

@@ -5,6 +5,7 @@ import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
 import { LearningJsonObjectSchema } from "./contracts.js";
 import { LearningDomainError } from "./errors.js";
 import { HumanFormSchema } from "../human-review/contracts.js";
+import { sha256 } from "@openpond/harness";
 export const DEFAULT_REWARD_VERIFIER_SOURCE = `export function verify({ output, expectedOutput }) {
   if (!expectedOutput || !Object.hasOwn(expectedOutput, "answer")) {
     throw new Error("An expected answer is required.");
@@ -19,19 +20,19 @@ export function compileRewardAuthoring(input) {
     const base = input.base ? RewardReleaseSchema.parse(input.base) : null;
     if (base && base.id !== input.id)
         throw new LearningDomainError("reward_authoring_base_mismatch", 422);
-    const fixtureAsset = fields.fixtures?.length ? createLearningTextAsset({ text: JSON.stringify(fields.fixtures), path: "reward-fixtures.json", mediaType: "application/json", visibility: "verifier" }) : null;
+    const fixtureAsset = fields.fixtures?.length ? rewardAsset(JSON.stringify(fields.fixtures), "reward-fixtures.json", "application/json") : null;
     let asset = null;
     let implementation;
     let calibrationCheckRef;
     if (fields.kind === "custom_verifier") {
-        asset = createLearningTextAsset({ text: fields.code, path: "verifier.mjs", mediaType: "application/javascript", visibility: "verifier" });
+        asset = rewardAsset(fields.code, "verifier.mjs", "application/javascript");
         implementation = { kind: fields.kind, verifierRef: asset.asset, exportName: fields.exportName, timeoutMs: authoringNumber(fields.timeout, "Time limit"), networkPolicy: "none",
             ...((fields.verifierRuntime ?? (base?.implementation.kind === "custom_verifier" ? base.implementation.runtime : undefined)) ? { runtime: fields.verifierRuntime ?? (base?.implementation.kind === "custom_verifier" ? base.implementation.runtime : undefined) } : {}) };
     }
     else if (fields.kind === "model_judge" || fields.kind === "human") {
         if (!fields.rubric.trim())
             throw new LearningDomainError("reward_rubric_required", 422, "Write the rubric before checking or publishing this Reward.");
-        asset = createLearningTextAsset({ text: fields.rubric, path: "rubric.md", mediaType: "text/markdown", visibility: "verifier" });
+        asset = rewardAsset(fields.rubric, "rubric.md", "text/markdown");
         if (fields.kind === "human")
             implementation = { kind: fields.kind, rubricRef: asset.asset, reviewerRole: fields.reviewerRole, form: HumanFormSchema.parse(JSON.parse(fields.humanForm || JSON.stringify({ schemaVersion: "openpond.humanForm.v1", mode: "individual", instructions: fields.rubric, criteria: [{ id: "quality", label: "Quality", instructions: fields.rubric, required: true, allowAbstain: true, kind: "score", minimum: 0, maximum: 1, step: 0.1 }] }))) };
         else {
@@ -49,7 +50,7 @@ export function compileRewardAuthoring(input) {
     }
     else if (fields.kind === "learned_model") {
         parseRewardAuthoringObject(fields.inputContract, "Model input contract");
-        asset = createLearningTextAsset({ text: fields.inputContract, path: "input-contract.json", mediaType: "application/json", visibility: "verifier" });
+        asset = rewardAsset(fields.inputContract, "input-contract.json", "application/json");
         implementation = { kind: fields.kind, modelVersion: { id: fields.learnedId, contentHash: fields.learnedHash }, inputContract: asset.asset };
     }
     else if (fields.kind === "state")
@@ -74,6 +75,11 @@ export function compileRewardAuthoring(input) {
     });
     return { reward, assets };
 }
+// Training and evaluation Rewards (including different revisions of one Reward)
+// share a private asset directory. Different bytes must never claim one path.
+function rewardAsset(text, filename, mediaType) {
+    return createLearningTextAsset({ text, path: `reward-assets/${sha256(text)}/${filename}`, mediaType, visibility: "verifier" });
+}
 export function rewardAuthoringFields(reward, sourceAsset, fixtureAsset = null) {
     const implementation = reward?.implementation;
     const ref = implementation && "verifierRef" in implementation ? implementation.verifierRef

package/dist/types/learning/reward-authoring.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reward-authoring.d.ts","sourceRoot":"","sources":["../../../src/learning/reward-authoring.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4C,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAE7F,OAAO,EAAmE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACzH,OAAO,EAAoD,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAKvG,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC1E,eAAO,MAAM,8BAA8B,yWAMzC,CAAC;AAEH,kGAAkG;AAClG,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,aAAa,GAAG,IAAI,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,iBAAiB,EAAE,CAAA;CAAE,CAgD/K;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,EAAE,WAAW,EAAE,iBAAiB,GAAG,IAAI,EAAE,YAAY,GAAE,iBAAiB,GAAG,IAAW,GAAG,qBAAqB,CAgC/K;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,0CAMtE;AACD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpE"}
+{"version":3,"file":"reward-authoring.d.ts","sourceRoot":"","sources":["../../../src/learning/reward-authoring.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4C,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAE7F,OAAO,EAAmE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACzH,OAAO,EAAoD,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAMvG,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC1E,eAAO,MAAM,8BAA8B,yWAMzC,CAAC;AAEH,kGAAkG;AAClG,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,aAAa,GAAG,IAAI,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,iBAAiB,EAAE,CAAA;CAAE,CAgD/K;AAQD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,EAAE,WAAW,EAAE,iBAAiB,GAAG,IAAI,EAAE,YAAY,GAAE,iBAAiB,GAAG,IAAW,GAAG,qBAAqB,CAgC/K;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,0CAMtE;AACD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpE"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.12.7",
+  "version": "0.12.8",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",