@openpond/evals 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/profile-evaluation-execution.js +24 -2
- package/dist/types/learning/repository.d.ts +1 -1
- package/dist/types/learning/transport.d.ts +2 -2
- package/dist/types/profile-evaluation-execution.d.ts +10 -0
- package/dist/types/profile-evaluation-execution.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ImmutableArtifactRefSchema, contentHash } from "@openpond/harness";
|
|
2
|
-
import { gradeTaskEvidence } from "./graders.js";
|
|
2
|
+
import { TaskGradeSchema, gradeTaskEvidence } from "./graders.js";
|
|
3
3
|
import { aggregateTasksetRunReceipts, assertTasksetMetricSource } from "./metrics.js";
|
|
4
|
-
import { createAttemptReceipt } from "./runs.js";
|
|
4
|
+
import { createAttemptReceipt, verifyAttemptReceipt } from "./runs.js";
|
|
5
5
|
import { assertProfileEvaluationRunAdmission } from "./taskset-run-contract.js";
|
|
6
6
|
import { policyTaskView } from "./tasksets.js";
|
|
7
7
|
/** The installation owns execution, tools, approvals and storage. This runner
|
|
@@ -17,6 +17,28 @@ export async function executeProfileEvaluationRun(input) {
|
|
|
17
17
|
for (const member of input.manifest.population) {
|
|
18
18
|
input.signal?.throwIfAborted();
|
|
19
19
|
const task = tasks.get(member.taskId);
|
|
20
|
+
const retained = await input.loadCompletedMember?.(member) ?? null;
|
|
21
|
+
if (retained) {
|
|
22
|
+
const { receipt, grade } = retained;
|
|
23
|
+
const { contentHash: gradeHash, ...gradeContent } = TaskGradeSchema.parse(grade);
|
|
24
|
+
if (!verifyAttemptReceipt(receipt)
|
|
25
|
+
|| gradeHash !== contentHash(gradeContent)
|
|
26
|
+
|| receipt.id !== member.receiptId
|
|
27
|
+
|| receipt.runManifest.id !== input.manifest.id
|
|
28
|
+
|| receipt.runManifest.contentHash !== input.manifest.contentHash
|
|
29
|
+
|| receipt.taskId !== member.taskId || receipt.seed !== member.seed
|
|
30
|
+
|| receipt.graderEvidenceRefs.length !== 1
|
|
31
|
+
|| receipt.graderEvidenceRefs[0]?.contentHash !== gradeHash
|
|
32
|
+
|| grade.taskHash !== contentHash(task)
|
|
33
|
+
|| grade.graderSetHash !== contentHash(input.taskset.graders)
|
|
34
|
+
|| receipt.metadata.score !== grade.score
|
|
35
|
+
|| receipt.metadata.passed !== grade.passed) {
|
|
36
|
+
throw new Error(`Retained Profile evaluation member ${member.receiptId} differs from its admitted evidence.`);
|
|
37
|
+
}
|
|
38
|
+
grades.push(grade);
|
|
39
|
+
receipts.push(receipt);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
20
42
|
const execution = await input.execute({
|
|
21
43
|
task: policyTaskView(task), seed: member.seed, source,
|
|
22
44
|
...(input.signal ? { signal: input.signal } : {}),
|
|
@@ -1870,10 +1870,10 @@ export declare const LearningResourceKindSchema: z.ZodEnum<{
|
|
|
1870
1870
|
source: "source";
|
|
1871
1871
|
asset: "asset";
|
|
1872
1872
|
decision: "decision";
|
|
1873
|
+
grade: "grade";
|
|
1873
1874
|
binding: "binding";
|
|
1874
1875
|
dispatch: "dispatch";
|
|
1875
1876
|
batch: "batch";
|
|
1876
|
-
grade: "grade";
|
|
1877
1877
|
schedule: "schedule";
|
|
1878
1878
|
definition: "definition";
|
|
1879
1879
|
draft: "draft";
|
|
@@ -1573,10 +1573,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
|
|
|
1573
1573
|
source: "source";
|
|
1574
1574
|
asset: "asset";
|
|
1575
1575
|
decision: "decision";
|
|
1576
|
+
grade: "grade";
|
|
1576
1577
|
binding: "binding";
|
|
1577
1578
|
dispatch: "dispatch";
|
|
1578
1579
|
batch: "batch";
|
|
1579
|
-
grade: "grade";
|
|
1580
1580
|
schedule: "schedule";
|
|
1581
1581
|
definition: "definition";
|
|
1582
1582
|
draft: "draft";
|
|
@@ -1601,10 +1601,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
|
|
|
1601
1601
|
source: "source";
|
|
1602
1602
|
asset: "asset";
|
|
1603
1603
|
decision: "decision";
|
|
1604
|
+
grade: "grade";
|
|
1604
1605
|
binding: "binding";
|
|
1605
1606
|
dispatch: "dispatch";
|
|
1606
1607
|
batch: "batch";
|
|
1607
|
-
grade: "grade";
|
|
1608
1608
|
schedule: "schedule";
|
|
1609
1609
|
definition: "definition";
|
|
1610
1610
|
draft: "draft";
|
|
@@ -30,6 +30,16 @@ export declare function executeProfileEvaluationRun(input: {
|
|
|
30
30
|
}>;
|
|
31
31
|
saveGrade: (grade: TaskGrade) => Promise<ImmutableArtifactRef>;
|
|
32
32
|
saveReceipt: (receipt: AttemptReceipt) => Promise<void>;
|
|
33
|
+
/** A retained complete member can be reused after an interrupted run. The
|
|
34
|
+
* runner checks its immutable manifest, task, grader, and grade references. */
|
|
35
|
+
loadCompletedMember?: (member: {
|
|
36
|
+
receiptId: string;
|
|
37
|
+
taskId: string;
|
|
38
|
+
seed: string;
|
|
39
|
+
}) => Promise<{
|
|
40
|
+
receipt: AttemptReceipt;
|
|
41
|
+
grade: TaskGrade;
|
|
42
|
+
} | null>;
|
|
33
43
|
modelJudge?: ModelJudgeRunner;
|
|
34
44
|
customVerifier?: CustomVerifierRunner;
|
|
35
45
|
metricSource?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,
|
|
1
|
+
{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAsC,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAC1J,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAA8C,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5F,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAErG,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD;mFAC+E;IAC/E,mBAAmB,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;QAC7F,OAAO,EAAE,cAAc,CAAC;QACxB,KAAK,EAAE,SAAS,CAAC;KAClB,GAAG,IAAI,CAAC,CAAC;IACV,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAwFD"}
|
package/package.json
CHANGED