@openpond/evals 0.11.3 → 0.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/dist/index.js +1 -0
  2. package/dist/profile-evaluation-execution.js +24 -2
  3. package/dist/profile-evaluation-suites.js +96 -0
  4. package/dist/types/index.d.ts +1 -0
  5. package/dist/types/index.d.ts.map +1 -1
  6. package/dist/types/learning/repository.d.ts +1 -1
  7. package/dist/types/learning/transport.d.ts +2 -2
  8. package/dist/types/profile-evaluation-execution.d.ts +10 -0
  9. package/dist/types/profile-evaluation-execution.d.ts.map +1 -1
  10. package/dist/types/profile-evaluation-suites.d.ts +75 -0
  11. package/dist/types/profile-evaluation-suites.d.ts.map +1 -0
  12. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
14
14
  export * from "./preferences.js";
15
15
  export * from "./profile-evaluations.js";
16
16
  export * from "./profile-evaluation-comparisons.js";
17
+ export * from "./profile-evaluation-suites.js";
17
18
  export * from "./profile-evaluation-execution.js";
18
19
  export * from "./review-conformance.js";
19
20
  export * from "./rollouts.js";
package/dist/profile-evaluation-execution.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { ImmutableArtifactRefSchema, contentHash } from "@openpond/harness";
2
- import { gradeTaskEvidence } from "./graders.js";
2
+ import { TaskGradeSchema, gradeTaskEvidence } from "./graders.js";
3
3
  import { aggregateTasksetRunReceipts, assertTasksetMetricSource } from "./metrics.js";
4
- import { createAttemptReceipt } from "./runs.js";
4
+ import { createAttemptReceipt, verifyAttemptReceipt } from "./runs.js";
5
5
  import { assertProfileEvaluationRunAdmission } from "./taskset-run-contract.js";
6
6
  import { policyTaskView } from "./tasksets.js";
7
7
  /** The installation owns execution, tools, approvals and storage. This runner
@@ -17,6 +17,28 @@ export async function executeProfileEvaluationRun(input) {
17
17
  for (const member of input.manifest.population) {
18
18
  input.signal?.throwIfAborted();
19
19
  const task = tasks.get(member.taskId);
20
+ const retained = await input.loadCompletedMember?.(member) ?? null;
21
+ if (retained) {
22
+ const { receipt, grade } = retained;
23
+ const { contentHash: gradeHash, ...gradeContent } = TaskGradeSchema.parse(grade);
24
+ if (!verifyAttemptReceipt(receipt)
25
+ || gradeHash !== contentHash(gradeContent)
26
+ || receipt.id !== member.receiptId
27
+ || receipt.runManifest.id !== input.manifest.id
28
+ || receipt.runManifest.contentHash !== input.manifest.contentHash
29
+ || receipt.taskId !== member.taskId || receipt.seed !== member.seed
30
+ || receipt.graderEvidenceRefs.length !== 1
31
+ || receipt.graderEvidenceRefs[0]?.contentHash !== gradeHash
32
+ || grade.taskHash !== contentHash(task)
33
+ || grade.graderSetHash !== contentHash(input.taskset.graders)
34
+ || receipt.metadata.score !== grade.score
35
+ || receipt.metadata.passed !== grade.passed) {
36
+ throw new Error(`Retained Profile evaluation member ${member.receiptId} differs from its admitted evidence.`);
37
+ }
38
+ grades.push(grade);
39
+ receipts.push(receipt);
40
+ continue;
41
+ }
20
42
  const execution = await input.execute({
21
43
  task: policyTaskView(task), seed: member.seed, source,
22
44
  ...(input.signal ? { signal: input.signal } : {}),
package/dist/profile-evaluation-suites.js ADDED
@@ -0,0 +1,96 @@
1
+ import { z } from "zod";
2
+ import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
3
+ import { ProfileEvaluationCatalogSchema } from "./profile-evaluations.js";
4
+ import { TasksetRunManifestSchema } from "./taskset-run-contract.js";
5
+ export const ProfileEvaluationSuiteRunContentSchema = z.object({
6
+ schemaVersion: z.literal("openpond.profileEvaluationSuiteRun.v1"),
7
+ id: ReleaseIdSchema,
8
+ suiteId: ReleaseIdSchema,
9
+ suiteHash: ReleaseHashSchema,
10
+ catalogHash: ReleaseHashSchema,
11
+ profileId: ReleaseIdSchema,
12
+ sourceRevision: z.string().trim().min(1).max(240),
13
+ harnessRelease: ImmutableReleaseRefSchema,
14
+ members: z.array(z.object({
15
+ definitionId: ReleaseIdSchema,
16
+ runManifest: ImmutableReleaseRefSchema,
17
+ runHash: ReleaseHashSchema,
18
+ passed: z.boolean(),
19
+ score: z.number().min(0).max(1).nullable(),
20
+ }).strict()).min(1).max(1_000),
21
+ passed: z.boolean(),
22
+ createdAt: ReleaseTimestampSchema,
23
+ completedAt: ReleaseTimestampSchema,
24
+ }).strict();
25
+ export const ProfileEvaluationSuiteRunSchema = ProfileEvaluationSuiteRunContentSchema.extend({
26
+ contentHash: ReleaseHashSchema,
27
+ }).strict();
28
+ /** Suite membership preserves each definition's own Taskset and metric. A
29
+ * suite passes only when every declared check passes; unlike scores are never
30
+ * averaged. */
31
+ export function createProfileEvaluationSuiteRun(input) {
32
+ const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
33
+ const suite = catalog.suites.find((candidate) => candidate.id === input.suiteId);
34
+ if (!suite)
35
+ throw new Error(`Profile evaluation suite ${input.suiteId} is absent from its catalog.`);
36
+ if (input.members.length !== suite.definitionIds.length) {
37
+ throw new Error("Profile evaluation suite has an incomplete run population.");
38
+ }
39
+ const parsed = input.members.map((member, index) => {
40
+ const definitionId = suite.definitionIds[index];
41
+ const definition = catalog.definitions.find((candidate) => candidate.id === definitionId);
42
+ if (!definition || member.definitionId !== definitionId) {
43
+ throw new Error("Profile evaluation suite members differ from declared definition order.");
44
+ }
45
+ const manifest = TasksetRunManifestSchema.parse(member.manifest);
46
+ assertContentHash(manifest, "Profile evaluation suite member manifest");
47
+ const source = manifest.profileEvaluation;
48
+ if (!source || manifest.execution.kind !== "harness"
49
+ || source.definitionId !== definitionId
50
+ || source.definitionHash !== contentHash(definition)
51
+ || contentHash(source.target) !== contentHash(definition.target)
52
+ || source.catalogHash !== contentHash(catalog)
53
+ || manifest.tasksetRelease.id !== definition.tasksetRelease.id
54
+ || manifest.tasksetRelease.contentHash !== definition.tasksetRelease.contentHash) {
55
+ throw new Error("Profile evaluation suite member differs from its released definition.");
56
+ }
57
+ return { ...member, manifest, source };
58
+ });
59
+ const first = parsed[0].source;
60
+ const firstPolicy = parsed[0].manifest.policy;
61
+ if (firstPolicy.kind !== "model")
62
+ throw new Error("Profile evaluation suite requires model runs.");
63
+ for (const member of parsed) {
64
+ if (member.source.profileId !== first.profileId
65
+ || member.source.sourceRevision !== first.sourceRevision
66
+ || member.source.harnessRelease.id !== first.harnessRelease.id
67
+ || member.source.harnessRelease.contentHash !== first.harnessRelease.contentHash) {
68
+ throw new Error("Profile evaluation suite members use different Profile releases.");
69
+ }
70
+ const policy = member.manifest.policy;
71
+ if (policy.kind !== "model" || contentHash(policy) !== contentHash(firstPolicy)) {
72
+ throw new Error("Profile evaluation suite members use different model configurations.");
73
+ }
74
+ }
75
+ const content = ProfileEvaluationSuiteRunContentSchema.parse({
76
+ schemaVersion: "openpond.profileEvaluationSuiteRun.v1",
77
+ id: input.id,
78
+ suiteId: suite.id,
79
+ suiteHash: contentHash(suite),
80
+ catalogHash: contentHash(catalog),
81
+ profileId: first.profileId,
82
+ sourceRevision: first.sourceRevision,
83
+ harnessRelease: first.harnessRelease,
84
+ members: parsed.map((member) => ({
85
+ definitionId: member.definitionId,
86
+ runManifest: { id: member.manifest.id, contentHash: member.manifest.contentHash },
87
+ runHash: member.runHash,
88
+ passed: member.passed,
89
+ score: member.score,
90
+ })),
91
+ passed: parsed.every((member) => member.passed),
92
+ createdAt: input.createdAt,
93
+ completedAt: input.completedAt,
94
+ });
95
+ return ProfileEvaluationSuiteRunSchema.parse({ ...content, contentHash: contentHash(content) });
96
+ }
package/dist/types/index.d.ts CHANGED
@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
14
14
  export * from "./preferences.js";
15
15
  export * from "./profile-evaluations.js";
16
16
  export * from "./profile-evaluation-comparisons.js";
17
+ export * from "./profile-evaluation-suites.js";
17
18
  export * from "./profile-evaluation-execution.js";
18
19
  export * from "./review-conformance.js";
19
20
  export * from "./rollouts.js";
package/dist/types/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
package/dist/types/learning/repository.d.ts CHANGED
@@ -1870,10 +1870,10 @@ export declare const LearningResourceKindSchema: z.ZodEnum<{
1870
1870
  source: "source";
1871
1871
  asset: "asset";
1872
1872
  decision: "decision";
1873
+ grade: "grade";
1873
1874
  binding: "binding";
1874
1875
  dispatch: "dispatch";
1875
1876
  batch: "batch";
1876
- grade: "grade";
1877
1877
  schedule: "schedule";
1878
1878
  definition: "definition";
1879
1879
  draft: "draft";
package/dist/types/learning/transport.d.ts CHANGED
@@ -1573,10 +1573,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
1573
1573
  source: "source";
1574
1574
  asset: "asset";
1575
1575
  decision: "decision";
1576
+ grade: "grade";
1576
1577
  binding: "binding";
1577
1578
  dispatch: "dispatch";
1578
1579
  batch: "batch";
1579
- grade: "grade";
1580
1580
  schedule: "schedule";
1581
1581
  definition: "definition";
1582
1582
  draft: "draft";
@@ -1601,10 +1601,10 @@ export declare const LearningReadRequestSchema: z.ZodDiscriminatedUnion<[z.ZodOb
1601
1601
  source: "source";
1602
1602
  asset: "asset";
1603
1603
  decision: "decision";
1604
+ grade: "grade";
1604
1605
  binding: "binding";
1605
1606
  dispatch: "dispatch";
1606
1607
  batch: "batch";
1607
- grade: "grade";
1608
1608
  schedule: "schedule";
1609
1609
  definition: "definition";
1610
1610
  draft: "draft";
package/dist/types/profile-evaluation-execution.d.ts CHANGED
@@ -30,6 +30,16 @@ export declare function executeProfileEvaluationRun(input: {
30
30
  }>;
31
31
  saveGrade: (grade: TaskGrade) => Promise<ImmutableArtifactRef>;
32
32
  saveReceipt: (receipt: AttemptReceipt) => Promise<void>;
33
+ /** A retained complete member can be reused after an interrupted run. The
34
+ * runner checks its immutable manifest, task, grader, and grade references. */
35
+ loadCompletedMember?: (member: {
36
+ receiptId: string;
37
+ taskId: string;
38
+ seed: string;
39
+ }) => Promise<{
40
+ receipt: AttemptReceipt;
41
+ grade: TaskGrade;
42
+ } | null>;
33
43
  modelJudge?: ModelJudgeRunner;
34
44
  customVerifier?: CustomVerifierRunner;
35
45
  metricSource?: string;
package/dist/types/profile-evaluation-execution.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAqB,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AACzI,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAErG,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAkED"}
1
+ {"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAsC,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAC1J,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAA8C,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5F,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAErG,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD;mFAC+E;IAC/E,mBAAmB,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;QAC7F,OAAO,EAAE,cAAc,CAAC;QACxB,KAAK,EAAE,SAAS,CAAC;KAClB,GAAG,IAAI,CAAC,CAAC;IACV,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAwFD"}
package/dist/types/profile-evaluation-suites.d.ts ADDED
@@ -0,0 +1,75 @@
1
+ import { z } from "zod";
2
+ import { type ProfileEvaluationCatalog } from "./profile-evaluations.js";
3
+ import { type TasksetRunManifest } from "./taskset-run-contract.js";
4
+ export declare const ProfileEvaluationSuiteRunContentSchema: z.ZodObject<{
5
+ schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
6
+ id: z.ZodString;
7
+ suiteId: z.ZodString;
8
+ suiteHash: z.ZodString;
9
+ catalogHash: z.ZodString;
10
+ profileId: z.ZodString;
11
+ sourceRevision: z.ZodString;
12
+ harnessRelease: z.ZodObject<{
13
+ id: z.ZodString;
14
+ contentHash: z.ZodString;
15
+ }, z.core.$strict>;
16
+ members: z.ZodArray<z.ZodObject<{
17
+ definitionId: z.ZodString;
18
+ runManifest: z.ZodObject<{
19
+ id: z.ZodString;
20
+ contentHash: z.ZodString;
21
+ }, z.core.$strict>;
22
+ runHash: z.ZodString;
23
+ passed: z.ZodBoolean;
24
+ score: z.ZodNullable<z.ZodNumber>;
25
+ }, z.core.$strict>>;
26
+ passed: z.ZodBoolean;
27
+ createdAt: z.ZodString;
28
+ completedAt: z.ZodString;
29
+ }, z.core.$strict>;
30
+ export declare const ProfileEvaluationSuiteRunSchema: z.ZodObject<{
31
+ schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
32
+ id: z.ZodString;
33
+ suiteId: z.ZodString;
34
+ suiteHash: z.ZodString;
35
+ catalogHash: z.ZodString;
36
+ profileId: z.ZodString;
37
+ sourceRevision: z.ZodString;
38
+ harnessRelease: z.ZodObject<{
39
+ id: z.ZodString;
40
+ contentHash: z.ZodString;
41
+ }, z.core.$strict>;
42
+ members: z.ZodArray<z.ZodObject<{
43
+ definitionId: z.ZodString;
44
+ runManifest: z.ZodObject<{
45
+ id: z.ZodString;
46
+ contentHash: z.ZodString;
47
+ }, z.core.$strict>;
48
+ runHash: z.ZodString;
49
+ passed: z.ZodBoolean;
50
+ score: z.ZodNullable<z.ZodNumber>;
51
+ }, z.core.$strict>>;
52
+ passed: z.ZodBoolean;
53
+ createdAt: z.ZodString;
54
+ completedAt: z.ZodString;
55
+ contentHash: z.ZodString;
56
+ }, z.core.$strict>;
57
+ export type ProfileEvaluationSuiteRun = z.infer<typeof ProfileEvaluationSuiteRunSchema>;
58
+ /** Suite membership preserves each definition's own Taskset and metric. A
59
+ * suite passes only when every declared check passes; unlike scores are never
60
+ * averaged. */
61
+ export declare function createProfileEvaluationSuiteRun(input: {
62
+ id: string;
63
+ suiteId: string;
64
+ catalog: ProfileEvaluationCatalog;
65
+ members: Array<{
66
+ definitionId: string;
67
+ manifest: TasksetRunManifest;
68
+ runHash: string;
69
+ passed: boolean;
70
+ score: number | null;
71
+ }>;
72
+ createdAt: string;
73
+ completedAt: string;
74
+ }): ProfileEvaluationSuiteRun;
75
+ //# sourceMappingURL=profile-evaluation-suites.d.ts.map
package/dist/types/profile-evaluation-suites.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"profile-evaluation-suites.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-suites.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAAkC,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzG,OAAO,EAA4B,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE9F,eAAO,MAAM,sCAAsC;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBxC,CAAC;AACZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEjC,CAAC;AACZ,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,CAAC;AAExF;;eAEe;AACf,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB,GAAG,yBAAyB,CA+D5B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openpond/evals",
3
- "version": "0.11.3",
3
+ "version": "0.11.5",
4
4
  "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
5
5
  "license": "MIT",
6
6
  "type": "module",