@openpond/evals 0.11.4 → 0.11.5

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./profile-evaluations.js";
 export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-suites.js";
 export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";

package/dist/profile-evaluation-suites.js

@@ -0,0 +1,96 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { ProfileEvaluationCatalogSchema } from "./profile-evaluations.js";
+import { TasksetRunManifestSchema } from "./taskset-run-contract.js";
+export const ProfileEvaluationSuiteRunContentSchema = z.object({
+    schemaVersion: z.literal("openpond.profileEvaluationSuiteRun.v1"),
+    id: ReleaseIdSchema,
+    suiteId: ReleaseIdSchema,
+    suiteHash: ReleaseHashSchema,
+    catalogHash: ReleaseHashSchema,
+    profileId: ReleaseIdSchema,
+    sourceRevision: z.string().trim().min(1).max(240),
+    harnessRelease: ImmutableReleaseRefSchema,
+    members: z.array(z.object({
+        definitionId: ReleaseIdSchema,
+        runManifest: ImmutableReleaseRefSchema,
+        runHash: ReleaseHashSchema,
+        passed: z.boolean(),
+        score: z.number().min(0).max(1).nullable(),
+    }).strict()).min(1).max(1_000),
+    passed: z.boolean(),
+    createdAt: ReleaseTimestampSchema,
+    completedAt: ReleaseTimestampSchema,
+}).strict();
+export const ProfileEvaluationSuiteRunSchema = ProfileEvaluationSuiteRunContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+/** Suite membership preserves each definition's own Taskset and metric. A
+ * suite passes only when every declared check passes; unlike scores are never
+ * averaged. */
+export function createProfileEvaluationSuiteRun(input) {
+    const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
+    const suite = catalog.suites.find((candidate) => candidate.id === input.suiteId);
+    if (!suite)
+        throw new Error(`Profile evaluation suite ${input.suiteId} is absent from its catalog.`);
+    if (input.members.length !== suite.definitionIds.length) {
+        throw new Error("Profile evaluation suite has an incomplete run population.");
+    }
+    const parsed = input.members.map((member, index) => {
+        const definitionId = suite.definitionIds[index];
+        const definition = catalog.definitions.find((candidate) => candidate.id === definitionId);
+        if (!definition || member.definitionId !== definitionId) {
+            throw new Error("Profile evaluation suite members differ from declared definition order.");
+        }
+        const manifest = TasksetRunManifestSchema.parse(member.manifest);
+        assertContentHash(manifest, "Profile evaluation suite member manifest");
+        const source = manifest.profileEvaluation;
+        if (!source || manifest.execution.kind !== "harness"
+            || source.definitionId !== definitionId
+            || source.definitionHash !== contentHash(definition)
+            || contentHash(source.target) !== contentHash(definition.target)
+            || source.catalogHash !== contentHash(catalog)
+            || manifest.tasksetRelease.id !== definition.tasksetRelease.id
+            || manifest.tasksetRelease.contentHash !== definition.tasksetRelease.contentHash) {
+            throw new Error("Profile evaluation suite member differs from its released definition.");
+        }
+        return { ...member, manifest, source };
+    });
+    const first = parsed[0].source;
+    const firstPolicy = parsed[0].manifest.policy;
+    if (firstPolicy.kind !== "model")
+        throw new Error("Profile evaluation suite requires model runs.");
+    for (const member of parsed) {
+        if (member.source.profileId !== first.profileId
+            || member.source.sourceRevision !== first.sourceRevision
+            || member.source.harnessRelease.id !== first.harnessRelease.id
+            || member.source.harnessRelease.contentHash !== first.harnessRelease.contentHash) {
+            throw new Error("Profile evaluation suite members use different Profile releases.");
+        }
+        const policy = member.manifest.policy;
+        if (policy.kind !== "model" || contentHash(policy) !== contentHash(firstPolicy)) {
+            throw new Error("Profile evaluation suite members use different model configurations.");
+        }
+    }
+    const content = ProfileEvaluationSuiteRunContentSchema.parse({
+        schemaVersion: "openpond.profileEvaluationSuiteRun.v1",
+        id: input.id,
+        suiteId: suite.id,
+        suiteHash: contentHash(suite),
+        catalogHash: contentHash(catalog),
+        profileId: first.profileId,
+        sourceRevision: first.sourceRevision,
+        harnessRelease: first.harnessRelease,
+        members: parsed.map((member) => ({
+            definitionId: member.definitionId,
+            runManifest: { id: member.manifest.id, contentHash: member.manifest.contentHash },
+            runHash: member.runHash,
+            passed: member.passed,
+            score: member.score,
+        })),
+        passed: parsed.every((member) => member.passed),
+        createdAt: input.createdAt,
+        completedAt: input.completedAt,
+    });
+    return ProfileEvaluationSuiteRunSchema.parse({ ...content, contentHash: contentHash(content) });
+}

package/dist/types/index.d.ts

@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./profile-evaluations.js";
 export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-suites.js";
 export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/profile-evaluation-suites.d.ts

@@ -0,0 +1,75 @@
+import { z } from "zod";
+import { type ProfileEvaluationCatalog } from "./profile-evaluations.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
+export declare const ProfileEvaluationSuiteRunContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
+    id: z.ZodString;
+    suiteId: z.ZodString;
+    suiteHash: z.ZodString;
+    catalogHash: z.ZodString;
+    profileId: z.ZodString;
+    sourceRevision: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    members: z.ZodArray<z.ZodObject<{
+        definitionId: z.ZodString;
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        runHash: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    passed: z.ZodBoolean;
+    createdAt: z.ZodString;
+    completedAt: z.ZodString;
+}, z.core.$strict>;
+export declare const ProfileEvaluationSuiteRunSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationSuiteRun.v1">;
+    id: z.ZodString;
+    suiteId: z.ZodString;
+    suiteHash: z.ZodString;
+    catalogHash: z.ZodString;
+    profileId: z.ZodString;
+    sourceRevision: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    members: z.ZodArray<z.ZodObject<{
+        definitionId: z.ZodString;
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        runHash: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    passed: z.ZodBoolean;
+    createdAt: z.ZodString;
+    completedAt: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileEvaluationSuiteRun = z.infer<typeof ProfileEvaluationSuiteRunSchema>;
+/** Suite membership preserves each definition's own Taskset and metric. A
+ * suite passes only when every declared check passes; unlike scores are never
+ * averaged. */
+export declare function createProfileEvaluationSuiteRun(input: {
+    id: string;
+    suiteId: string;
+    catalog: ProfileEvaluationCatalog;
+    members: Array<{
+        definitionId: string;
+        manifest: TasksetRunManifest;
+        runHash: string;
+        passed: boolean;
+        score: number | null;
+    }>;
+    createdAt: string;
+    completedAt: string;
+}): ProfileEvaluationSuiteRun;
+//# sourceMappingURL=profile-evaluation-suites.d.ts.map

package/dist/types/profile-evaluation-suites.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluation-suites.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-suites.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAAkC,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzG,OAAO,EAA4B,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE9F,eAAO,MAAM,sCAAsC;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBxC,CAAC;AACZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEjC,CAAC;AACZ,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,CAAC;AAExF;;eAEe;AACf,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB,GAAG,yBAAyB,CA+D5B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.4",
+  "version": "0.11.5",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",