@openpond/evals 0.11.1 → 0.11.2

package/dist/index.js

@@ -12,6 +12,8 @@ export * from "./metrics.js";
 export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
+export * from "./profile-evaluations.js";
+export * from "./profile-evaluation-comparisons.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/profile-evaluation-comparisons.js

@@ -0,0 +1,86 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { assertTasksetMetricResult } from "./metric-policy.js";
+import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
+import { TasksetRunManifestSchema } from "./taskset-run-contract.js";
+export const ProfileEvaluationComparisonContentSchema = z.object({
+    schemaVersion: z.literal("openpond.profileEvaluationComparison.v1"),
+    id: ReleaseIdSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    populationHash: ReleaseHashSchema,
+    environmentHash: ReleaseHashSchema,
+    metricPolicyHash: ReleaseHashSchema,
+    limitsHash: ReleaseHashSchema,
+    members: z.array(z.object({
+        runManifest: ImmutableReleaseRefSchema,
+        metricResultHash: ReleaseHashSchema,
+        source: ProfileEvaluationRunSourceSchema,
+        policy: TasksetRunManifestSchema.shape.policy,
+        score: z.number().min(0).max(1).nullable(),
+    }).strict()).min(2).max(100),
+    createdAt: ReleaseTimestampSchema,
+}).strict();
+export const ProfileEvaluationComparisonSchema = ProfileEvaluationComparisonContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+/** Comparison membership is immutable and refers to existing run evidence.
+ * The same Taskset, population, environment, metric and limits are required;
+ * model and released Profile source may vary independently. */
+export function createProfileEvaluationComparison(input) {
+    if (input.members.length < 2)
+        throw new Error("A comparison requires at least two runs.");
+    const parsed = input.members.map(({ manifest, result }) => {
+        TasksetRunManifestSchema.parse(manifest);
+        assertContentHash(manifest, "Taskset run manifest");
+        assertTasksetMetricResult(result);
+        if (!manifest.profileEvaluation || manifest.execution.kind !== "harness") {
+            throw new Error("Profile comparison requires released Profile evaluation runs.");
+        }
+        if (result.runManifest.id !== manifest.id || result.runManifest.contentHash !== manifest.contentHash
+            || result.tasksetRelease.id !== manifest.tasksetRelease.id
+            || result.tasksetRelease.contentHash !== manifest.tasksetRelease.contentHash
+            || result.policyHash !== contentHash(manifest.metricPolicy)) {
+            throw new Error("Profile comparison metric differs from its run manifest.");
+        }
+        return { manifest, result, source: manifest.profileEvaluation };
+    });
+    const baseline = parsed[0];
+    const populationHash = contentHash(baseline.manifest.population.map(({ taskId, seed, fixtureId }) => ({ taskId, seed, fixtureId })));
+    const metricPolicyHash = contentHash(baseline.manifest.metricPolicy);
+    const limitsHash = contentHash(baseline.manifest.limits);
+    const ids = new Set();
+    for (const member of parsed) {
+        if (ids.has(member.manifest.id))
+            throw new Error("Profile comparison repeats a run.");
+        ids.add(member.manifest.id);
+        if (member.manifest.tasksetRelease.id !== baseline.manifest.tasksetRelease.id
+            || member.manifest.tasksetRelease.contentHash !== baseline.manifest.tasksetRelease.contentHash
+            || contentHash(member.manifest.population.map(({ taskId, seed, fixtureId }) => ({ taskId, seed, fixtureId }))) !== populationHash
+            || member.source.environmentHash !== baseline.source.environmentHash
+            || contentHash(member.manifest.metricPolicy) !== metricPolicyHash
+            || contentHash(member.manifest.limits) !== limitsHash
+            || member.manifest.runtimeTarget.adapterId !== baseline.manifest.runtimeTarget.adapterId
+            || member.manifest.runtimeTarget.placement !== baseline.manifest.runtimeTarget.placement
+            || member.manifest.runtimeTarget.runtimeVersion !== baseline.manifest.runtimeTarget.runtimeVersion) {
+            throw new Error("Profile comparison runs differ in frozen tasks, environment, grading, limits, or runtime.");
+        }
+    }
+    const content = ProfileEvaluationComparisonContentSchema.parse({
+        schemaVersion: "openpond.profileEvaluationComparison.v1",
+        id: input.id,
+        tasksetRelease: baseline.manifest.tasksetRelease,
+        populationHash,
+        environmentHash: baseline.source.environmentHash,
+        metricPolicyHash,
+        limitsHash,
+        members: parsed.map(({ manifest, result, source }) => ({
+            runManifest: { id: manifest.id, contentHash: manifest.contentHash },
+            metricResultHash: result.contentHash,
+            source,
+            policy: manifest.policy,
+            score: result.value,
+        })),
+        createdAt: input.createdAt,
+    });
+    return ProfileEvaluationComparisonSchema.parse({ ...content, contentHash: contentHash(content) });
+}

package/dist/profile-evaluations.js

@@ -0,0 +1,153 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash, harnessSourcePackageFiles, loadReleasedProfileWorkflowCatalogAssets, sha256, validateHarnessSourcePackage, } from "@openpond/harness";
+import { TaskSplitSchema } from "./tasksets.js";
+const RelativePathSchema = z.string().min(1).max(2_000).refine((value) => !value.includes("\\") && !value.includes(":") && !value.startsWith("/")
+    && value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), "evaluation paths must be portable and relative");
+/** Portable selection only. Taskset releases own task data and graders; run
+ * manifests and evidence stay in the installation's evaluation store. */
+export const ProfileEvaluationTargetSchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("profile") }).strict(),
+    z.object({ kind: z.literal("workflow"), workflowId: ReleaseIdSchema }).strict(),
+    z.object({ kind: z.literal("skill"), skillPath: RelativePathSchema }).strict(),
+    z.object({ kind: z.literal("agent_action"), actionId: ReleaseIdSchema }).strict(),
+]);
+export const ProfileEvaluationDefinitionSchema = z.object({
+    id: ReleaseIdSchema,
+    label: z.string().trim().min(1).max(240),
+    description: z.string().max(4_000),
+    target: ProfileEvaluationTargetSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    split: TaskSplitSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(100_000),
+    seeds: z.array(z.string().trim().min(1).max(500)).min(1).max(100),
+    criterion: z.object({
+        minimumPassRate: z.number().min(0).max(1),
+        requireComplete: z.boolean(),
+    }).strict(),
+}).strict().superRefine((definition, context) => {
+    if (new Set(definition.taskIds).size !== definition.taskIds.length) {
+        context.addIssue({ code: "custom", path: ["taskIds"], message: "Evaluation task identities must be unique." });
+    }
+    if (new Set(definition.seeds).size !== definition.seeds.length) {
+        context.addIssue({ code: "custom", path: ["seeds"], message: "Evaluation seeds must be unique." });
+    }
+});
+export const ProfileEvaluationSuiteSchema = z.object({
+    id: ReleaseIdSchema,
+    label: z.string().trim().min(1).max(240),
+    scope: z.enum(["component", "profile"]),
+    definitionIds: z.array(ReleaseIdSchema).min(1).max(1_000),
+}).strict();
+export const ProfileEvaluationCatalogSchema = z.object({
+    schemaVersion: z.literal("openpond.profileEvaluations.v1"),
+    definitions: z.array(ProfileEvaluationDefinitionSchema).max(1_000),
+    suites: z.array(ProfileEvaluationSuiteSchema).max(100),
+}).strict();
+/** Added to a Taskset run manifest only when a Profile component supplies the
+ * candidate behavior. It does not change the Taskset's frozen cases/graders. */
+export const ProfileEvaluationRunSourceSchema = z.object({
+    profileId: ReleaseIdSchema,
+    sourceRevision: z.string().trim().min(1).max(240),
+    harnessRelease: ImmutableReleaseRefSchema,
+    catalogHash: ReleaseHashSchema,
+    definitionId: ReleaseIdSchema,
+    definitionHash: ReleaseHashSchema,
+    target: ProfileEvaluationTargetSchema,
+    environmentHash: ReleaseHashSchema,
+}).strict();
+/** Validate component and suite references against the same Profile release. */
+export function validateProfileEvaluationCatalog(input) {
+    const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
+    const ids = new Set();
+    for (const definition of catalog.definitions) {
+        if (ids.has(definition.id))
+            throw new Error(`Duplicate Profile evaluation ${definition.id}.`);
+        ids.add(definition.id);
+        switch (definition.target.kind) {
+            case "workflow":
+                if (!input.workflowIds.has(definition.target.workflowId))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing workflow.`);
+                break;
+            case "skill":
+                if (!input.skillPaths.has(definition.target.skillPath))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing Skill.`);
+                break;
+            case "agent_action":
+                if (!input.actionIds.has(definition.target.actionId))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing Agent action.`);
+                break;
+        }
+    }
+    const suiteIds = new Set();
+    for (const suite of catalog.suites) {
+        if (suiteIds.has(suite.id))
+            throw new Error(`Duplicate Profile evaluation suite ${suite.id}.`);
+        suiteIds.add(suite.id);
+        if (new Set(suite.definitionIds).size !== suite.definitionIds.length)
+            throw new Error(`Profile evaluation suite ${suite.id} repeats a definition.`);
+        for (const definitionId of suite.definitionIds) {
+            if (!ids.has(definitionId))
+                throw new Error(`Profile evaluation suite ${suite.id} references missing definition ${definitionId}.`);
+        }
+        if (suite.scope === "profile" && suite.definitionIds.every((id) => catalog.definitions.find((definition) => definition.id === id)?.target.kind !== "profile")) {
+            throw new Error(`Profile evaluation suite ${suite.id} needs an end-to-end Profile definition.`);
+        }
+    }
+    return { catalog, contentHash: contentHash(catalog) };
+}
+export function resolveProfileEvaluationRunSource(input) {
+    const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
+    const definition = catalog.definitions.find((candidate) => candidate.id === input.definitionId);
+    if (!definition)
+        throw new Error(`Profile evaluation ${input.definitionId} is absent from its catalog.`);
+    return ProfileEvaluationRunSourceSchema.parse({
+        profileId: input.profileId,
+        sourceRevision: input.sourceRevision,
+        harnessRelease: input.harnessRelease,
+        catalogHash: contentHash(catalog),
+        definitionId: definition.id,
+        definitionHash: contentHash(definition),
+        target: definition.target,
+        environmentHash: input.environmentHash,
+    });
+}
+/** Evaluation definitions are read from the verifier-only bytes of an exact
+ * released source package. The caller must never forward this asset to the
+ * model-visible policy surface. */
+export function loadReleasedProfileEvaluationCatalog(value) {
+    const source = validateHarnessSourcePackage(value);
+    const files = harnessSourcePackageFiles(source);
+    const result = loadReleasedProfileEvaluationCatalogAssets({
+        agentSnapshot: source.agentSnapshot,
+        harnessRelease: source.harnessRelease,
+        catalogBytes: files.get("evals/catalog.json"),
+        workflowBytes: files.get("workflows/catalog.json"),
+        actionBytes: files.get("workflows/actions.json"),
+    });
+    return { ...result, harnessRelease: {
+            id: source.harnessRelease.id, contentHash: source.harnessRelease.contentHash,
+        } };
+}
+/** For a local release whose caller already verified its full file inventory. */
+export function loadReleasedProfileEvaluationCatalogAssets(input) {
+    const asset = input.harnessRelease.files.find((file) => file.path === "evals/catalog.json");
+    const bytes = input.catalogBytes;
+    if (!asset || asset.visibility !== "verifier" || !bytes || sha256(bytes) !== asset.contentHash) {
+        throw new Error("Released Profile evaluation catalog is unavailable or not verifier-private.");
+    }
+    const workflows = input.workflowBytes || input.actionBytes
+        ? loadReleasedProfileWorkflowCatalogAssets({
+            agentSnapshot: input.agentSnapshot,
+            harnessRelease: input.harnessRelease,
+            catalogBytes: input.workflowBytes,
+            actionBytes: input.actionBytes,
+        })
+        : null;
+    const { catalog, contentHash: catalogHash } = validateProfileEvaluationCatalog({
+        catalog: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)),
+        workflowIds: new Set(workflows?.catalog.workflows.map((workflow) => workflow.id) ?? []),
+        skillPaths: new Set(input.agentSnapshot.skills.map((skill) => skill.path)),
+        actionIds: new Set(workflows?.actions.map((action) => action.id) ?? []),
+    });
+    return { catalog, catalogHash };
+}

package/dist/taskset-run-contract.js

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { assertContentHash, contentHash, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
 import { RunLimitsSchema, RuntimeTargetBindingSchema, verifyAttemptReceipt } from "./runs.js";
 import { TasksetMetricPolicySchema } from "./metric-policy.js";
+import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
 import { assertTasksetRelease } from "./tasksets.js";
 /** A fixture check owns no model identity and cannot become a model evaluation. */
 export const TasksetRunPolicySchema = z.discriminatedUnion("kind", [
@@ -24,6 +25,7 @@ export const TasksetRunManifestContentSchema = z.object({
     tasksetRelease: ImmutableReleaseRefSchema,
     packageHash: ReleaseHashSchema,
     execution: TasksetRunExecutionSchema,
+    profileEvaluation: ProfileEvaluationRunSourceSchema.optional(),
     policy: TasksetRunPolicySchema,
     gradingRole: z.literal("evaluation"),
     metricPolicy: TasksetMetricPolicySchema,
@@ -37,6 +39,11 @@ export const TasksetRunManifestContentSchema = z.object({
         context.addIssue({ code: "custom", path: ["population"], message: "Run receipt identities must be unique." });
     if (value.population.some(member => (member.fixtureId !== null) !== (value.policy.kind === "fixture")))
         context.addIssue({ code: "custom", path: ["population"], message: "Only fixture runs require a fixture identity for every member." });
+    if (value.profileEvaluation && (value.execution.kind !== "harness"
+        || value.execution.harnessRelease.id !== value.profileEvaluation.harnessRelease.id
+        || value.execution.harnessRelease.contentHash !== value.profileEvaluation.harnessRelease.contentHash)) {
+        context.addIssue({ code: "custom", path: ["profileEvaluation"], message: "Profile evaluation must execute its exact bound Harness release." });
+    }
 });
 export const TasksetRunManifestSchema = TasksetRunManifestContentSchema.safeExtend({ contentHash: ReleaseHashSchema });
 export function createTasksetRunManifest(input) {

package/dist/types/index.d.ts

@@ -12,6 +12,8 @@ export * from "./metrics.js";
 export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
+export * from "./profile-evaluations.js";
+export * from "./profile-evaluation-comparisons.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/profile-evaluation-comparisons.d.ts

@@ -0,0 +1,135 @@
+import { z } from "zod";
+import { type TasksetMetricResult } from "./metric-policy.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
+export declare const ProfileEvaluationComparisonContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationComparison.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    populationHash: z.ZodString;
+    environmentHash: z.ZodString;
+    metricPolicyHash: z.ZodString;
+    limitsHash: z.ZodString;
+    members: z.ZodArray<z.ZodObject<{
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        metricResultHash: z.ZodString;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+        policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"model">;
+            model: z.ZodObject<{
+                provider: z.ZodString;
+                model: z.ZodString;
+                revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            }, z.core.$strict>;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"fixture">;
+        }, z.core.$strict>], "kind">;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+}, z.core.$strict>;
+export declare const ProfileEvaluationComparisonSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationComparison.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    populationHash: z.ZodString;
+    environmentHash: z.ZodString;
+    metricPolicyHash: z.ZodString;
+    limitsHash: z.ZodString;
+    members: z.ZodArray<z.ZodObject<{
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        metricResultHash: z.ZodString;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+        policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"model">;
+            model: z.ZodObject<{
+                provider: z.ZodString;
+                model: z.ZodString;
+                revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            }, z.core.$strict>;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"fixture">;
+        }, z.core.$strict>], "kind">;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileEvaluationComparison = z.infer<typeof ProfileEvaluationComparisonSchema>;
+/** Comparison membership is immutable and refers to existing run evidence.
+ * The same Taskset, population, environment, metric and limits are required;
+ * model and released Profile source may vary independently. */
+export declare function createProfileEvaluationComparison(input: {
+    id: string;
+    members: Array<{
+        manifest: TasksetRunManifest;
+        result: TasksetMetricResult;
+    }>;
+    createdAt: string;
+}): ProfileEvaluationComparison;
+//# sourceMappingURL=profile-evaluation-comparisons.d.ts.map

package/dist/types/profile-evaluation-comparisons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluation-comparisons.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-comparisons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzF,OAAO,EAA4B,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE9F,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgB1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AACZ,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC;AAE5F;;+DAE+D;AAC/D,wBAAgB,iCAAiC,CAAC,KAAK,EAAE;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,MAAM,EAAE,mBAAmB,CAAA;KAAE,CAAC,CAAC;IAC9E,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,2BAA2B,CAuD9B"}

package/dist/types/profile-evaluations.d.ts

@@ -0,0 +1,176 @@
+import { z } from "zod";
+import { type AgentSnapshot, type HarnessRelease } from "@openpond/harness";
+/** Portable selection only. Taskset releases own task data and graders; run
+ * manifests and evidence stay in the installation's evaluation store. */
+export declare const ProfileEvaluationTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"profile">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"workflow">;
+    workflowId: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"skill">;
+    skillPath: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"agent_action">;
+    actionId: z.ZodString;
+}, z.core.$strict>], "kind">;
+export declare const ProfileEvaluationDefinitionSchema: z.ZodObject<{
+    id: z.ZodString;
+    label: z.ZodString;
+    description: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    split: z.ZodEnum<{
+        train: "train";
+        validation: "validation";
+        test: "test";
+        frozen_eval: "frozen_eval";
+    }>;
+    taskIds: z.ZodArray<z.ZodString>;
+    seeds: z.ZodArray<z.ZodString>;
+    criterion: z.ZodObject<{
+        minimumPassRate: z.ZodNumber;
+        requireComplete: z.ZodBoolean;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export declare const ProfileEvaluationSuiteSchema: z.ZodObject<{
+    id: z.ZodString;
+    label: z.ZodString;
+    scope: z.ZodEnum<{
+        profile: "profile";
+        component: "component";
+    }>;
+    definitionIds: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const ProfileEvaluationCatalogSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluations.v1">;
+    definitions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodString;
+        description: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        tasksetRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        split: z.ZodEnum<{
+            train: "train";
+            validation: "validation";
+            test: "test";
+            frozen_eval: "frozen_eval";
+        }>;
+        taskIds: z.ZodArray<z.ZodString>;
+        seeds: z.ZodArray<z.ZodString>;
+        criterion: z.ZodObject<{
+            minimumPassRate: z.ZodNumber;
+            requireComplete: z.ZodBoolean;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+    suites: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodString;
+        scope: z.ZodEnum<{
+            profile: "profile";
+            component: "component";
+        }>;
+        definitionIds: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type ProfileEvaluationCatalog = z.infer<typeof ProfileEvaluationCatalogSchema>;
+export type ProfileEvaluationDefinition = z.infer<typeof ProfileEvaluationDefinitionSchema>;
+/** Added to a Taskset run manifest only when a Profile component supplies the
+ * candidate behavior. It does not change the Taskset's frozen cases/graders. */
+export declare const ProfileEvaluationRunSourceSchema: z.ZodObject<{
+    profileId: z.ZodString;
+    sourceRevision: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    catalogHash: z.ZodString;
+    definitionId: z.ZodString;
+    definitionHash: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    environmentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileEvaluationRunSource = z.infer<typeof ProfileEvaluationRunSourceSchema>;
+/** Validate component and suite references against the same Profile release. */
+export declare function validateProfileEvaluationCatalog(input: {
+    catalog: unknown;
+    workflowIds: ReadonlySet<string>;
+    skillPaths: ReadonlySet<string>;
+    actionIds: ReadonlySet<string>;
+}): {
+    catalog: ProfileEvaluationCatalog;
+    contentHash: string;
+};
+export declare function resolveProfileEvaluationRunSource(input: {
+    catalog: ProfileEvaluationCatalog;
+    definitionId: string;
+    profileId: string;
+    sourceRevision: string;
+    harnessRelease: {
+        id: string;
+        contentHash: string;
+    };
+    environmentHash: string;
+}): ProfileEvaluationRunSource;
+/** Evaluation definitions are read from the verifier-only bytes of an exact
+ * released source package. The caller must never forward this asset to the
+ * model-visible policy surface. */
+export declare function loadReleasedProfileEvaluationCatalog(value: unknown): {
+    catalog: ProfileEvaluationCatalog;
+    catalogHash: string;
+    harnessRelease: {
+        id: string;
+        contentHash: string;
+    };
+};
+/** For a local release whose caller already verified its full file inventory. */
+export declare function loadReleasedProfileEvaluationCatalogAssets(input: {
+    agentSnapshot: AgentSnapshot;
+    harnessRelease: HarnessRelease;
+    catalogBytes?: Uint8Array;
+    workflowBytes?: Uint8Array;
+    actionBytes?: Uint8Array;
+}): {
+    catalog: ProfileEvaluationCatalog;
+    catalogHash: string;
+};
+//# sourceMappingURL=profile-evaluations.d.ts.map

package/dist/types/profile-evaluations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluations.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAGyB,KAAK,aAAa,EAAE,KAAK,cAAc,EACtE,MAAM,mBAAmB,CAAC;AAS3B;yEACyE;AACzE,eAAO,MAAM,6BAA6B;;;;;;;;;;;4BAKxC,CAAC;AAEH,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAoB5C,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;;;;;kBAK9B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAIhC,CAAC;AAEZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC;AAE5F;gFACgF;AAChF,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;kBASlC,CAAC;AACZ,MAAM,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gCAAgC,CAAC,CAAC;AAE1F,gFAAgF;AAChF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAChC,GAAG;IAAE,OAAO,EAAE,wBAAwB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CA+B7D;AAED,wBAAgB,iCAAiC,CAAC,KAAK,EAAE;IACvD,OAAO,EAAE,wBAAwB,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,0BAA0B,CAc7B;AAED;;mCAEmC;AACnC,wBAAgB,oCAAoC,CAAC,KAAK,EAAE,OAAO,GAAG;IACpE,OAAO,EAAE,wBAAwB,CAAC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD,CAaA;AAED,iFAAiF;AACjF,wBAAgB,0CAA0C,CAAC,KAAK,EAAE;IAChE,aAAa,EAAE,aAAa,CAAC;IAC7B,cAAc,EAAE,cAAc,CAAC;IAC/B,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,aAAa,CAAC,EAAE,UAAU,CAAC;IAC3B,WAAW,CAAC,EAAE,UAAU,CAAC;CAC1B,GAAG;IAAE,OAAO,EAAE,wBAAwB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAqB7D"}

package/dist/types/taskset-run-contract.d.ts

@@ -67,6 +67,30 @@ export declare const TasksetRunManifestContentSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
     }, z.core.$strict>], "kind">;
+    profileEvaluation: z.ZodOptional<z.ZodObject<{
+        profileId: z.ZodString;
+        sourceRevision: z.ZodString;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        catalogHash: z.ZodString;
+        definitionId: z.ZodString;
+        definitionHash: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        environmentHash: z.ZodString;
+    }, z.core.$strict>>;
     policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"model">;
         model: z.ZodObject<{
@@ -156,6 +180,30 @@ export declare const TasksetRunManifestSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
     }, z.core.$strict>], "kind">;
+    profileEvaluation: z.ZodOptional<z.ZodObject<{
+        profileId: z.ZodString;
+        sourceRevision: z.ZodString;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        catalogHash: z.ZodString;
+        definitionId: z.ZodString;
+        definitionHash: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        environmentHash: z.ZodString;
+    }, z.core.$strict>>;
     policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"model">;
         model: z.ZodObject<{

package/dist/types/taskset-run-contract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}
+{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.1",
+  "version": "0.11.2",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -158,7 +158,7 @@
     "zod": "^4.1.13"
   },
   "peerDependencies": {
-    "@openpond/harness": "^0.2.0 || ^0.3.0"
+    "@openpond/harness": "^0.4.0"
   },
   "devDependencies": {
     "@openpond/harness": "workspace:*",