@openpond/evals 0.11.1 → 0.11.3

package/dist/index.js

@@ -12,6 +12,9 @@ export * from "./metrics.js";
 export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
+export * from "./profile-evaluations.js";
+export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/profile-evaluation-comparisons.js

@@ -0,0 +1,86 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { assertTasksetMetricResult } from "./metric-policy.js";
+import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
+import { TasksetRunManifestSchema } from "./taskset-run-contract.js";
+export const ProfileEvaluationComparisonContentSchema = z.object({
+    schemaVersion: z.literal("openpond.profileEvaluationComparison.v1"),
+    id: ReleaseIdSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    populationHash: ReleaseHashSchema,
+    environmentHash: ReleaseHashSchema,
+    metricPolicyHash: ReleaseHashSchema,
+    limitsHash: ReleaseHashSchema,
+    members: z.array(z.object({
+        runManifest: ImmutableReleaseRefSchema,
+        metricResultHash: ReleaseHashSchema,
+        source: ProfileEvaluationRunSourceSchema,
+        policy: TasksetRunManifestSchema.shape.policy,
+        score: z.number().min(0).max(1).nullable(),
+    }).strict()).min(2).max(100),
+    createdAt: ReleaseTimestampSchema,
+}).strict();
+export const ProfileEvaluationComparisonSchema = ProfileEvaluationComparisonContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+/** Comparison membership is immutable and refers to existing run evidence.
+ * The same Taskset, population, environment, metric and limits are required;
+ * model and released Profile source may vary independently. */
+export function createProfileEvaluationComparison(input) {
+    if (input.members.length < 2)
+        throw new Error("A comparison requires at least two runs.");
+    const parsed = input.members.map(({ manifest, result }) => {
+        TasksetRunManifestSchema.parse(manifest);
+        assertContentHash(manifest, "Taskset run manifest");
+        assertTasksetMetricResult(result);
+        if (!manifest.profileEvaluation || manifest.execution.kind !== "harness") {
+            throw new Error("Profile comparison requires released Profile evaluation runs.");
+        }
+        if (result.runManifest.id !== manifest.id || result.runManifest.contentHash !== manifest.contentHash
+            || result.tasksetRelease.id !== manifest.tasksetRelease.id
+            || result.tasksetRelease.contentHash !== manifest.tasksetRelease.contentHash
+            || result.policyHash !== contentHash(manifest.metricPolicy)) {
+            throw new Error("Profile comparison metric differs from its run manifest.");
+        }
+        return { manifest, result, source: manifest.profileEvaluation };
+    });
+    const baseline = parsed[0];
+    const populationHash = contentHash(baseline.manifest.population.map(({ taskId, seed, fixtureId }) => ({ taskId, seed, fixtureId })));
+    const metricPolicyHash = contentHash(baseline.manifest.metricPolicy);
+    const limitsHash = contentHash(baseline.manifest.limits);
+    const ids = new Set();
+    for (const member of parsed) {
+        if (ids.has(member.manifest.id))
+            throw new Error("Profile comparison repeats a run.");
+        ids.add(member.manifest.id);
+        if (member.manifest.tasksetRelease.id !== baseline.manifest.tasksetRelease.id
+            || member.manifest.tasksetRelease.contentHash !== baseline.manifest.tasksetRelease.contentHash
+            || contentHash(member.manifest.population.map(({ taskId, seed, fixtureId }) => ({ taskId, seed, fixtureId }))) !== populationHash
+            || member.source.environmentHash !== baseline.source.environmentHash
+            || contentHash(member.manifest.metricPolicy) !== metricPolicyHash
+            || contentHash(member.manifest.limits) !== limitsHash
+            || member.manifest.runtimeTarget.adapterId !== baseline.manifest.runtimeTarget.adapterId
+            || member.manifest.runtimeTarget.placement !== baseline.manifest.runtimeTarget.placement
+            || member.manifest.runtimeTarget.runtimeVersion !== baseline.manifest.runtimeTarget.runtimeVersion) {
+            throw new Error("Profile comparison runs differ in frozen tasks, environment, grading, limits, or runtime.");
+        }
+    }
+    const content = ProfileEvaluationComparisonContentSchema.parse({
+        schemaVersion: "openpond.profileEvaluationComparison.v1",
+        id: input.id,
+        tasksetRelease: baseline.manifest.tasksetRelease,
+        populationHash,
+        environmentHash: baseline.source.environmentHash,
+        metricPolicyHash,
+        limitsHash,
+        members: parsed.map(({ manifest, result, source }) => ({
+            runManifest: { id: manifest.id, contentHash: manifest.contentHash },
+            metricResultHash: result.contentHash,
+            source,
+            policy: manifest.policy,
+            score: result.value,
+        })),
+        createdAt: input.createdAt,
+    });
+    return ProfileEvaluationComparisonSchema.parse({ ...content, contentHash: contentHash(content) });
+}

package/dist/profile-evaluation-execution.js

@@ -0,0 +1,76 @@
+import { ImmutableArtifactRefSchema, contentHash } from "@openpond/harness";
+import { gradeTaskEvidence } from "./graders.js";
+import { aggregateTasksetRunReceipts, assertTasksetMetricSource } from "./metrics.js";
+import { createAttemptReceipt } from "./runs.js";
+import { assertProfileEvaluationRunAdmission } from "./taskset-run-contract.js";
+import { policyTaskView } from "./tasksets.js";
+/** The installation owns execution, tools, approvals and storage. This runner
+ * owns frozen population dispatch and private grading for both Desktop and web. */
+export async function executeProfileEvaluationRun(input) {
+    assertProfileEvaluationRunAdmission(input.manifest, input.taskset, input.catalog);
+    assertTasksetMetricSource(input.taskset, input.metricSource);
+    const source = input.manifest.profileEvaluation;
+    const definition = input.catalog.definitions.find((item) => item.id === source.definitionId);
+    const tasks = new Map(input.taskset.tasks.map((task) => [task.id, task]));
+    const receipts = [];
+    const grades = [];
+    for (const member of input.manifest.population) {
+        input.signal?.throwIfAborted();
+        const task = tasks.get(member.taskId);
+        const execution = await input.execute({
+            task: policyTaskView(task), seed: member.seed, source,
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        input.signal?.throwIfAborted();
+        const grade = await gradeTaskEvidence({
+            task, evidence: execution.evidence, graders: input.taskset.graders,
+            ...(input.modelJudge ? { modelJudge: input.modelJudge } : {}),
+            ...(input.customVerifier ? { customVerifier: input.customVerifier } : {}),
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        const gradeRef = ImmutableArtifactRefSchema.parse(await input.saveGrade(grade));
+        if (gradeRef.contentHash !== grade.contentHash)
+            throw new Error("Stored Profile evaluation grade differs from its graded evidence.");
+        const failureClass = execution.failureClass ?? grade.failureClass;
+        const receipt = createAttemptReceipt({
+            schemaVersion: "openpond.attemptReceipt.v1",
+            id: member.receiptId,
+            runManifest: { id: input.manifest.id, contentHash: input.manifest.contentHash },
+            taskId: member.taskId,
+            seed: member.seed,
+            terminal: execution.terminal,
+            failureClass,
+            outputHash: execution.evidence.infrastructureError ? null : contentHash(execution.evidence.output),
+            traceHash: execution.traceHash,
+            artifactRefs: execution.artifactRefs,
+            graderEvidenceRefs: [gradeRef],
+            startedAt: execution.startedAt,
+            completedAt: execution.completedAt,
+            latencyMs: execution.latencyMs,
+            costUsd: execution.costUsd,
+            metadata: {
+                gradingRole: "evaluation",
+                score: grade.score,
+                passed: grade.passed,
+                rewardEligible: grade.rewardEligible,
+                sourceRevision: source.sourceRevision,
+                definitionHash: source.definitionHash,
+                environmentHash: source.environmentHash,
+            },
+        });
+        await input.saveReceipt(receipt);
+        grades.push(grade);
+        receipts.push(receipt);
+    }
+    const metric = await aggregateTasksetRunReceipts({
+        manifest: input.manifest, taskset: input.taskset, receipts,
+        ...(input.metricSource !== undefined ? { source: input.metricSource } : {}),
+        ...(input.signal ? { signal: input.signal } : {}),
+    }, input.metricExecutor);
+    const passRate = grades.filter((grade) => grade.passed).length / grades.length;
+    return {
+        receipts, grades, metric, passRate,
+        passed: passRate >= definition.criterion.minimumPassRate
+            && (!definition.criterion.requireComplete || receipts.every((receipt) => receipt.terminal)),
+    };
+}

package/dist/profile-evaluations.js

@@ -0,0 +1,153 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash, harnessSourcePackageFiles, loadReleasedProfileWorkflowCatalogAssets, sha256, validateHarnessSourcePackage, } from "@openpond/harness";
+import { TaskSplitSchema } from "./tasksets.js";
+const RelativePathSchema = z.string().min(1).max(2_000).refine((value) => !value.includes("\\") && !value.includes(":") && !value.startsWith("/")
+    && value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), "evaluation paths must be portable and relative");
+/** Portable selection only. Taskset releases own task data and graders; run
+ * manifests and evidence stay in the installation's evaluation store. */
+export const ProfileEvaluationTargetSchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("profile") }).strict(),
+    z.object({ kind: z.literal("workflow"), workflowId: ReleaseIdSchema }).strict(),
+    z.object({ kind: z.literal("skill"), skillPath: RelativePathSchema }).strict(),
+    z.object({ kind: z.literal("agent_action"), actionId: ReleaseIdSchema }).strict(),
+]);
+export const ProfileEvaluationDefinitionSchema = z.object({
+    id: ReleaseIdSchema,
+    label: z.string().trim().min(1).max(240),
+    description: z.string().max(4_000),
+    target: ProfileEvaluationTargetSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    split: TaskSplitSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(100_000),
+    seeds: z.array(z.string().trim().min(1).max(500)).min(1).max(100),
+    criterion: z.object({
+        minimumPassRate: z.number().min(0).max(1),
+        requireComplete: z.boolean(),
+    }).strict(),
+}).strict().superRefine((definition, context) => {
+    if (new Set(definition.taskIds).size !== definition.taskIds.length) {
+        context.addIssue({ code: "custom", path: ["taskIds"], message: "Evaluation task identities must be unique." });
+    }
+    if (new Set(definition.seeds).size !== definition.seeds.length) {
+        context.addIssue({ code: "custom", path: ["seeds"], message: "Evaluation seeds must be unique." });
+    }
+});
+export const ProfileEvaluationSuiteSchema = z.object({
+    id: ReleaseIdSchema,
+    label: z.string().trim().min(1).max(240),
+    scope: z.enum(["component", "profile"]),
+    definitionIds: z.array(ReleaseIdSchema).min(1).max(1_000),
+}).strict();
+export const ProfileEvaluationCatalogSchema = z.object({
+    schemaVersion: z.literal("openpond.profileEvaluations.v1"),
+    definitions: z.array(ProfileEvaluationDefinitionSchema).max(1_000),
+    suites: z.array(ProfileEvaluationSuiteSchema).max(100),
+}).strict();
+/** Added to a Taskset run manifest only when a Profile component supplies the
+ * candidate behavior. It does not change the Taskset's frozen cases/graders. */
+export const ProfileEvaluationRunSourceSchema = z.object({
+    profileId: ReleaseIdSchema,
+    sourceRevision: z.string().trim().min(1).max(240),
+    harnessRelease: ImmutableReleaseRefSchema,
+    catalogHash: ReleaseHashSchema,
+    definitionId: ReleaseIdSchema,
+    definitionHash: ReleaseHashSchema,
+    target: ProfileEvaluationTargetSchema,
+    environmentHash: ReleaseHashSchema,
+}).strict();
+/** Validate component and suite references against the same Profile release. */
+export function validateProfileEvaluationCatalog(input) {
+    const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
+    const ids = new Set();
+    for (const definition of catalog.definitions) {
+        if (ids.has(definition.id))
+            throw new Error(`Duplicate Profile evaluation ${definition.id}.`);
+        ids.add(definition.id);
+        switch (definition.target.kind) {
+            case "workflow":
+                if (!input.workflowIds.has(definition.target.workflowId))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing workflow.`);
+                break;
+            case "skill":
+                if (!input.skillPaths.has(definition.target.skillPath))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing Skill.`);
+                break;
+            case "agent_action":
+                if (!input.actionIds.has(definition.target.actionId))
+                    throw new Error(`Profile evaluation ${definition.id} references a missing Agent action.`);
+                break;
+        }
+    }
+    const suiteIds = new Set();
+    for (const suite of catalog.suites) {
+        if (suiteIds.has(suite.id))
+            throw new Error(`Duplicate Profile evaluation suite ${suite.id}.`);
+        suiteIds.add(suite.id);
+        if (new Set(suite.definitionIds).size !== suite.definitionIds.length)
+            throw new Error(`Profile evaluation suite ${suite.id} repeats a definition.`);
+        for (const definitionId of suite.definitionIds) {
+            if (!ids.has(definitionId))
+                throw new Error(`Profile evaluation suite ${suite.id} references missing definition ${definitionId}.`);
+        }
+        if (suite.scope === "profile" && suite.definitionIds.every((id) => catalog.definitions.find((definition) => definition.id === id)?.target.kind !== "profile")) {
+            throw new Error(`Profile evaluation suite ${suite.id} needs an end-to-end Profile definition.`);
+        }
+    }
+    return { catalog, contentHash: contentHash(catalog) };
+}
+export function resolveProfileEvaluationRunSource(input) {
+    const catalog = ProfileEvaluationCatalogSchema.parse(input.catalog);
+    const definition = catalog.definitions.find((candidate) => candidate.id === input.definitionId);
+    if (!definition)
+        throw new Error(`Profile evaluation ${input.definitionId} is absent from its catalog.`);
+    return ProfileEvaluationRunSourceSchema.parse({
+        profileId: input.profileId,
+        sourceRevision: input.sourceRevision,
+        harnessRelease: input.harnessRelease,
+        catalogHash: contentHash(catalog),
+        definitionId: definition.id,
+        definitionHash: contentHash(definition),
+        target: definition.target,
+        environmentHash: input.environmentHash,
+    });
+}
+/** Evaluation definitions are read from the verifier-only bytes of an exact
+ * released source package. The caller must never forward this asset to the
+ * model-visible policy surface. */
+export function loadReleasedProfileEvaluationCatalog(value) {
+    const source = validateHarnessSourcePackage(value);
+    const files = harnessSourcePackageFiles(source);
+    const result = loadReleasedProfileEvaluationCatalogAssets({
+        agentSnapshot: source.agentSnapshot,
+        harnessRelease: source.harnessRelease,
+        catalogBytes: files.get("evals/catalog.json"),
+        workflowBytes: files.get("workflows/catalog.json"),
+        actionBytes: files.get("workflows/actions.json"),
+    });
+    return { ...result, harnessRelease: {
+            id: source.harnessRelease.id, contentHash: source.harnessRelease.contentHash,
+        } };
+}
+/** For a local release whose caller already verified its full file inventory. */
+export function loadReleasedProfileEvaluationCatalogAssets(input) {
+    const asset = input.harnessRelease.files.find((file) => file.path === "evals/catalog.json");
+    const bytes = input.catalogBytes;
+    if (!asset || asset.visibility !== "verifier" || !bytes || sha256(bytes) !== asset.contentHash) {
+        throw new Error("Released Profile evaluation catalog is unavailable or not verifier-private.");
+    }
+    const workflows = input.workflowBytes || input.actionBytes
+        ? loadReleasedProfileWorkflowCatalogAssets({
+            agentSnapshot: input.agentSnapshot,
+            harnessRelease: input.harnessRelease,
+            catalogBytes: input.workflowBytes,
+            actionBytes: input.actionBytes,
+        })
+        : null;
+    const { catalog, contentHash: catalogHash } = validateProfileEvaluationCatalog({
+        catalog: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)),
+        workflowIds: new Set(workflows?.catalog.workflows.map((workflow) => workflow.id) ?? []),
+        skillPaths: new Set(input.agentSnapshot.skills.map((skill) => skill.path)),
+        actionIds: new Set(workflows?.actions.map((action) => action.id) ?? []),
+    });
+    return { catalog, catalogHash };
+}

package/dist/taskset-run-contract.js

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { assertContentHash, contentHash, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
 import { RunLimitsSchema, RuntimeTargetBindingSchema, verifyAttemptReceipt } from "./runs.js";
 import { TasksetMetricPolicySchema } from "./metric-policy.js";
+import { ProfileEvaluationCatalogSchema, ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
 import { assertTasksetRelease } from "./tasksets.js";
 /** A fixture check owns no model identity and cannot become a model evaluation. */
 export const TasksetRunPolicySchema = z.discriminatedUnion("kind", [
@@ -24,6 +25,7 @@ export const TasksetRunManifestContentSchema = z.object({
     tasksetRelease: ImmutableReleaseRefSchema,
     packageHash: ReleaseHashSchema,
     execution: TasksetRunExecutionSchema,
+    profileEvaluation: ProfileEvaluationRunSourceSchema.optional(),
     policy: TasksetRunPolicySchema,
     gradingRole: z.literal("evaluation"),
     metricPolicy: TasksetMetricPolicySchema,
@@ -37,6 +39,11 @@ export const TasksetRunManifestContentSchema = z.object({
         context.addIssue({ code: "custom", path: ["population"], message: "Run receipt identities must be unique." });
     if (value.population.some(member => (member.fixtureId !== null) !== (value.policy.kind === "fixture")))
         context.addIssue({ code: "custom", path: ["population"], message: "Only fixture runs require a fixture identity for every member." });
+    if (value.profileEvaluation && (value.execution.kind !== "harness"
+        || value.execution.harnessRelease.id !== value.profileEvaluation.harnessRelease.id
+        || value.execution.harnessRelease.contentHash !== value.profileEvaluation.harnessRelease.contentHash)) {
+        context.addIssue({ code: "custom", path: ["profileEvaluation"], message: "Profile evaluation must execute its exact bound Harness release." });
+    }
 });
 export const TasksetRunManifestSchema = TasksetRunManifestContentSchema.safeExtend({ contentHash: ReleaseHashSchema });
 export function createTasksetRunManifest(input) {
@@ -68,6 +75,39 @@ export function assertTasksetRunRelease(manifest, taskset) {
         || manifest.execution.policyHash !== contentHash(taskset.policy)))
         throw new Error("Taskset-owned execution differs from its pinned release.");
 }
+/** Resolve the verifier-only definition before dispatch. A Profile run must
+ * evaluate exactly its declared frozen cases and seeds through its release. */
+export function assertProfileEvaluationRunAdmission(manifest, taskset, catalogInput) {
+    assertTasksetRunRelease(manifest, taskset);
+    const catalog = ProfileEvaluationCatalogSchema.parse(catalogInput);
+    const source = manifest.profileEvaluation;
+    if (!source || manifest.execution.kind !== "harness" || manifest.policy.kind !== "model") {
+        throw new Error("Profile evaluation requires fresh model execution through its bound Harness.");
+    }
+    if (source.catalogHash !== contentHash(catalog))
+        throw new Error("Profile evaluation catalog differs from its pinned source.");
+    const definition = catalog.definitions.find((entry) => entry.id === source.definitionId);
+    if (!definition || source.definitionHash !== contentHash(definition)
+        || contentHash(source.target) !== contentHash(definition.target)) {
+        throw new Error("Profile evaluation definition differs from its pinned source.");
+    }
+    if (definition.tasksetRelease.id !== taskset.id || definition.tasksetRelease.contentHash !== taskset.contentHash) {
+        throw new Error("Profile evaluation Taskset differs from its pinned definition.");
+    }
+    if (definition.split === "train")
+        throw new Error("Profile evaluation cannot execute training cases.");
+    const tasks = new Map(taskset.tasks.map((task) => [task.id, task]));
+    if (definition.taskIds.some((id) => tasks.get(id)?.split !== definition.split)) {
+        throw new Error("Profile evaluation case is absent from its declared Taskset split.");
+    }
+    const expected = new Set(definition.taskIds.flatMap((taskId) => definition.seeds.map((seed) => JSON.stringify([taskId, seed]))));
+    const actual = new Set(manifest.population.map(({ taskId, seed }) => JSON.stringify([taskId, seed])));
+    if (manifest.population.some((member) => member.fixtureId !== null)
+        || actual.size !== expected.size || manifest.population.length !== expected.size
+        || [...actual].some((member) => !expected.has(member))) {
+        throw new Error("Profile evaluation population differs from its declared cases and seeds.");
+    }
+}
 /** Complete runs cannot drop failed members, add convenient attempts or relabel
  * another task/seed's evidence. Ordering is always the admitted population's. */
 export function orderTasksetRunReceipts(manifest, receipts) {

package/dist/types/index.d.ts

@@ -12,6 +12,9 @@ export * from "./metrics.js";
 export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
+export * from "./profile-evaluations.js";
+export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/profile-evaluation-comparisons.d.ts

@@ -0,0 +1,135 @@
+import { z } from "zod";
+import { type TasksetMetricResult } from "./metric-policy.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
+export declare const ProfileEvaluationComparisonContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationComparison.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    populationHash: z.ZodString;
+    environmentHash: z.ZodString;
+    metricPolicyHash: z.ZodString;
+    limitsHash: z.ZodString;
+    members: z.ZodArray<z.ZodObject<{
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        metricResultHash: z.ZodString;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+        policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"model">;
+            model: z.ZodObject<{
+                provider: z.ZodString;
+                model: z.ZodString;
+                revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            }, z.core.$strict>;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"fixture">;
+        }, z.core.$strict>], "kind">;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+}, z.core.$strict>;
+export declare const ProfileEvaluationComparisonSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluationComparison.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    populationHash: z.ZodString;
+    environmentHash: z.ZodString;
+    metricPolicyHash: z.ZodString;
+    limitsHash: z.ZodString;
+    members: z.ZodArray<z.ZodObject<{
+        runManifest: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        metricResultHash: z.ZodString;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+        policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"model">;
+            model: z.ZodObject<{
+                provider: z.ZodString;
+                model: z.ZodString;
+                revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            }, z.core.$strict>;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"fixture">;
+        }, z.core.$strict>], "kind">;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileEvaluationComparison = z.infer<typeof ProfileEvaluationComparisonSchema>;
+/** Comparison membership is immutable and refers to existing run evidence.
+ * The same Taskset, population, environment, metric and limits are required;
+ * model and released Profile source may vary independently. */
+export declare function createProfileEvaluationComparison(input: {
+    id: string;
+    members: Array<{
+        manifest: TasksetRunManifest;
+        result: TasksetMetricResult;
+    }>;
+    createdAt: string;
+}): ProfileEvaluationComparison;
+//# sourceMappingURL=profile-evaluation-comparisons.d.ts.map

package/dist/types/profile-evaluation-comparisons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluation-comparisons.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-comparisons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzF,OAAO,EAA4B,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE9F,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgB1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AACZ,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC;AAE5F;;+DAE+D;AAC/D,wBAAgB,iCAAiC,CAAC,KAAK,EAAE;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,MAAM,EAAE,mBAAmB,CAAA;KAAE,CAAC,CAAC;IAC9E,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,2BAA2B,CAuD9B"}

package/dist/types/profile-evaluation-execution.d.ts

@@ -0,0 +1,46 @@
+import { type ImmutableArtifactRef } from "@openpond/harness";
+import { type AttemptEvidence, type CustomVerifierRunner, type ModelJudgeRunner, type TaskGrade } from "./graders.js";
+import { type TasksetMetricExecutor, type TasksetMetricResult } from "./metrics.js";
+import { type AttemptReceipt } from "./runs.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
+import { policyTaskView, type TasksetRelease } from "./tasksets.js";
+import type { ProfileEvaluationCatalog, ProfileEvaluationRunSource } from "./profile-evaluations.js";
+type PolicyTask = ReturnType<typeof policyTaskView>;
+/** The installation owns execution, tools, approvals and storage. This runner
+ * owns frozen population dispatch and private grading for both Desktop and web. */
+export declare function executeProfileEvaluationRun(input: {
+    manifest: TasksetRunManifest;
+    taskset: TasksetRelease;
+    catalog: ProfileEvaluationCatalog;
+    execute: (member: {
+        task: PolicyTask;
+        seed: string;
+        source: ProfileEvaluationRunSource;
+        signal?: AbortSignal;
+    }) => Promise<{
+        evidence: AttemptEvidence;
+        traceHash: string;
+        artifactRefs: ImmutableArtifactRef[];
+        startedAt: string;
+        completedAt: string;
+        latencyMs: number;
+        costUsd: number | null;
+        terminal: boolean;
+        failureClass?: AttemptReceipt["failureClass"];
+    }>;
+    saveGrade: (grade: TaskGrade) => Promise<ImmutableArtifactRef>;
+    saveReceipt: (receipt: AttemptReceipt) => Promise<void>;
+    modelJudge?: ModelJudgeRunner;
+    customVerifier?: CustomVerifierRunner;
+    metricSource?: string;
+    metricExecutor?: TasksetMetricExecutor;
+    signal?: AbortSignal;
+}): Promise<{
+    receipts: AttemptReceipt[];
+    grades: TaskGrade[];
+    metric: TasksetMetricResult;
+    passRate: number;
+    passed: boolean;
+}>;
+export {};
+//# sourceMappingURL=profile-evaluation-execution.d.ts.map

package/dist/types/profile-evaluation-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAqB,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AACzI,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAErG,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAkED"}

package/dist/types/profile-evaluations.d.ts

@@ -0,0 +1,176 @@
+import { z } from "zod";
+import { type AgentSnapshot, type HarnessRelease } from "@openpond/harness";
+/** Portable selection only. Taskset releases own task data and graders; run
+ * manifests and evidence stay in the installation's evaluation store. */
+export declare const ProfileEvaluationTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"profile">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"workflow">;
+    workflowId: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"skill">;
+    skillPath: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"agent_action">;
+    actionId: z.ZodString;
+}, z.core.$strict>], "kind">;
+export declare const ProfileEvaluationDefinitionSchema: z.ZodObject<{
+    id: z.ZodString;
+    label: z.ZodString;
+    description: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    split: z.ZodEnum<{
+        train: "train";
+        validation: "validation";
+        test: "test";
+        frozen_eval: "frozen_eval";
+    }>;
+    taskIds: z.ZodArray<z.ZodString>;
+    seeds: z.ZodArray<z.ZodString>;
+    criterion: z.ZodObject<{
+        minimumPassRate: z.ZodNumber;
+        requireComplete: z.ZodBoolean;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export declare const ProfileEvaluationSuiteSchema: z.ZodObject<{
+    id: z.ZodString;
+    label: z.ZodString;
+    scope: z.ZodEnum<{
+        profile: "profile";
+        component: "component";
+    }>;
+    definitionIds: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const ProfileEvaluationCatalogSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileEvaluations.v1">;
+    definitions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodString;
+        description: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        tasksetRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        split: z.ZodEnum<{
+            train: "train";
+            validation: "validation";
+            test: "test";
+            frozen_eval: "frozen_eval";
+        }>;
+        taskIds: z.ZodArray<z.ZodString>;
+        seeds: z.ZodArray<z.ZodString>;
+        criterion: z.ZodObject<{
+            minimumPassRate: z.ZodNumber;
+            requireComplete: z.ZodBoolean;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+    suites: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodString;
+        scope: z.ZodEnum<{
+            profile: "profile";
+            component: "component";
+        }>;
+        definitionIds: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type ProfileEvaluationCatalog = z.infer<typeof ProfileEvaluationCatalogSchema>;
+export type ProfileEvaluationDefinition = z.infer<typeof ProfileEvaluationDefinitionSchema>;
+/** Added to a Taskset run manifest only when a Profile component supplies the
+ * candidate behavior. It does not change the Taskset's frozen cases/graders. */
+export declare const ProfileEvaluationRunSourceSchema: z.ZodObject<{
+    profileId: z.ZodString;
+    sourceRevision: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    catalogHash: z.ZodString;
+    definitionId: z.ZodString;
+    definitionHash: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    environmentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileEvaluationRunSource = z.infer<typeof ProfileEvaluationRunSourceSchema>;
+/** Validate component and suite references against the same Profile release. */
+export declare function validateProfileEvaluationCatalog(input: {
+    catalog: unknown;
+    workflowIds: ReadonlySet<string>;
+    skillPaths: ReadonlySet<string>;
+    actionIds: ReadonlySet<string>;
+}): {
+    catalog: ProfileEvaluationCatalog;
+    contentHash: string;
+};
+export declare function resolveProfileEvaluationRunSource(input: {
+    catalog: ProfileEvaluationCatalog;
+    definitionId: string;
+    profileId: string;
+    sourceRevision: string;
+    harnessRelease: {
+        id: string;
+        contentHash: string;
+    };
+    environmentHash: string;
+}): ProfileEvaluationRunSource;
+/** Evaluation definitions are read from the verifier-only bytes of an exact
+ * released source package. The caller must never forward this asset to the
+ * model-visible policy surface. */
+export declare function loadReleasedProfileEvaluationCatalog(value: unknown): {
+    catalog: ProfileEvaluationCatalog;
+    catalogHash: string;
+    harnessRelease: {
+        id: string;
+        contentHash: string;
+    };
+};
+/** For a local release whose caller already verified its full file inventory. */
+export declare function loadReleasedProfileEvaluationCatalogAssets(input: {
+    agentSnapshot: AgentSnapshot;
+    harnessRelease: HarnessRelease;
+    catalogBytes?: Uint8Array;
+    workflowBytes?: Uint8Array;
+    actionBytes?: Uint8Array;
+}): {
+    catalog: ProfileEvaluationCatalog;
+    catalogHash: string;
+};
+//# sourceMappingURL=profile-evaluations.d.ts.map

package/dist/types/profile-evaluations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluations.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAGyB,KAAK,aAAa,EAAE,KAAK,cAAc,EACtE,MAAM,mBAAmB,CAAC;AAS3B;yEACyE;AACzE,eAAO,MAAM,6BAA6B;;;;;;;;;;;4BAKxC,CAAC;AAEH,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAoB5C,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;;;;;kBAK9B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAIhC,CAAC;AAEZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC;AAE5F;gFACgF;AAChF,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;kBASlC,CAAC;AACZ,MAAM,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gCAAgC,CAAC,CAAC;AAE1F,gFAAgF;AAChF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAChC,GAAG;IAAE,OAAO,EAAE,wBAAwB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CA+B7D;AAED,wBAAgB,iCAAiC,CAAC,KAAK,EAAE;IACvD,OAAO,EAAE,wBAAwB,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,0BAA0B,CAc7B;AAED;;mCAEmC;AACnC,wBAAgB,oCAAoC,CAAC,KAAK,EAAE,OAAO,GAAG;IACpE,OAAO,EAAE,wBAAwB,CAAC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD,CAaA;AAED,iFAAiF;AACjF,wBAAgB,0CAA0C,CAAC,KAAK,EAAE;IAChE,aAAa,EAAE,aAAa,CAAC;IAC7B,cAAc,EAAE,cAAc,CAAC;IAC/B,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,aAAa,CAAC,EAAE,UAAU,CAAC;IAC3B,WAAW,CAAC,EAAE,UAAU,CAAC;CAC1B,GAAG;IAAE,OAAO,EAAE,wBAAwB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAqB7D"}

package/dist/types/taskset-run-contract.d.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { type AttemptReceipt } from "./runs.js";
 import { type TasksetMetricPolicy } from "./metric-policy.js";
+import { type ProfileEvaluationCatalog } from "./profile-evaluations.js";
 import { type TasksetRelease } from "./tasksets.js";
 /** A fixture check owns no model identity and cannot become a model evaluation. */
 export declare const TasksetRunPolicySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -67,6 +68,30 @@ export declare const TasksetRunManifestContentSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
     }, z.core.$strict>], "kind">;
+    profileEvaluation: z.ZodOptional<z.ZodObject<{
+        profileId: z.ZodString;
+        sourceRevision: z.ZodString;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        catalogHash: z.ZodString;
+        definitionId: z.ZodString;
+        definitionHash: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        environmentHash: z.ZodString;
+    }, z.core.$strict>>;
     policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"model">;
         model: z.ZodObject<{
@@ -156,6 +181,30 @@ export declare const TasksetRunManifestSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
     }, z.core.$strict>], "kind">;
+    profileEvaluation: z.ZodOptional<z.ZodObject<{
+        profileId: z.ZodString;
+        sourceRevision: z.ZodString;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        catalogHash: z.ZodString;
+        definitionId: z.ZodString;
+        definitionHash: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        environmentHash: z.ZodString;
+    }, z.core.$strict>>;
     policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
         kind: z.ZodLiteral<"model">;
         model: z.ZodObject<{
@@ -226,6 +275,9 @@ export declare function tasksetRunMetricPolicy(taskset: TasksetRelease): Taskset
 /** Admission verifies this before dispatch. Package byte verification belongs
  * to the package loader; this validates the release and population semantics. */
 export declare function assertTasksetRunRelease(manifest: TasksetRunManifest, taskset: TasksetRelease): void;
+/** Resolve the verifier-only definition before dispatch. A Profile run must
+ * evaluate exactly its declared frozen cases and seeds through its release. */
+export declare function assertProfileEvaluationRunAdmission(manifest: TasksetRunManifest, taskset: TasksetRelease, catalogInput: ProfileEvaluationCatalog): void;
 /** Complete runs cannot drop failed members, add convenient attempts or relabel
  * another task/seed's evidence. Ordering is always the admitted population's. */
 export declare function orderTasksetRunReceipts(manifest: TasksetRunManifest, receipts: AttemptReceipt[]): AttemptReceipt[];

package/dist/types/taskset-run-contract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}
+{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAoE,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AAC3I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;+EAC+E;AAC/E,wBAAgB,mCAAmC,CACjD,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,cAAc,EACvB,YAAY,EAAE,wBAAwB,GACrC,IAAI,CA4BN;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.1",
+  "version": "0.11.3",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -158,7 +158,7 @@
     "zod": "^4.1.13"
   },
   "peerDependencies": {
-    "@openpond/harness": "^0.2.0 || ^0.3.0"
+    "@openpond/harness": "^0.4.0"
   },
   "devDependencies": {
     "@openpond/harness": "workspace:*",