@openpond/evals 0.2.0 → 0.3.0

package/CONTRACT.md

@@ -6,13 +6,13 @@ authorization, model streaming, artifact bytes, and runtime processes.
 
 | Existing object | Portable object | Migration rule |
 | --- | --- | --- |
-| `Taskset` | `TasksetRelease` | Project only the released tasks, policy, environment, tools, capabilities, graders, and immutable Harness binding. Authoring state and UI readiness remain host state. |
+| `Taskset` | `TasksetRelease` | Project only the released tasks, policy, environment, tools, capabilities, and graders. A Taskset is deliberately independent of any Harness so the same workload can run against local or hosted execution. Authoring state and UI readiness remain host state. |
 | `HarnessRunManifest` (`openpond.harnessRunManifest.v1`) | `RunManifest` (`openpond.runManifest.v1`) | Treat the old object as a legacy training projection. Normalize its release/model/runtime identities into one new manifest; recipe, compute, engine, secret leases, and approval records remain host bindings referenced by hashes. |
 | `HarnessRunTrace` | `HarnessTrace` | Preserve ordered actions, observations, lifecycle events, terminal state, failure class, and trace hash. Learning-signal envelopes remain a training projection of the trace and receipt. |
 | `TaskAttemptResult` | `AttemptReceipt` | Preserve the old record for application persistence while adding a lossless receipt reference. Output becomes `outputHash`; trace and artifacts are separately hash-bound. |
 | `GradeResult` | `GraderEvidence[]` | Preserve component score, pass, reward eligibility, failure class, feedback, and visible/private evidence references. Aggregate UI results remain host projections. |
 | managed-RL local receipt | `AttemptReceipt` | Submit canonical manifest/task/trace/artifact/grader identities; policy token responses and provider request IDs remain host-private trace data. |
-| resolved training bundle | `HarnessRelease` + host training binding | Environment, tools, program, policy, files, and grader interface belong to the Harness. Dataset/evidence, recipe, compute, engine, approval, and opaque leases remain explicit host-side bindings. |
+| resolved training bundle | `HarnessRelease` + host training binding | Agent snapshot, program, lifecycle, tool declarations, files, and grader interface belong to the Harness. Taskset environment/policy/graders and dataset/evidence, recipe, compute, engine, approval, and opaque leases remain explicit host-side bindings. |
 | completed Work or Development turn | `WorkEvidenceReceipt` | Project the authoritative terminal turn, immutable Agent snapshot when available, model/runtime identity, sanitized trace reference, exact output revisions, validation evidence, interventions, timing, usage, and explicit consent provenance. Keep the raw source and trace host-private. |
 | Agent plus environment runtime events | `WorkProcessTrace` | Emit one ordered trace with `agent` and `environment` layers. Bind every environment step to its outer Agent tool call or stable Agent-turn receipt hash. Hash inputs/outputs and expose only enumerated, bounded attributes. |
 | user feedback on Work output | `WorkFeedbackReceipt` | Append a new receipt bound to the evidence receipt and, when selected, the exact content-addressed output-revision descriptor. Corrections are separate artifacts and never mutate prior receipts. |
@@ -26,6 +26,17 @@ authorization, model streaming, artifact bytes, and runtime processes.
 - Compatible package releases may add optional helpers and exports. Changing a
   required field, identity hash, or privacy boundary requires a new schema
   literal and an explicit normalizer.
+- `openpond.agentSnapshot.v2`, `openpond.harnessRelease.v2`, and
+  `openpond.tasksetRelease.v2` define the Harness-first boundary. The v2
+  contracts remove the Profile reference from the Agent snapshot and keep the
+  Taskset independent of a concrete Harness, environment, or policy binding.
+- Runs with different Harness releases require an explicit
+  `HarnessCompatibilityReceipt` binding both Harnesses to the same Taskset and
+  recording environment, tool, policy, and grader-interface contract hashes.
+  Callers with materialized releases should use
+  `createVerifiedHarnessCompatibilityReceipt`; it derives those hashes from the
+  immutable objects and rejects lifecycle, tool, grader-interface, or required
+  Environment-tool drift before issuing the receipt.
 - The initial support target is Node.js ESM on Node 22.14 through Node 24.
 - Portable paths are relative and at most 2,000 characters. Individual assets
   are at most 250 MB. Tasksets, traces, and evidence arrays have schema-level

package/RELEASING.md

@@ -1,19 +1,20 @@
 # Releasing `@openpond/evals`
 
 The package version is independent from OpenPond application and schema
-versions. Version `0.1.1` is the current provenance-backed baseline and supports
+versions. Version `0.3.0` is the next Harness-first release and supports
 these initial schema literals:
 
-- `openpond.agentSnapshot.v1`
-- `openpond.harnessRelease.v1`
-- `openpond.tasksetRelease.v1`
+- `openpond.agentSnapshot.v2`
+- `openpond.harnessRelease.v2`
+- `openpond.tasksetRelease.v2`
 - `openpond.runManifest.v1`
 - `openpond.attemptReceipt.v1`
 - `openpond.harnessTrace.v1`
+- `openpond.harnessCompatibility.v1`
 - `openpond.graderEvidence.v1`
 - `openpond.evaluationResult.v1`
 
-This branch prepares the additive Work evidence schemas for `0.2.0`:
+The package also carries the Work evidence schemas introduced in `0.2.0`:
 
 - `openpond.workEvidenceReceipt.v1`
 - `openpond.workProcessTrace.v1`

package/dist/compatibility.js

@@ -0,0 +1,45 @@
+import { assertContentHash, contentHash } from "./common.js";
+import { HarnessReleaseSchema, } from "./harness.js";
+import { createHarnessCompatibilityReceipt, } from "./runs.js";
+import { TasksetReleaseSchema, } from "./tasksets.js";
+export function createVerifiedHarnessCompatibilityReceipt(input) {
+    const base = HarnessReleaseSchema.parse(input.baseHarnessRelease);
+    const candidate = HarnessReleaseSchema.parse(input.candidateHarnessRelease);
+    const taskset = TasksetReleaseSchema.parse(input.tasksetRelease);
+    assertContentHash(base, "Base Harness release");
+    assertContentHash(candidate, "Candidate Harness release");
+    assertContentHash(taskset, "Taskset release");
+    requireSameContract("lifecycle", base.lifecycle, candidate.lifecycle);
+    requireSameContract("tool", base.tools, candidate.tools);
+    requireSameContract("grader interface", base.graderInterface, candidate.graderInterface);
+    const environmentTools = new Set(taskset.tools.map((tool) => tool.name));
+    const unsupportedTools = base.tools
+        .map((tool) => tool.name)
+        .filter((name) => !environmentTools.has(name));
+    if (unsupportedTools.length) {
+        throw new Error(`Harness compatibility failed: Taskset Environment does not provide ${unsupportedTools.join(", ")}.`);
+    }
+    return createHarnessCompatibilityReceipt({
+        schemaVersion: "openpond.harnessCompatibility.v1",
+        id: input.id,
+        baseHarnessRelease: { id: base.id, contentHash: base.contentHash },
+        candidateHarnessRelease: {
+            id: candidate.id,
+            contentHash: candidate.contentHash,
+        },
+        tasksetRelease: { id: taskset.id, contentHash: taskset.contentHash },
+        environmentHash: contentHash(taskset.environment),
+        toolContractHash: contentHash(taskset.tools),
+        policyHash: contentHash(taskset.policy),
+        graderInterfaceHash: contentHash({
+            harness: base.graderInterface,
+            taskset: taskset.graders,
+        }),
+        metadata: input.metadata ?? {},
+    });
+}
+function requireSameContract(label, base, candidate) {
+    if (contentHash(base) !== contentHash(candidate)) {
+        throw new Error(`Harness compatibility failed: ${label} contract changed.`);
+    }
+}

package/dist/conformance.js

@@ -14,32 +14,29 @@ export const marketingPortfolioConformance = fixture("marketing-portfolio-v1", [
 ]);
 function fixture(id, tools) {
     const snapshot = createAgentSnapshot({
-        schemaVersion: "openpond.agentSnapshot.v1",
+        schemaVersion: "openpond.agentSnapshot.v2",
         id: `${id}-agent`,
-        profileRelease: null,
+        sourceRelease: null,
         instructions: [], skills: [], agents: [], toolDeclarations: tools,
         capabilityRequirements: [], dependencyLock,
         portability: { portable: true, blockers: [], localOnlyAssetRefs: [], hostPrivateAssetRefs: [] },
         metadata: { conformanceFixture: id },
     });
     const harness = createHarnessRelease({
-        schemaVersion: "openpond.harnessRelease.v1",
+        schemaVersion: "openpond.harnessRelease.v2",
         id: `${id}-harness`,
         agentSnapshot: { id: snapshot.id, contentHash: snapshot.contentHash },
         program,
-        environment: { protocolVersion: "openpond.environment.v1", kind: "agent", entrypoint: id, stateful: true, deterministicSeeds: true, lifecycle: ["create", "reset", "step", "collect", "destroy"], networkPolicy: "none", defaultTimeoutMs: 5_000 },
         tools,
         lifecycle: { create: true, reset: true, step: true, collect: true, destroy: true, resetScope: "attempt" },
         graderInterface: { visibleEvidence: ["output"], privilegedEvidence: ["expected"], privateVerifierIsolation: true },
-        policy: { policyVisibleFields: ["input"], privilegedFields: ["expectedOutput"], hiddenGraderRefs: [], connectedAppScopes: [] },
         files: [], metadata: { conformanceFixture: id },
     });
     const tasksetContent = {
-        schemaVersion: "openpond.tasksetRelease.v1",
+        schemaVersion: "openpond.tasksetRelease.v2",
         id: `${id}-taskset`, revision: 1,
-        harnessRelease: { id: harness.id, contentHash: harness.contentHash },
-        policy: harness.policy,
-        environment: harness.environment,
+        policy: { policyVisibleFields: ["input"], privilegedFields: ["expectedOutput"], hiddenGraderRefs: [], connectedAppScopes: [] },
+        environment: { protocolVersion: "openpond.environment.v1", kind: "agent", entrypoint: id, stateful: true, deterministicSeeds: true, lifecycle: ["create", "reset", "step", "collect", "destroy"], networkPolicy: "none", defaultTimeoutMs: 5_000 },
         tools,
         capabilities: [],
         tasks: [

package/dist/harness-improvements.js

@@ -0,0 +1,329 @@
+import { z } from "zod";
+import { HarnessImprovementRouteSchema, HarnessOverlaySnapshotRefSchema, } from "./harness-workspaces.js";
+import { contentHash, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "./common.js";
+const BoundedTextSchema = z.string().trim().min(1).max(100_000);
+const MetadataSchema = z.record(z.string(), z.unknown()).default({});
+export const ImprovementSafeBoundaryKindSchema = z.enum([
+    "completed_tool_batch",
+    "before_model_step",
+    "turn_completed",
+    "turn_paused",
+]);
+export const ImprovementSafeBoundarySchema = z
+    .object({
+    kind: ImprovementSafeBoundaryKindSchema,
+    eventSequence: z.number().int().nonnegative(),
+    occurredAt: ReleaseTimestampSchema,
+})
+    .strict();
+export const ImprovementEventRefSchema = z
+    .object({
+    id: ReleaseIdSchema,
+    sequence: z.number().int().nonnegative().nullable(),
+    contentHash: ReleaseHashSchema,
+})
+    .strict();
+export const ImprovementObservationKindSchema = z.enum([
+    "tool_failure",
+    "retry",
+    "recovery",
+    "validation",
+    "user_correction",
+    "reusable_success",
+    "completion_detour",
+]);
+export const ImprovementObservationStateSchema = z.enum([
+    "open",
+    "recovered",
+    "terminal",
+]);
+export const ImprovementToolIdentitySchema = z
+    .object({
+    name: z.string().trim().min(1).max(500),
+    invocationKey: ReleaseHashSchema,
+})
+    .strict();
+export const ImprovementObservationContentSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.improvementObservation.v1"),
+    id: ReleaseIdSchema,
+    runRef: ReleaseIdSchema,
+    turnId: ReleaseIdSchema,
+    harnessRelease: ImmutableReleaseRefSchema,
+    overlay: HarnessOverlaySnapshotRefSchema.nullable(),
+    eventRefs: z.array(ImprovementEventRefSchema).min(1).max(100),
+    kind: ImprovementObservationKindSchema,
+    state: ImprovementObservationStateSchema,
+    tool: ImprovementToolIdentitySchema.nullable(),
+    deterministicClass: z.string().trim().min(1).max(500).nullable(),
+    summary: BoundedTextSchema,
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+})
+    .strict()
+    .superRefine((observation, context) => {
+    if (new Set(observation.eventRefs.map((reference) => reference.id)).size !==
+        observation.eventRefs.length) {
+        context.addIssue({
+            code: "custom",
+            message: "observation event refs must be unique",
+            path: ["eventRefs"],
+        });
+    }
+    if (["tool_failure", "retry", "recovery", "completion_detour"].includes(observation.kind) &&
+        observation.tool === null) {
+        context.addIssue({
+            code: "custom",
+            message: `${observation.kind} observations require a tool identity`,
+            path: ["tool"],
+        });
+    }
+});
+export const ImprovementObservationSchema = ImprovementObservationContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export const RefinementTriggerPolicySchema = z
+    .object({
+    schemaVersion: z.literal("openpond.refinementTriggerPolicy.v1"),
+    maxEstimatedCostUsd: z.number().finite().nonnegative(),
+    cooldownMs: z.number().int().nonnegative(),
+    maxPendingPlans: z.number().int().min(1).max(100),
+    maxEvidenceEvents: z.number().int().min(1).max(1_000),
+    maxProposalEdits: z.number().int().min(1).max(1_000),
+    maxProposalBytes: z.number().int().min(1).max(10_000_000),
+})
+    .strict();
+export const RefinementTriggerDecisionKindSchema = z.enum([
+    "no_action",
+    "route_deterministically",
+    "queue_refiner",
+]);
+export const RefinementTriggerDecisionContentSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.refinementTriggerDecision.v1"),
+    id: ReleaseIdSchema,
+    runRef: ReleaseIdSchema,
+    turnId: ReleaseIdSchema,
+    harnessRelease: ImmutableReleaseRefSchema,
+    overlay: HarnessOverlaySnapshotRefSchema.nullable(),
+    observations: z.array(ImmutableReleaseRefSchema).max(100),
+    decision: RefinementTriggerDecisionKindSchema,
+    deterministicRoute: HarnessImprovementRouteSchema.nullable(),
+    suggestedRoutes: z.array(HarnessImprovementRouteSchema).max(8),
+    reason: BoundedTextSchema,
+    deduplicationKey: ReleaseHashSchema,
+    policy: RefinementTriggerPolicySchema,
+    estimatedMaxCostUsd: z.number().finite().nonnegative(),
+    pendingPlanCount: z.number().int().nonnegative(),
+    boundary: ImprovementSafeBoundarySchema,
+    cooldownUntil: ReleaseTimestampSchema.nullable(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+})
+    .strict()
+    .superRefine((trigger, context) => {
+    if (trigger.estimatedMaxCostUsd > trigger.policy.maxEstimatedCostUsd) {
+        context.addIssue({
+            code: "custom",
+            message: "estimated Refiner cost exceeds the trigger policy budget",
+            path: ["estimatedMaxCostUsd"],
+        });
+    }
+    if (trigger.pendingPlanCount > trigger.policy.maxPendingPlans) {
+        context.addIssue({
+            code: "custom",
+            message: "pending plan count exceeds the trigger policy limit",
+            path: ["pendingPlanCount"],
+        });
+    }
+    if (trigger.observations.length > trigger.policy.maxEvidenceEvents) {
+        context.addIssue({
+            code: "custom",
+            message: "trigger observations exceed the evidence budget",
+            path: ["observations"],
+        });
+    }
+    if (trigger.decision === "route_deterministically" &&
+        trigger.deterministicRoute === null) {
+        context.addIssue({
+            code: "custom",
+            message: "deterministic decisions require a route",
+            path: ["deterministicRoute"],
+        });
+    }
+    if (trigger.decision !== "route_deterministically" &&
+        trigger.deterministicRoute !== null) {
+        context.addIssue({
+            code: "custom",
+            message: "only deterministic decisions may declare a deterministic route",
+            path: ["deterministicRoute"],
+        });
+    }
+    if (trigger.decision === "no_action" && trigger.suggestedRoutes.length > 0) {
+        context.addIssue({
+            code: "custom",
+            message: "no-action decisions cannot suggest routes",
+            path: ["suggestedRoutes"],
+        });
+    }
+    if (trigger.decision !== "no_action" && trigger.observations.length === 0) {
+        context.addIssue({
+            code: "custom",
+            message: "actionable trigger decisions require observations",
+            path: ["observations"],
+        });
+    }
+});
+export const RefinementTriggerDecisionSchema = RefinementTriggerDecisionContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export const ImprovementRouteAuthoritySchema = z.enum([
+    "runtime_service",
+    "refiner_model",
+    "human_review",
+    "evaluation_system",
+    "training_system",
+]);
+export const ImprovementRouteDecisionContentSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.improvementRouteDecision.v1"),
+    id: ReleaseIdSchema,
+    trigger: ImmutableReleaseRefSchema,
+    route: HarnessImprovementRouteSchema,
+    authority: ImprovementRouteAuthoritySchema,
+    automatic: z.boolean(),
+    reason: BoundedTextSchema,
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+})
+    .strict()
+    .superRefine((decision, context) => {
+    if (decision.automatic &&
+        ["human_review", "evaluation_system", "training_system"].includes(decision.authority)) {
+        context.addIssue({
+            code: "custom",
+            message: `${decision.authority} routes cannot be marked automatic`,
+            path: ["automatic"],
+        });
+    }
+    if (decision.route === "training" && decision.authority !== "training_system") {
+        context.addIssue({
+            code: "custom",
+            message: "training routes require training-system authority",
+            path: ["authority"],
+        });
+    }
+});
+export const ImprovementRouteDecisionSchema = ImprovementRouteDecisionContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export const HarnessRefinerOutcomeContentSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.harnessRefinerOutcome.v1"),
+    id: ReleaseIdSchema,
+    trigger: ImmutableReleaseRefSchema,
+    decision: z.enum(["no_action", "proposed"]),
+    proposal: ImmutableReleaseRefSchema.nullable(),
+    reason: BoundedTextSchema,
+    evidenceRefs: z.array(ImmutableReleaseRefSchema).max(100),
+    estimatedCostUsd: z.number().finite().nonnegative(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+})
+    .strict()
+    .superRefine((outcome, context) => {
+    if ((outcome.decision === "proposed") !== (outcome.proposal !== null)) {
+        context.addIssue({
+            code: "custom",
+            message: "proposed Refiner outcomes require a proposal; no-action outcomes cannot include one",
+            path: ["proposal"],
+        });
+    }
+});
+export const HarnessRefinerOutcomeSchema = HarnessRefinerOutcomeContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export const ImprovementApplyDecisionSchema = z.enum([
+    "applied",
+    "retained",
+    "declined",
+    "conflict",
+    "rolled_back",
+]);
+export const ImprovementApplyReceiptContentSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.improvementApplyReceipt.v1"),
+    id: ReleaseIdSchema,
+    proposal: ImmutableReleaseRefSchema,
+    beforeOverlay: HarnessOverlaySnapshotRefSchema,
+    afterOverlay: HarnessOverlaySnapshotRefSchema.nullable(),
+    decision: ImprovementApplyDecisionSchema,
+    boundary: ImprovementSafeBoundarySchema,
+    validationRefs: z.array(ImmutableReleaseRefSchema).max(100),
+    outcomeEvidenceRefs: z.array(ImprovementEventRefSchema).max(1_000),
+    rollbackOf: ImmutableReleaseRefSchema.nullable(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+})
+    .strict()
+    .superRefine((receipt, context) => {
+    if ((receipt.decision === "applied") !== (receipt.afterOverlay !== null)) {
+        context.addIssue({
+            code: "custom",
+            message: "applied receipts require an after-overlay; other decisions cannot include one",
+            path: ["afterOverlay"],
+        });
+    }
+    if ((receipt.decision === "rolled_back") !== (receipt.rollbackOf !== null)) {
+        context.addIssue({
+            code: "custom",
+            message: "only rollback receipts require rollbackOf",
+            path: ["rollbackOf"],
+        });
+    }
+});
+export const ImprovementApplyReceiptSchema = ImprovementApplyReceiptContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+function createHashedContract(input) {
+    const parsed = input.contentSchema.parse(input.content);
+    return input.resultSchema.parse({
+        ...parsed,
+        contentHash: contentHash(parsed),
+    });
+}
+export function createImprovementObservation(content) {
+    return createHashedContract({
+        content,
+        contentSchema: ImprovementObservationContentSchema,
+        resultSchema: ImprovementObservationSchema,
+    });
+}
+export function createRefinementTriggerDecision(content) {
+    return createHashedContract({
+        content,
+        contentSchema: RefinementTriggerDecisionContentSchema,
+        resultSchema: RefinementTriggerDecisionSchema,
+    });
+}
+export function createImprovementRouteDecision(content) {
+    return createHashedContract({
+        content,
+        contentSchema: ImprovementRouteDecisionContentSchema,
+        resultSchema: ImprovementRouteDecisionSchema,
+    });
+}
+export function createHarnessRefinerOutcome(content) {
+    return createHashedContract({
+        content,
+        contentSchema: HarnessRefinerOutcomeContentSchema,
+        resultSchema: HarnessRefinerOutcomeSchema,
+    });
+}
+export function createImprovementApplyReceipt(content) {
+    return createHashedContract({
+        content,
+        contentSchema: ImprovementApplyReceiptContentSchema,
+        resultSchema: ImprovementApplyReceiptSchema,
+    });
+}