@openpond/evals 0.4.0 → 0.4.1

package/README.md

@@ -1,9 +1,9 @@
 # `@openpond/evals`
 
-Portable evaluation contracts and pure helpers for Tasksets, graders, run
-manifests, attempt and evaluation receipts, execution adapters, conformance
-fixtures, Work-evidence eligibility, and no-training/SFT/preference/RL
-qualification receipts. The package depends on
+Portable evaluation and benchmark contracts plus pure helpers for Tasksets,
+graders, run manifests, attempt and evaluation receipts, paired benchmark
+comparisons, execution adapters, conformance fixtures, Work-evidence
+eligibility, and no-training/SFT/preference/RL qualification receipts. The package depends on
 [`@openpond/harness`](../harness/README.md) for exact Harness identities but
 does not re-export Harness APIs. Applications import the two packages directly,
 which keeps refinement and evaluation authority visibly separate.
@@ -13,6 +13,8 @@ import {
   AttemptReceiptSchema,
   TasksetReleaseSchema,
   ModelImprovementQualificationReceiptSchema,
+  BenchmarkDefinitionSchema,
+  compareBenchmarkRuns,
   validateTasksetRelease,
   verifyAttemptReceipt,
 } from "@openpond/evals";
@@ -26,12 +28,27 @@ import {
 } from "@openpond/evals/evidence";
 ```
 
-Subpath exports are available at `/harness`, `/tasksets`, `/graders`, `/runs`,
+Subpath exports are available at `/harness`, `/tasksets`, `/benchmarks`, `/graders`, `/runs`,
 `/conformance`, `/evidence`, `/review`, and
 `/model-improvement-qualification`. The package is an evaluation protocol library,
 not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
 resolve credentials, or persist artifacts.
 
+## Benchmarks
+
+`BenchmarkDefinition` binds a named benchmark to an immutable Taskset Release,
+its adaptation and held-out splits, primary metric, and quality gate.
+`BenchmarkRunSummary` records the pinned model and reasoning effort together
+with pass counts, foreground provider usage, cost, and latency. Compare a
+baseline and candidate with `compareBenchmarkRuns`; it rejects mismatched
+Taskset releases, models, reasoning effort, cases, seeds, repetitions, runtime,
+environment, tools, or limits and never reports an efficiency win when the
+configured quality gate fails.
+
+The package defines portable schemas and comparison math. Hosts remain
+responsible for scheduling cases, pinning runtime and tools, persisting
+receipts, and keeping held-out evidence out of adaptation.
+
 ## Harness Evaluation review
 
 `@openpond/harness` owns the public model-driven Refiner and continuous-review

package/dist/benchmarks.js

@@ -0,0 +1,218 @@
+import { z } from "zod";
+import { ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
+import { TaskSplitSchema } from "./tasksets.js";
+export { harnessRefinerBenchmarkAssets, harnessRefinerBenchmarkRelease, } from "./builtin-benchmarks/harness-refiner.js";
+export const BenchmarkMetricSchema = z.enum([
+    "foreground_tokens",
+    "success_rate",
+    "latency_ms",
+    "cost_usd",
+]);
+export const BenchmarkRunPhaseSchema = z.enum(["baseline", "candidate"]);
+export const BenchmarkProtocolSchema = z.object({
+    split: TaskSplitSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(100_000),
+    seeds: z.array(z.string().trim().min(1).max(500)).min(1).max(100),
+    repetitions: z.number().int().positive().max(20),
+    runtimeTargetHash: ReleaseHashSchema,
+    environmentHash: ReleaseHashSchema,
+    toolContractHash: ReleaseHashSchema,
+    limitsHash: ReleaseHashSchema,
+}).strict();
+export const BenchmarkDefinitionSchema = z.object({
+    schemaVersion: z.literal("openpond.benchmarkDefinition.v1"),
+    id: ReleaseIdSchema,
+    title: z.string().trim().min(1).max(500),
+    description: z.string().trim().min(1).max(5_000),
+    tasksetRelease: ImmutableReleaseRefSchema,
+    adaptationSplit: TaskSplitSchema,
+    evaluationSplit: TaskSplitSchema,
+    primaryMetric: BenchmarkMetricSchema,
+    qualityGate: z.enum(["none", "non_regression", "all_pass"]),
+    caseCounts: z.object({
+        adaptation: z.number().int().nonnegative(),
+        evaluation: z.number().int().positive(),
+    }).strict(),
+    metadata: MetadataSchema,
+}).strict();
+export const BenchmarkRunRequestSchema = z.object({
+    schemaVersion: z.literal("openpond.benchmarkRunRequest.v1"),
+    phase: BenchmarkRunPhaseSchema,
+    model: ModelRefSchema,
+    reasoningEffort: z.string().trim().min(1).max(100).nullable(),
+    split: TaskSplitSchema,
+    seeds: z.array(z.string().trim().min(1).max(500)).min(1).max(100),
+    repetitions: z.number().int().positive().max(20),
+    metadata: MetadataSchema,
+}).strict();
+const BenchmarkUsageSchema = z.object({
+    inputTokens: z.number().int().nonnegative(),
+    outputTokens: z.number().int().nonnegative(),
+    totalTokens: z.number().int().nonnegative(),
+}).strict();
+export const BenchmarkRunSummaryContentSchema = z.object({
+    schemaVersion: z.literal("openpond.benchmarkRunSummary.v1"),
+    id: ReleaseIdSchema,
+    phase: BenchmarkRunPhaseSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    harnessRelease: ImmutableReleaseRefSchema,
+    evaluationResult: ImmutableReleaseRefSchema,
+    model: ModelRefSchema,
+    reasoningEffort: z.string().trim().min(1).max(100).nullable(),
+    protocol: BenchmarkProtocolSchema,
+    attemptCount: z.number().int().positive(),
+    passedCount: z.number().int().nonnegative(),
+    terminalCount: z.number().int().nonnegative(),
+    usage: BenchmarkUsageSchema,
+    costUsd: z.number().nonnegative().nullable(),
+    latencyMs: z.number().int().nonnegative(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+export const BenchmarkRunSummarySchema = BenchmarkRunSummaryContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export const BenchmarkComparisonContentSchema = z.object({
+    schemaVersion: z.literal("openpond.benchmarkComparison.v1"),
+    id: ReleaseIdSchema,
+    baseline: ImmutableReleaseRefSchema,
+    candidate: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    primaryMetric: BenchmarkMetricSchema,
+    qualityPassed: z.boolean(),
+    baselinePassRate: z.number().min(0).max(1),
+    candidatePassRate: z.number().min(0).max(1),
+    foregroundTokenDelta: z.number().int(),
+    foregroundTokenDeltaPercent: z.number().finite().nullable(),
+    improved: z.boolean(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+export const BenchmarkComparisonSchema = BenchmarkComparisonContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export function createBenchmarkDefinition(input) {
+    return BenchmarkDefinitionSchema.parse(input);
+}
+export function createBenchmarkRunSummary(input) {
+    if (input.receipts.length !== input.evaluation.attemptCount) {
+        throw new Error("Benchmark receipt count does not match its Evaluation result.");
+    }
+    const usage = input.receipts.reduce((total, receipt) => addUsage(total, providerUsage(receipt.metadata.usage)), emptyUsage());
+    const costs = input.receipts.flatMap((receipt) => typeof receipt.costUsd === "number" ? [receipt.costUsd] : []);
+    const content = BenchmarkRunSummaryContentSchema.parse({
+        schemaVersion: "openpond.benchmarkRunSummary.v1",
+        id: input.id,
+        phase: input.phase,
+        tasksetRelease: input.evaluation.tasksetRelease,
+        harnessRelease: input.evaluation.harnessRelease,
+        evaluationResult: {
+            id: input.evaluation.id,
+            contentHash: input.evaluation.contentHash,
+        },
+        model: input.evaluation.model,
+        reasoningEffort: input.reasoningEffort,
+        protocol: input.protocol,
+        attemptCount: input.evaluation.attemptCount,
+        passedCount: input.receipts.filter((receipt) => receipt.metadata.passed === true).length,
+        terminalCount: input.evaluation.terminalCount,
+        usage,
+        costUsd: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null,
+        latencyMs: input.receipts.reduce((sum, receipt) => sum + receipt.latencyMs, 0),
+        createdAt: input.createdAt,
+        metadata: input.metadata ?? {},
+    });
+    return BenchmarkRunSummarySchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+export function compareBenchmarkRuns(input) {
+    const { baseline, candidate } = input;
+    if (baseline.tasksetRelease.contentHash !== candidate.tasksetRelease.contentHash
+        || baseline.model.provider !== candidate.model.provider
+        || baseline.model.model !== candidate.model.model
+        || baseline.reasoningEffort !== candidate.reasoningEffort
+        || contentHash(baseline.protocol) !== contentHash(candidate.protocol)) {
+        throw new Error("Benchmark runs are not comparable under the pinned protocol.");
+    }
+    const baselinePassRate = baseline.passedCount / baseline.attemptCount;
+    const candidatePassRate = candidate.passedCount / candidate.attemptCount;
+    const complete = baseline.terminalCount === baseline.attemptCount
+        && candidate.terminalCount === candidate.attemptCount;
+    const qualityPassed = complete && (input.qualityGate === "none"
+        || (input.qualityGate === "all_pass"
+            ? candidatePassRate === 1
+            : (baseline.passedCount > 0 || candidate.passedCount > 0)
+                && candidatePassRate >= baselinePassRate));
+    const foregroundTokenDelta = candidate.usage.totalTokens - baseline.usage.totalTokens;
+    const foregroundTokenDeltaPercent = baseline.usage.totalTokens > 0
+        ? (foregroundTokenDelta / baseline.usage.totalTokens) * 100
+        : null;
+    const metricImproved = input.primaryMetric === "foreground_tokens"
+        ? foregroundTokenDelta < 0
+        : input.primaryMetric === "success_rate"
+            ? candidatePassRate > baselinePassRate
+            : input.primaryMetric === "latency_ms"
+                ? candidate.latencyMs < baseline.latencyMs
+                : candidate.costUsd !== null
+                    && baseline.costUsd !== null
+                    && candidate.costUsd < baseline.costUsd;
+    const content = BenchmarkComparisonContentSchema.parse({
+        schemaVersion: "openpond.benchmarkComparison.v1",
+        id: input.id,
+        baseline: { id: baseline.id, contentHash: baseline.contentHash },
+        candidate: { id: candidate.id, contentHash: candidate.contentHash },
+        tasksetRelease: baseline.tasksetRelease,
+        primaryMetric: input.primaryMetric,
+        qualityPassed,
+        baselinePassRate,
+        candidatePassRate,
+        foregroundTokenDelta,
+        foregroundTokenDeltaPercent,
+        improved: qualityPassed && metricImproved,
+        createdAt: input.createdAt,
+        metadata: input.metadata ?? {},
+    });
+    return BenchmarkComparisonSchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+function providerUsage(input) {
+    const records = Array.isArray(input) ? input : input ? [input] : [];
+    return records.reduce((total, value) => addUsage(total, usageRecord(value)), emptyUsage());
+}
+function usageRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return emptyUsage();
+    }
+    const record = value;
+    const inputTokens = token(record, ["inputTokens", "input_tokens", "promptTokens", "prompt_tokens"]);
+    const outputTokens = token(record, ["outputTokens", "output_tokens", "completionTokens", "completion_tokens"]);
+    const reportedTotal = token(record, ["totalTokens", "total_tokens"]);
+    return {
+        inputTokens,
+        outputTokens,
+        totalTokens: reportedTotal || inputTokens + outputTokens,
+    };
+}
+function token(record, keys) {
+    for (const key of keys) {
+        const value = record[key];
+        if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+            return Math.trunc(value);
+        }
+    }
+    return 0;
+}
+function emptyUsage() {
+    return { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+}
+function addUsage(left, right) {
+    return {
+        inputTokens: left.inputTokens + right.inputTokens,
+        outputTokens: left.outputTokens + right.outputTokens,
+        totalTokens: left.totalTokens + right.totalTokens,
+    };
+}