@openpond/evals 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CONTRACT.md +11 -3
  2. package/README.md +27 -8
  3. package/RELEASING.md +13 -46
  4. package/dist/compatibility.js +2 -2
  5. package/dist/conformance.js +1 -2
  6. package/dist/evidence/authoring.js +1 -1
  7. package/dist/evidence/conformance.js +1 -1
  8. package/dist/evidence/contracts.js +1 -2
  9. package/dist/evidence/eligibility.js +1 -1
  10. package/dist/graders.js +1 -1
  11. package/dist/harness.js +33 -101
  12. package/dist/index.js +2 -3
  13. package/dist/model-improvement-qualification.js +125 -0
  14. package/dist/review-conformance.js +209 -0
  15. package/dist/runs.js +1 -9
  16. package/dist/tasksets.js +2 -15
  17. package/dist/types/compatibility.d.ts +1 -1
  18. package/dist/types/compatibility.d.ts.map +1 -1
  19. package/dist/types/conformance.d.ts +2 -0
  20. package/dist/types/conformance.d.ts.map +1 -1
  21. package/dist/types/evidence/conformance.d.ts +20 -20
  22. package/dist/types/evidence/contracts.d.ts +18 -18
  23. package/dist/types/harness.d.ts +3 -409
  24. package/dist/types/harness.d.ts.map +1 -1
  25. package/dist/types/index.d.ts +2 -3
  26. package/dist/types/index.d.ts.map +1 -1
  27. package/dist/types/model-improvement-qualification.d.ts +222 -0
  28. package/dist/types/model-improvement-qualification.d.ts.map +1 -0
  29. package/dist/types/review-conformance.d.ts +607 -0
  30. package/dist/types/review-conformance.d.ts.map +1 -0
  31. package/dist/types/runs.d.ts +1 -9
  32. package/dist/types/runs.d.ts.map +1 -1
  33. package/dist/types/tasksets.d.ts +2 -25
  34. package/dist/types/tasksets.d.ts.map +1 -1
  35. package/package.json +14 -10
  36. package/dist/common.js +0 -67
  37. package/dist/harness-improvements.js +0 -329
  38. package/dist/harness-workspaces.js +0 -368
  39. package/dist/sha256.js +0 -91
  40. package/dist/types/common.d.ts +0 -51
  41. package/dist/types/common.d.ts.map +0 -1
  42. package/dist/types/harness-improvements.d.ts +0 -523
  43. package/dist/types/harness-improvements.d.ts.map +0 -1
  44. package/dist/types/harness-workspaces.d.ts +0 -802
  45. package/dist/types/harness-workspaces.d.ts.map +0 -1
  46. package/dist/types/sha256.d.ts +0 -2
  47. package/dist/types/sha256.d.ts.map +0 -1
package/CONTRACT.md CHANGED
@@ -1,8 +1,14 @@
1
1
  # Portable contract map
2
2
 
3
- `@openpond/evals` owns the portable wire formats and pure protocol logic. Host
4
- applications own persistence, provider sessions, secret leases, connected-app
5
- authorization, model streaming, artifact bytes, and runtime processes.
3
+ `@openpond/harness` owns immutable Harness identities, workspaces,
4
+ model-driven refinement/review policy, improvements, traces, tools, model
5
+ identities, and shared hashing.
6
+ `@openpond/evals` owns Tasksets, graders, evaluation runs and receipts,
7
+ execution orchestration, conformance fixtures, and Work-evidence eligibility.
8
+ Evals depends on Harness for exact release identities but does not re-export
9
+ Harness APIs; Harness never depends on Evals. Host applications own persistence, provider sessions, secret
10
+ leases, connected-app authorization, model streaming, artifact bytes, and
11
+ runtime processes.
6
12
 
7
13
  | Existing object | Portable object | Migration rule |
8
14
  | --- | --- | --- |
@@ -16,6 +22,8 @@ authorization, model streaming, artifact bytes, and runtime processes.
16
22
  | completed Work or Development turn | `WorkEvidenceReceipt` | Project the authoritative terminal turn, immutable Agent snapshot when available, model/runtime identity, sanitized trace reference, exact output revisions, validation evidence, interventions, timing, usage, and explicit consent provenance. Keep the raw source and trace host-private. |
17
23
  | Agent plus environment runtime events | `WorkProcessTrace` | Emit one ordered trace with `agent` and `environment` layers. Bind every environment step to its outer Agent tool call or stable Agent-turn receipt hash. Hash inputs/outputs and expose only enumerated, bounded attributes. |
18
24
  | user feedback on Work output | `WorkFeedbackReceipt` | Append a new receipt bound to the evidence receipt and, when selected, the exact content-addressed output-revision descriptor. Corrections are separate artifacts and never mutate prior receipts. |
25
+ | bounded cross-Work review | `HarnessEvaluationReviewReceipt` | Select only currently authorized immutable evidence, advance one watermark, group one stable claim, route to the smallest correct layer, and name the next authority without performing downstream effects. |
26
+ | model-improvement qualification | `ModelImprovementQualificationReceipt` | Bind the originating review, exact Harness, Taskset, real baseline Evaluation, Model, Environment/tool/permission/policy hashes, Verifier, source policies, privacy, budget, and non-frozen signal. Weak or confounded evidence emits `no_training`; training and activation remain host effects. |
19
27
 
20
28
  ## Compatibility policy
21
29
 
package/README.md CHANGED
@@ -1,18 +1,24 @@
1
1
  # `@openpond/evals`
2
2
 
3
- Portable OpenPond contracts and reference helpers for immutable Agent snapshots,
4
- Harness releases, Taskset releases, run manifests, attempt receipts, observational
5
- Work evidence, deterministic graders, and conformance fixtures.
3
+ Portable evaluation contracts and pure helpers for Tasksets, graders, run
4
+ manifests, attempt and evaluation receipts, execution adapters, conformance
5
+ fixtures, Work-evidence eligibility, and no-training/SFT/preference/RL
6
+ qualification receipts. The package depends on
7
+ [`@openpond/harness`](../harness/README.md) for exact Harness identities but
8
+ does not re-export Harness APIs. Applications import the two packages directly,
9
+ which keeps refinement and evaluation authority visibly separate.
6
10
 
7
11
  ```ts
8
12
  import {
9
13
  AttemptReceiptSchema,
10
- HarnessReleaseSchema,
11
14
  TasksetReleaseSchema,
15
+ ModelImprovementQualificationReceiptSchema,
12
16
  validateTasksetRelease,
13
17
  verifyAttemptReceipt,
14
18
  } from "@openpond/evals";
15
19
 
20
+ import { HarnessReleaseSchema } from "@openpond/harness";
21
+
16
22
  import {
17
23
  WorkEvidenceReceiptSchema,
18
24
  classifyWorkEvidence,
@@ -21,9 +27,22 @@ import {
21
27
  ```
22
28
 
23
29
  Subpath exports are available at `/harness`, `/tasksets`, `/graders`, `/runs`,
24
- `/conformance`, and `/evidence`. The package is a protocol library, not a hosted client. It
25
- does not execute OpenPond Desktop or Sandbox sessions, resolve credentials, or
26
- persist artifacts.
30
+ `/conformance`, `/evidence`, `/review`, and
31
+ `/model-improvement-qualification`. The package is an evaluation protocol library,
32
+ not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
33
+ resolve credentials, or persist artifacts.
34
+
35
+ ## Harness Evaluation review
36
+
37
+ `@openpond/harness` owns the public model-driven Refiner and continuous-review
38
+ policy plus immutable bounded decisions and receipts. Evals does not own
39
+ learning prompts, evidence selection, scheduling, or Harness mutation. It
40
+ binds an accepted review to the exact Harness, Taskset, scored
41
+ baseline Evaluation, base Model, Environment/tool/permission/policy hashes,
42
+ Verifier, source-policy checks, privacy approval, budget approval, and eligible
43
+ non-frozen learning signal. Frozen Evaluation evidence is rejected as training
44
+ evidence. Weak, constant, confounded, uncalibrated, revoked, or incompletely
45
+ authorized signal produces `no_training`; it cannot qualify a managed run.
27
46
 
28
47
  ## Work evidence
29
48
 
@@ -66,7 +85,7 @@ overrides that state.
66
85
  Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
67
86
  the host's existing model/Agent loop. Do not reimplement prompting, tool dispatch,
68
87
  session persistence, cost accounting, cancellation, or cleanup in this package.
69
- A Prime-style environment maps reset/step/observation semantics to the same
88
+ A step-based environment maps reset/step/observation semantics to the same
70
89
  interfaces and keeps provider allocation, authentication, and cleanup in its host.
71
90
 
72
91
  ## Conformance
package/RELEASING.md CHANGED
@@ -1,55 +1,22 @@
1
1
  # Releasing `@openpond/evals`
2
2
 
3
- The package version is independent from OpenPond application and schema
4
- versions. Version `0.3.0` is the next Harness-first release and supports
5
- these initial schema literals:
6
-
7
- - `openpond.agentSnapshot.v2`
8
- - `openpond.harnessRelease.v2`
9
- - `openpond.tasksetRelease.v2`
10
- - `openpond.runManifest.v1`
11
- - `openpond.attemptReceipt.v1`
12
- - `openpond.harnessTrace.v1`
13
- - `openpond.harnessCompatibility.v1`
14
- - `openpond.graderEvidence.v1`
15
- - `openpond.evaluationResult.v1`
16
-
17
- The package also carries the Work evidence schemas introduced in `0.2.0`:
18
-
19
- - `openpond.workEvidenceReceipt.v1`
20
- - `openpond.workProcessTrace.v1`
21
- - `openpond.workFeedbackReceipt.v1`
22
- - `openpond.workEvidenceEligibility.v1`
23
-
24
- ## Trusted publishing
25
-
26
- The package already exists on npm and the `npm-production` environment is
27
- configured for trusted publishing through `release-evals.yml`. Do not run a
28
- manual `npm publish` or repeat the original bootstrap. The workflow uses its own
29
- pinned npm version for trusted publication and provenance.
30
-
31
- To inspect an already-published version:
3
+ `@openpond/evals` is the public evaluation package for OpenPond Harnesses. It
4
+ depends on `@openpond/harness` for exact Harness identities but exposes only
5
+ evaluation APIs. Publish the matching Harness peer before Evals so npm can
6
+ satisfy the declared range.
32
7
 
33
8
  ```bash
34
9
  pnpm evals:check
35
- npm trust list @openpond/evals
36
- npm view @openpond/evals version dist.integrity dist.attestations
37
- ```
38
-
39
- ## Later releases
40
-
41
- For releases that do not already carry an intentional version bump, merge
42
- feature work without changing the package version. When the intended package
43
- changes are on `master`, prepare a separate release from a clean current
44
- `master` checkout:
45
-
46
- ```bash
47
10
  pnpm release:evals:patch
48
11
  # or release:evals:minor / release:evals:major
49
12
  ```
50
13
 
51
- The release helper creates an Evals-only release PR and updates the package
52
- version and lockfile. Merging it triggers trusted publishing with provenance,
53
- registry verification, and a package-specific tag. Record the registry
54
- integrity/provenance evidence and merge commit in the release notes after
55
- publication.
14
+ Merging a release PR triggers `release-evals.yml`, trusted npm publishing,
15
+ registry verification, provenance verification, and the package-specific tag.
16
+ Release notes must call out removal or movement of a public subpath and use the
17
+ appropriate semver increment.
18
+
19
+ Version `0.4.0` removes the retired Harness root barrel and
20
+ `./harness-improvements` / `./harness-workspaces` compatibility subpaths.
21
+ Consumers must import those public contracts from `@openpond/harness`; this is
22
+ an intentional breaking package-boundary correction.
@@ -1,5 +1,5 @@
1
- import { assertContentHash, contentHash } from "./common.js";
2
- import { HarnessReleaseSchema, } from "./harness.js";
1
+ import { assertContentHash, contentHash } from "@openpond/harness";
2
+ import { HarnessReleaseSchema, } from "@openpond/harness";
3
3
  import { createHarnessCompatibilityReceipt, } from "./runs.js";
4
4
  import { TasksetReleaseSchema, } from "./tasksets.js";
5
5
  export function createVerifiedHarnessCompatibilityReceipt(input) {
@@ -1,5 +1,4 @@
1
- import { contentHash, withContentHash } from "./common.js";
2
- import { createAgentSnapshot, createHarnessRelease } from "./harness.js";
1
+ import { contentHash, createAgentSnapshot, createHarnessRelease, withContentHash, } from "@openpond/harness";
3
2
  import { createRunManifest } from "./runs.js";
4
3
  import { TasksetReleaseSchema } from "./tasksets.js";
5
4
  const EMPTY_HASH = contentHash("");
@@ -1,5 +1,5 @@
1
1
  import { z } from "zod";
2
- import { ImmutableReleaseRefSchema, ReleaseHashSchema } from "../common.js";
2
+ import { ImmutableReleaseRefSchema, ReleaseHashSchema } from "@openpond/harness";
3
3
  import { EvidenceArtifactRefSchema, WorkEvidenceReceiptSchema, evidenceArtifactRef, } from "./contracts.js";
4
4
  import { WorkEvidenceEligibilitySchema } from "./eligibility.js";
5
5
  export const WorkEvidenceAuthoringInputSchema = z.object({
@@ -1,4 +1,4 @@
1
- import { canonicalJson, contentHash } from "../common.js";
1
+ import { canonicalJson, contentHash } from "@openpond/harness";
2
2
  import { createWorkEvidenceReceipt, createWorkFeedbackReceipt, createWorkProcessTrace, evidenceArtifactRef, workEvidenceReceiptRef, workSourceOpaqueRef, } from "./contracts.js";
3
3
  import { classifyWorkEvidence } from "./eligibility.js";
4
4
  const startedAt = "2026-08-04T12:00:00.000Z";
@@ -1,6 +1,5 @@
1
1
  import { z } from "zod";
2
- import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "../common.js";
3
- import { ModelRefSchema } from "../runs.js";
2
+ import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
4
3
  export const WORK_EVIDENCE_SCHEMA_VERSION = "openpond.workEvidenceReceipt.v1";
5
4
  export const WORK_PROCESS_TRACE_SCHEMA_VERSION = "openpond.workProcessTrace.v1";
6
5
  export const WORK_FEEDBACK_SCHEMA_VERSION = "openpond.workFeedbackReceipt.v1";
@@ -1,5 +1,5 @@
1
1
  import { z } from "zod";
2
- import { contentHash, ReleaseHashSchema } from "../common.js";
2
+ import { contentHash, ReleaseHashSchema } from "@openpond/harness";
3
3
  import { AttemptReceiptSchema, verifyAttemptReceipt } from "../runs.js";
4
4
  import { WORK_EVIDENCE_ELIGIBILITY_SCHEMA_VERSION, WorkFeedbackReceiptSchema, createWorkFeedbackReceipt, verifyWorkEvidenceReceipt, } from "./contracts.js";
5
5
  export const WorkEvidencePolicyStateSchema = z.enum([
package/dist/graders.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { z } from "zod";
2
- import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "./common.js";
2
+ import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
3
3
  export const GraderEvidenceContentSchema = z.object({
4
4
  schemaVersion: z.literal("openpond.graderEvidence.v1"),
5
5
  graderId: ReleaseIdSchema,
package/dist/harness.js CHANGED
@@ -1,71 +1,6 @@
1
- import { z } from "zod";
2
- import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableAssetRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash, } from "./common.js";
3
- import { ToolDeclarationSchema } from "./tasksets.js";
4
- import { createAttemptReceipt } from "./runs.js";
5
- export const PortabilityReportSchema = z.object({
6
- portable: z.boolean(),
7
- blockers: z.array(z.string().trim().min(1).max(2_000)).max(1_000),
8
- localOnlyAssetRefs: z.array(ReleaseIdSchema).max(10_000),
9
- hostPrivateAssetRefs: z.array(ReleaseIdSchema).max(10_000),
10
- }).strict();
11
- export const AgentSnapshotContentSchema = z.object({
12
- schemaVersion: z.literal("openpond.agentSnapshot.v2"),
13
- id: ReleaseIdSchema,
14
- sourceRelease: ImmutableReleaseRefSchema.nullable(),
15
- instructions: z.array(ImmutableAssetRefSchema).max(10_000),
16
- skills: z.array(ImmutableAssetRefSchema).max(10_000),
17
- agents: z.array(ImmutableAssetRefSchema).max(10_000),
18
- toolDeclarations: z.array(ToolDeclarationSchema).max(200),
19
- capabilityRequirements: z.array(z.object({
20
- id: ReleaseIdSchema,
21
- required: z.boolean(),
22
- scopes: z.array(z.string().trim().min(1).max(500)).max(100),
23
- }).strict()).max(200),
24
- dependencyLock: ImmutableAssetRefSchema,
25
- portability: PortabilityReportSchema,
26
- metadata: MetadataSchema,
27
- }).strict();
28
- export const AgentSnapshotSchema = AgentSnapshotContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
29
- export const LifecycleContractSchema = z.object({
30
- create: z.literal(true),
31
- reset: z.literal(true),
32
- step: z.literal(true),
33
- collect: z.literal(true),
34
- destroy: z.literal(true),
35
- resetScope: z.enum(["task", "attempt"]),
36
- }).strict();
37
- export const GraderInterfaceContractSchema = z.object({
38
- visibleEvidence: z.array(ReleaseIdSchema).max(1_000),
39
- privilegedEvidence: z.array(ReleaseIdSchema).max(1_000),
40
- privateVerifierIsolation: z.boolean(),
41
- }).strict();
42
- export const HarnessReleaseContentSchema = z.object({
43
- schemaVersion: z.literal("openpond.harnessRelease.v2"),
44
- id: ReleaseIdSchema,
45
- agentSnapshot: ImmutableReleaseRefSchema,
46
- program: ImmutableAssetRefSchema,
47
- tools: z.array(ToolDeclarationSchema).max(200),
48
- lifecycle: LifecycleContractSchema,
49
- graderInterface: GraderInterfaceContractSchema,
50
- files: z.array(ImmutableAssetRefSchema).max(100_000),
51
- metadata: MetadataSchema,
52
- }).strict();
53
- export const HarnessReleaseSchema = HarnessReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
54
- export const ModelActionSchema = z.object({
55
- id: ReleaseIdSchema,
56
- turn: z.number().int().nonnegative(),
57
- kind: z.enum(["message", "tool_call", "terminal"]),
58
- name: ReleaseIdSchema.nullable(),
59
- arguments: z.record(z.string(), z.unknown()),
60
- content: z.string().max(1_000_000).nullable(),
61
- }).strict();
62
- export const ToolObservationSchema = z.object({
63
- actionId: ReleaseIdSchema,
64
- turn: z.number().int().nonnegative(),
65
- terminal: z.boolean(),
66
- output: z.record(z.string(), z.unknown()),
67
- artifactRefs: z.array(ImmutableArtifactRefSchema).max(100_000),
68
- }).strict();
1
+ import { HarnessTraceSchema, ModelActionSchema, ToolObservationSchema, contentHash, } from "@openpond/harness";
2
+ import { createAttemptReceipt, } from "./runs.js";
3
+ export * from "@openpond/harness/harness";
69
4
  export async function executeRuntimeProtocol(input) {
70
5
  const now = input.now ?? (() => new Date().toISOString());
71
6
  const startedAt = now();
@@ -84,12 +19,25 @@ export async function executeRuntimeProtocol(input) {
84
19
  let output = {};
85
20
  let artifactRefs = [];
86
21
  const event = (type, payload, metadata = {}) => {
87
- events.push({ sequence: sequence++, type, payloadHash: contentHash(payload), metadata });
22
+ events.push({
23
+ sequence: sequence++,
24
+ type,
25
+ payloadHash: contentHash(payload),
26
+ metadata,
27
+ });
88
28
  };
89
29
  try {
90
- lease = await input.runtime.create({ manifest: input.manifest, taskId: input.taskId, seed: input.seed, signal: controller.signal });
30
+ lease = await input.runtime.create({
31
+ manifest: input.manifest,
32
+ taskId: input.taskId,
33
+ seed: input.seed,
34
+ signal: controller.signal,
35
+ });
91
36
  event("created", { leaseId: lease.id });
92
- await input.runtime.reset(lease, { seed: input.seed, signal: controller.signal });
37
+ await input.runtime.reset(lease, {
38
+ seed: input.seed,
39
+ signal: controller.signal,
40
+ });
93
41
  event("reset", { seed: input.seed });
94
42
  for (const candidate of input.actions.slice(0, input.manifest.limits.maxTurns)) {
95
43
  if (controller.signal.aborted)
@@ -152,12 +100,23 @@ export async function executeRuntimeProtocol(input) {
152
100
  failureClass,
153
101
  output,
154
102
  };
155
- const trace = HarnessTraceSchema.parse({ ...traceContent, contentHash: contentHash(traceContent) });
103
+ const trace = HarnessTraceSchema.parse({
104
+ ...traceContent,
105
+ contentHash: contentHash(traceContent),
106
+ });
156
107
  const completedAt = now();
157
108
  const receipt = createAttemptReceipt({
158
109
  schemaVersion: "openpond.attemptReceipt.v1",
159
- id: `attempt-${contentHash([input.manifest.contentHash, input.taskId, input.seed, trace.contentHash]).slice(0, 24)}`,
160
- runManifest: { id: input.manifest.id, contentHash: input.manifest.contentHash },
110
+ id: `attempt-${contentHash([
111
+ input.manifest.contentHash,
112
+ input.taskId,
113
+ input.seed,
114
+ trace.contentHash,
115
+ ]).slice(0, 24)}`,
116
+ runManifest: {
117
+ id: input.manifest.id,
118
+ contentHash: input.manifest.contentHash,
119
+ },
161
120
  taskId: input.taskId,
162
121
  seed: input.seed,
163
122
  terminal,
@@ -175,30 +134,3 @@ export async function executeRuntimeProtocol(input) {
175
134
  });
176
135
  return { receipt, trace, output };
177
136
  }
178
- export function createAgentSnapshot(input) {
179
- const content = AgentSnapshotContentSchema.parse(input);
180
- return AgentSnapshotSchema.parse({ ...content, contentHash: contentHash(content) });
181
- }
182
- export function createHarnessRelease(input) {
183
- const content = HarnessReleaseContentSchema.parse(input);
184
- return HarnessReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
185
- }
186
- export const HarnessLifecycleEventSchema = z.object({
187
- sequence: z.number().int().nonnegative(),
188
- type: z.enum(["created", "reset", "action", "observation", "terminal", "failure", "collected", "destroyed"]),
189
- payloadHash: ReleaseHashSchema,
190
- metadata: MetadataSchema,
191
- }).strict();
192
- export const HarnessTraceSchema = z.object({
193
- schemaVersion: z.literal("openpond.harnessTrace.v1"),
194
- manifest: ImmutableReleaseRefSchema,
195
- taskId: ReleaseIdSchema,
196
- seed: z.string().trim().min(1).max(500),
197
- events: z.array(HarnessLifecycleEventSchema).max(1_000_000),
198
- actions: z.array(ModelActionSchema).max(100_000),
199
- observations: z.array(ToolObservationSchema).max(100_000),
200
- terminal: z.boolean(),
201
- failureClass: FailureClassSchema.nullable(),
202
- output: z.record(z.string(), z.unknown()),
203
- contentHash: ReleaseHashSchema,
204
- }).strict();
package/dist/index.js CHANGED
@@ -1,9 +1,8 @@
1
- export * from "./common.js";
2
1
  export * from "./compatibility.js";
3
2
  export * from "./evidence/index.js";
4
3
  export * from "./graders.js";
5
4
  export * from "./harness.js";
6
- export * from "./harness-improvements.js";
7
- export * from "./harness-workspaces.js";
8
5
  export * from "./runs.js";
6
+ export * from "./model-improvement-qualification.js";
7
+ export * from "./review-conformance.js";
9
8
  export * from "./tasksets.js";
@@ -0,0 +1,125 @@
1
+ import { z } from "zod";
2
+ import { HarnessReviewSourcePolicyRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
3
+ const BoundedTextSchema = z.string().trim().min(1).max(100_000);
4
+ export const ModelImprovementDecisionSchema = z.enum([
5
+ "no_training",
6
+ "sft",
7
+ "preference",
8
+ "rl",
9
+ ]);
10
+ export const ModelImprovementSignalSchema = z
11
+ .object({
12
+ kind: z.enum(["none", "demonstrations", "chosen_rejected", "scalar_reward"]),
13
+ strength: z.enum(["absent", "weak", "usable"]),
14
+ calibrated: z.boolean(),
15
+ confounded: z.boolean(),
16
+ variance: z.number().finite().nonnegative().nullable(),
17
+ evidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
18
+ })
19
+ .strict();
20
+ export const ModelImprovementQualificationReceiptContentSchema = z
21
+ .object({
22
+ schemaVersion: z.literal("openpond.modelImprovementQualificationReceipt.v1"),
23
+ id: ReleaseIdSchema,
24
+ review: ImmutableReleaseRefSchema,
25
+ harnessRelease: ImmutableReleaseRefSchema,
26
+ tasksetRelease: ImmutableReleaseRefSchema.nullable(),
27
+ baselineEvaluation: ImmutableReleaseRefSchema.nullable(),
28
+ model: ModelRefSchema,
29
+ environmentHash: ReleaseHashSchema.nullable(),
30
+ toolContractHash: ReleaseHashSchema.nullable(),
31
+ permissionContractHash: ReleaseHashSchema.nullable(),
32
+ policyHash: ReleaseHashSchema.nullable(),
33
+ verifierRef: ImmutableReleaseRefSchema.nullable(),
34
+ sourcePolicies: z.array(HarnessReviewSourcePolicyRefSchema).max(10_000),
35
+ trainingEvidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
36
+ frozenEvaluationEvidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
37
+ privacyApproval: ImmutableReleaseRefSchema.nullable(),
38
+ budgetApproval: ImmutableReleaseRefSchema.nullable(),
39
+ maximumCostUsd: z.number().finite().nonnegative(),
40
+ signal: ModelImprovementSignalSchema,
41
+ decision: ModelImprovementDecisionSchema,
42
+ reasons: z.array(BoundedTextSchema).min(1).max(100),
43
+ createdAt: ReleaseTimestampSchema,
44
+ metadata: MetadataSchema,
45
+ })
46
+ .strict()
47
+ .superRefine((receipt, context) => {
48
+ const trainingKeys = new Set(receipt.trainingEvidenceRefs.map(refKey));
49
+ if (receipt.frozenEvaluationEvidenceRefs.some((reference) => trainingKeys.has(refKey(reference)))) {
50
+ context.addIssue({
51
+ code: "custom",
52
+ message: "frozen Evaluation evidence cannot be used as training evidence",
53
+ path: ["frozenEvaluationEvidenceRefs"],
54
+ });
55
+ }
56
+ if (receipt.decision === "no_training")
57
+ return;
58
+ const missingGate = !receipt.tasksetRelease ||
59
+ !receipt.baselineEvaluation ||
60
+ !receipt.environmentHash ||
61
+ !receipt.toolContractHash ||
62
+ !receipt.permissionContractHash ||
63
+ !receipt.policyHash ||
64
+ !receipt.verifierRef ||
65
+ !receipt.privacyApproval ||
66
+ !receipt.budgetApproval ||
67
+ receipt.sourcePolicies.length === 0 ||
68
+ receipt.sourcePolicies.some((policy) => policy.state !== "authorized") ||
69
+ receipt.trainingEvidenceRefs.length === 0 ||
70
+ receipt.signal.strength !== "usable" ||
71
+ !receipt.signal.calibrated ||
72
+ receipt.signal.confounded;
73
+ if (missingGate) {
74
+ context.addIssue({
75
+ code: "custom",
76
+ message: "qualified model improvement requires frozen lineage, authorized signal, privacy, and budget gates",
77
+ });
78
+ }
79
+ if (receipt.decision === "sft" &&
80
+ receipt.signal.kind !== "demonstrations") {
81
+ context.addIssue({
82
+ code: "custom",
83
+ message: "SFT qualification requires demonstration signal",
84
+ path: ["signal", "kind"],
85
+ });
86
+ }
87
+ if (receipt.decision === "preference" &&
88
+ receipt.signal.kind !== "chosen_rejected") {
89
+ context.addIssue({
90
+ code: "custom",
91
+ message: "preference qualification requires chosen/rejected signal",
92
+ path: ["signal", "kind"],
93
+ });
94
+ }
95
+ if (receipt.decision === "rl" &&
96
+ (receipt.signal.kind !== "scalar_reward" ||
97
+ receipt.signal.variance === null ||
98
+ receipt.signal.variance <= 0)) {
99
+ context.addIssue({
100
+ code: "custom",
101
+ message: "RL qualification requires a usable scalar reward with variance",
102
+ path: ["signal"],
103
+ });
104
+ }
105
+ });
106
+ export const ModelImprovementQualificationReceiptSchema = ModelImprovementQualificationReceiptContentSchema.extend({
107
+ contentHash: ReleaseHashSchema,
108
+ }).strict();
109
+ export function createModelImprovementQualificationReceipt(input) {
110
+ const content = ModelImprovementQualificationReceiptContentSchema.parse(input);
111
+ return ModelImprovementQualificationReceiptSchema.parse({
112
+ ...content,
113
+ contentHash: contentHash(content),
114
+ });
115
+ }
116
+ export function verifyModelImprovementQualificationReceipt(value) {
117
+ const parsed = ModelImprovementQualificationReceiptSchema.safeParse(value);
118
+ if (!parsed.success)
119
+ return false;
120
+ const { contentHash: actual, ...content } = parsed.data;
121
+ return (contentHash(ModelImprovementQualificationReceiptContentSchema.parse(content)) === actual);
122
+ }
123
+ function refKey(reference) {
124
+ return `${reference.id}:${reference.contentHash}`;
125
+ }