@openpond/evals 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +11 -3
- package/README.md +27 -8
- package/RELEASING.md +13 -46
- package/dist/compatibility.js +2 -2
- package/dist/conformance.js +1 -2
- package/dist/evidence/authoring.js +1 -1
- package/dist/evidence/conformance.js +1 -1
- package/dist/evidence/contracts.js +1 -2
- package/dist/evidence/eligibility.js +1 -1
- package/dist/graders.js +1 -1
- package/dist/harness.js +33 -101
- package/dist/index.js +2 -3
- package/dist/model-improvement-qualification.js +125 -0
- package/dist/review-conformance.js +209 -0
- package/dist/runs.js +1 -9
- package/dist/tasksets.js +2 -15
- package/dist/types/compatibility.d.ts +1 -1
- package/dist/types/compatibility.d.ts.map +1 -1
- package/dist/types/conformance.d.ts +2 -0
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +20 -20
- package/dist/types/evidence/contracts.d.ts +18 -18
- package/dist/types/harness.d.ts +3 -409
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/model-improvement-qualification.d.ts +222 -0
- package/dist/types/model-improvement-qualification.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +607 -0
- package/dist/types/review-conformance.d.ts.map +1 -0
- package/dist/types/runs.d.ts +1 -9
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -25
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +14 -10
- package/dist/common.js +0 -67
- package/dist/harness-improvements.js +0 -329
- package/dist/harness-workspaces.js +0 -368
- package/dist/sha256.js +0 -91
- package/dist/types/common.d.ts +0 -51
- package/dist/types/common.d.ts.map +0 -1
- package/dist/types/harness-improvements.d.ts +0 -523
- package/dist/types/harness-improvements.d.ts.map +0 -1
- package/dist/types/harness-workspaces.d.ts +0 -802
- package/dist/types/harness-workspaces.d.ts.map +0 -1
- package/dist/types/sha256.d.ts +0 -2
- package/dist/types/sha256.d.ts.map +0 -1
package/CONTRACT.md
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
# Portable contract map
|
|
2
2
|
|
|
3
|
-
`@openpond/
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
`@openpond/harness` owns immutable Harness identities, workspaces,
|
|
4
|
+
model-driven refinement/review policy, improvements, traces, tools, model
|
|
5
|
+
identities, and shared hashing.
|
|
6
|
+
`@openpond/evals` owns Tasksets, graders, evaluation runs and receipts,
|
|
7
|
+
execution orchestration, conformance fixtures, and Work-evidence eligibility.
|
|
8
|
+
Evals depends on Harness for exact release identities but does not re-export
|
|
9
|
+
Harness APIs; Harness never depends on Evals. Host applications own persistence, provider sessions, secret
|
|
10
|
+
leases, connected-app authorization, model streaming, artifact bytes, and
|
|
11
|
+
runtime processes.
|
|
6
12
|
|
|
7
13
|
| Existing object | Portable object | Migration rule |
|
|
8
14
|
| --- | --- | --- |
|
|
@@ -16,6 +22,8 @@ authorization, model streaming, artifact bytes, and runtime processes.
|
|
|
16
22
|
| completed Work or Development turn | `WorkEvidenceReceipt` | Project the authoritative terminal turn, immutable Agent snapshot when available, model/runtime identity, sanitized trace reference, exact output revisions, validation evidence, interventions, timing, usage, and explicit consent provenance. Keep the raw source and trace host-private. |
|
|
17
23
|
| Agent plus environment runtime events | `WorkProcessTrace` | Emit one ordered trace with `agent` and `environment` layers. Bind every environment step to its outer Agent tool call or stable Agent-turn receipt hash. Hash inputs/outputs and expose only enumerated, bounded attributes. |
|
|
18
24
|
| user feedback on Work output | `WorkFeedbackReceipt` | Append a new receipt bound to the evidence receipt and, when selected, the exact content-addressed output-revision descriptor. Corrections are separate artifacts and never mutate prior receipts. |
|
|
25
|
+
| bounded cross-Work review | `HarnessEvaluationReviewReceipt` | Select only currently authorized immutable evidence, advance one watermark, group one stable claim, route to the smallest correct layer, and name the next authority without performing downstream effects. |
|
|
26
|
+
| model-improvement qualification | `ModelImprovementQualificationReceipt` | Bind the originating review, exact Harness, Taskset, real baseline Evaluation, Model, Environment/tool/permission/policy hashes, Verifier, source policies, privacy, budget, and non-frozen signal. Weak or confounded evidence emits `no_training`; training and activation remain host effects. |
|
|
19
27
|
|
|
20
28
|
## Compatibility policy
|
|
21
29
|
|
package/README.md
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
# `@openpond/evals`
|
|
2
2
|
|
|
3
|
-
Portable
|
|
4
|
-
|
|
5
|
-
Work
|
|
3
|
+
Portable evaluation contracts and pure helpers for Tasksets, graders, run
|
|
4
|
+
manifests, attempt and evaluation receipts, execution adapters, conformance
|
|
5
|
+
fixtures, Work-evidence eligibility, and no-training/SFT/preference/RL
|
|
6
|
+
qualification receipts. The package depends on
|
|
7
|
+
[`@openpond/harness`](../harness/README.md) for exact Harness identities but
|
|
8
|
+
does not re-export Harness APIs. Applications import the two packages directly,
|
|
9
|
+
which keeps refinement and evaluation authority visibly separate.
|
|
6
10
|
|
|
7
11
|
```ts
|
|
8
12
|
import {
|
|
9
13
|
AttemptReceiptSchema,
|
|
10
|
-
HarnessReleaseSchema,
|
|
11
14
|
TasksetReleaseSchema,
|
|
15
|
+
ModelImprovementQualificationReceiptSchema,
|
|
12
16
|
validateTasksetRelease,
|
|
13
17
|
verifyAttemptReceipt,
|
|
14
18
|
} from "@openpond/evals";
|
|
15
19
|
|
|
20
|
+
import { HarnessReleaseSchema } from "@openpond/harness";
|
|
21
|
+
|
|
16
22
|
import {
|
|
17
23
|
WorkEvidenceReceiptSchema,
|
|
18
24
|
classifyWorkEvidence,
|
|
@@ -21,9 +27,22 @@ import {
|
|
|
21
27
|
```
|
|
22
28
|
|
|
23
29
|
Subpath exports are available at `/harness`, `/tasksets`, `/graders`, `/runs`,
|
|
24
|
-
`/conformance`,
|
|
25
|
-
|
|
26
|
-
|
|
30
|
+
`/conformance`, `/evidence`, `/review`, and
|
|
31
|
+
`/model-improvement-qualification`. The package is an evaluation protocol library,
|
|
32
|
+
not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
|
|
33
|
+
resolve credentials, or persist artifacts.
|
|
34
|
+
|
|
35
|
+
## Harness Evaluation review
|
|
36
|
+
|
|
37
|
+
`@openpond/harness` owns the public model-driven Refiner and continuous-review
|
|
38
|
+
policy plus immutable bounded decisions and receipts. Evals does not own
|
|
39
|
+
learning prompts, evidence selection, scheduling, or Harness mutation. It
|
|
40
|
+
binds an accepted review to the exact Harness, Taskset, scored
|
|
41
|
+
baseline Evaluation, base Model, Environment/tool/permission/policy hashes,
|
|
42
|
+
Verifier, source-policy checks, privacy approval, budget approval, and eligible
|
|
43
|
+
non-frozen learning signal. Frozen Evaluation evidence is rejected as training
|
|
44
|
+
evidence. Weak, constant, confounded, uncalibrated, revoked, or incompletely
|
|
45
|
+
authorized signal produces `no_training`; it cannot qualify a managed run.
|
|
27
46
|
|
|
28
47
|
## Work evidence
|
|
29
48
|
|
|
@@ -66,7 +85,7 @@ overrides that state.
|
|
|
66
85
|
Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
|
|
67
86
|
the host's existing model/Agent loop. Do not reimplement prompting, tool dispatch,
|
|
68
87
|
session persistence, cost accounting, cancellation, or cleanup in this package.
|
|
69
|
-
A
|
|
88
|
+
A step-based environment maps reset/step/observation semantics to the same
|
|
70
89
|
interfaces and keeps provider allocation, authentication, and cleanup in its host.
|
|
71
90
|
|
|
72
91
|
## Conformance
|
package/RELEASING.md
CHANGED
|
@@ -1,55 +1,22 @@
|
|
|
1
1
|
# Releasing `@openpond/evals`
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- `openpond.agentSnapshot.v2`
|
|
8
|
-
- `openpond.harnessRelease.v2`
|
|
9
|
-
- `openpond.tasksetRelease.v2`
|
|
10
|
-
- `openpond.runManifest.v1`
|
|
11
|
-
- `openpond.attemptReceipt.v1`
|
|
12
|
-
- `openpond.harnessTrace.v1`
|
|
13
|
-
- `openpond.harnessCompatibility.v1`
|
|
14
|
-
- `openpond.graderEvidence.v1`
|
|
15
|
-
- `openpond.evaluationResult.v1`
|
|
16
|
-
|
|
17
|
-
The package also carries the Work evidence schemas introduced in `0.2.0`:
|
|
18
|
-
|
|
19
|
-
- `openpond.workEvidenceReceipt.v1`
|
|
20
|
-
- `openpond.workProcessTrace.v1`
|
|
21
|
-
- `openpond.workFeedbackReceipt.v1`
|
|
22
|
-
- `openpond.workEvidenceEligibility.v1`
|
|
23
|
-
|
|
24
|
-
## Trusted publishing
|
|
25
|
-
|
|
26
|
-
The package already exists on npm and the `npm-production` environment is
|
|
27
|
-
configured for trusted publishing through `release-evals.yml`. Do not run a
|
|
28
|
-
manual `npm publish` or repeat the original bootstrap. The workflow uses its own
|
|
29
|
-
pinned npm version for trusted publication and provenance.
|
|
30
|
-
|
|
31
|
-
To inspect an already-published version:
|
|
3
|
+
`@openpond/evals` is the public evaluation package for OpenPond Harnesses. It
|
|
4
|
+
depends on `@openpond/harness` for exact Harness identities but exposes only
|
|
5
|
+
evaluation APIs. Publish the matching Harness peer before Evals so npm can
|
|
6
|
+
satisfy the declared range.
|
|
32
7
|
|
|
33
8
|
```bash
|
|
34
9
|
pnpm evals:check
|
|
35
|
-
npm trust list @openpond/evals
|
|
36
|
-
npm view @openpond/evals version dist.integrity dist.attestations
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
## Later releases
|
|
40
|
-
|
|
41
|
-
For releases that do not already carry an intentional version bump, merge
|
|
42
|
-
feature work without changing the package version. When the intended package
|
|
43
|
-
changes are on `master`, prepare a separate release from a clean current
|
|
44
|
-
`master` checkout:
|
|
45
|
-
|
|
46
|
-
```bash
|
|
47
10
|
pnpm release:evals:patch
|
|
48
11
|
# or release:evals:minor / release:evals:major
|
|
49
12
|
```
|
|
50
13
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
14
|
+
Merging a release PR triggers `release-evals.yml`, trusted npm publishing,
|
|
15
|
+
registry verification, provenance verification, and the package-specific tag.
|
|
16
|
+
Release notes must call out removal or movement of a public subpath and use the
|
|
17
|
+
appropriate semver increment.
|
|
18
|
+
|
|
19
|
+
Version `0.4.0` removes the retired Harness root barrel and
|
|
20
|
+
`./harness-improvements` / `./harness-workspaces` compatibility subpaths.
|
|
21
|
+
Consumers must import those public contracts from `@openpond/harness`; this is
|
|
22
|
+
an intentional breaking package-boundary correction.
|
package/dist/compatibility.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { assertContentHash, contentHash } from "
|
|
2
|
-
import { HarnessReleaseSchema, } from "
|
|
1
|
+
import { assertContentHash, contentHash } from "@openpond/harness";
|
|
2
|
+
import { HarnessReleaseSchema, } from "@openpond/harness";
|
|
3
3
|
import { createHarnessCompatibilityReceipt, } from "./runs.js";
|
|
4
4
|
import { TasksetReleaseSchema, } from "./tasksets.js";
|
|
5
5
|
export function createVerifiedHarnessCompatibilityReceipt(input) {
|
package/dist/conformance.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { contentHash, withContentHash } from "
|
|
2
|
-
import { createAgentSnapshot, createHarnessRelease } from "./harness.js";
|
|
1
|
+
import { contentHash, createAgentSnapshot, createHarnessRelease, withContentHash, } from "@openpond/harness";
|
|
3
2
|
import { createRunManifest } from "./runs.js";
|
|
4
3
|
import { TasksetReleaseSchema } from "./tasksets.js";
|
|
5
4
|
const EMPTY_HASH = contentHash("");
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { ImmutableReleaseRefSchema, ReleaseHashSchema } from "
|
|
2
|
+
import { ImmutableReleaseRefSchema, ReleaseHashSchema } from "@openpond/harness";
|
|
3
3
|
import { EvidenceArtifactRefSchema, WorkEvidenceReceiptSchema, evidenceArtifactRef, } from "./contracts.js";
|
|
4
4
|
import { WorkEvidenceEligibilitySchema } from "./eligibility.js";
|
|
5
5
|
export const WorkEvidenceAuthoringInputSchema = z.object({
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { canonicalJson, contentHash } from "
|
|
1
|
+
import { canonicalJson, contentHash } from "@openpond/harness";
|
|
2
2
|
import { createWorkEvidenceReceipt, createWorkFeedbackReceipt, createWorkProcessTrace, evidenceArtifactRef, workEvidenceReceiptRef, workSourceOpaqueRef, } from "./contracts.js";
|
|
3
3
|
import { classifyWorkEvidence } from "./eligibility.js";
|
|
4
4
|
const startedAt = "2026-08-04T12:00:00.000Z";
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "
|
|
3
|
-
import { ModelRefSchema } from "../runs.js";
|
|
2
|
+
import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
|
|
4
3
|
export const WORK_EVIDENCE_SCHEMA_VERSION = "openpond.workEvidenceReceipt.v1";
|
|
5
4
|
export const WORK_PROCESS_TRACE_SCHEMA_VERSION = "openpond.workProcessTrace.v1";
|
|
6
5
|
export const WORK_FEEDBACK_SCHEMA_VERSION = "openpond.workFeedbackReceipt.v1";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { contentHash, ReleaseHashSchema } from "
|
|
2
|
+
import { contentHash, ReleaseHashSchema } from "@openpond/harness";
|
|
3
3
|
import { AttemptReceiptSchema, verifyAttemptReceipt } from "../runs.js";
|
|
4
4
|
import { WORK_EVIDENCE_ELIGIBILITY_SCHEMA_VERSION, WorkFeedbackReceiptSchema, createWorkFeedbackReceipt, verifyWorkEvidenceReceipt, } from "./contracts.js";
|
|
5
5
|
export const WorkEvidencePolicyStateSchema = z.enum([
|
package/dist/graders.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "
|
|
2
|
+
import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
|
|
3
3
|
export const GraderEvidenceContentSchema = z.object({
|
|
4
4
|
schemaVersion: z.literal("openpond.graderEvidence.v1"),
|
|
5
5
|
graderId: ReleaseIdSchema,
|
package/dist/harness.js
CHANGED
|
@@ -1,71 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
import { createAttemptReceipt } from "./runs.js";
|
|
5
|
-
export const PortabilityReportSchema = z.object({
|
|
6
|
-
portable: z.boolean(),
|
|
7
|
-
blockers: z.array(z.string().trim().min(1).max(2_000)).max(1_000),
|
|
8
|
-
localOnlyAssetRefs: z.array(ReleaseIdSchema).max(10_000),
|
|
9
|
-
hostPrivateAssetRefs: z.array(ReleaseIdSchema).max(10_000),
|
|
10
|
-
}).strict();
|
|
11
|
-
export const AgentSnapshotContentSchema = z.object({
|
|
12
|
-
schemaVersion: z.literal("openpond.agentSnapshot.v2"),
|
|
13
|
-
id: ReleaseIdSchema,
|
|
14
|
-
sourceRelease: ImmutableReleaseRefSchema.nullable(),
|
|
15
|
-
instructions: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
16
|
-
skills: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
17
|
-
agents: z.array(ImmutableAssetRefSchema).max(10_000),
|
|
18
|
-
toolDeclarations: z.array(ToolDeclarationSchema).max(200),
|
|
19
|
-
capabilityRequirements: z.array(z.object({
|
|
20
|
-
id: ReleaseIdSchema,
|
|
21
|
-
required: z.boolean(),
|
|
22
|
-
scopes: z.array(z.string().trim().min(1).max(500)).max(100),
|
|
23
|
-
}).strict()).max(200),
|
|
24
|
-
dependencyLock: ImmutableAssetRefSchema,
|
|
25
|
-
portability: PortabilityReportSchema,
|
|
26
|
-
metadata: MetadataSchema,
|
|
27
|
-
}).strict();
|
|
28
|
-
export const AgentSnapshotSchema = AgentSnapshotContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
|
|
29
|
-
export const LifecycleContractSchema = z.object({
|
|
30
|
-
create: z.literal(true),
|
|
31
|
-
reset: z.literal(true),
|
|
32
|
-
step: z.literal(true),
|
|
33
|
-
collect: z.literal(true),
|
|
34
|
-
destroy: z.literal(true),
|
|
35
|
-
resetScope: z.enum(["task", "attempt"]),
|
|
36
|
-
}).strict();
|
|
37
|
-
export const GraderInterfaceContractSchema = z.object({
|
|
38
|
-
visibleEvidence: z.array(ReleaseIdSchema).max(1_000),
|
|
39
|
-
privilegedEvidence: z.array(ReleaseIdSchema).max(1_000),
|
|
40
|
-
privateVerifierIsolation: z.boolean(),
|
|
41
|
-
}).strict();
|
|
42
|
-
export const HarnessReleaseContentSchema = z.object({
|
|
43
|
-
schemaVersion: z.literal("openpond.harnessRelease.v2"),
|
|
44
|
-
id: ReleaseIdSchema,
|
|
45
|
-
agentSnapshot: ImmutableReleaseRefSchema,
|
|
46
|
-
program: ImmutableAssetRefSchema,
|
|
47
|
-
tools: z.array(ToolDeclarationSchema).max(200),
|
|
48
|
-
lifecycle: LifecycleContractSchema,
|
|
49
|
-
graderInterface: GraderInterfaceContractSchema,
|
|
50
|
-
files: z.array(ImmutableAssetRefSchema).max(100_000),
|
|
51
|
-
metadata: MetadataSchema,
|
|
52
|
-
}).strict();
|
|
53
|
-
export const HarnessReleaseSchema = HarnessReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
|
|
54
|
-
export const ModelActionSchema = z.object({
|
|
55
|
-
id: ReleaseIdSchema,
|
|
56
|
-
turn: z.number().int().nonnegative(),
|
|
57
|
-
kind: z.enum(["message", "tool_call", "terminal"]),
|
|
58
|
-
name: ReleaseIdSchema.nullable(),
|
|
59
|
-
arguments: z.record(z.string(), z.unknown()),
|
|
60
|
-
content: z.string().max(1_000_000).nullable(),
|
|
61
|
-
}).strict();
|
|
62
|
-
export const ToolObservationSchema = z.object({
|
|
63
|
-
actionId: ReleaseIdSchema,
|
|
64
|
-
turn: z.number().int().nonnegative(),
|
|
65
|
-
terminal: z.boolean(),
|
|
66
|
-
output: z.record(z.string(), z.unknown()),
|
|
67
|
-
artifactRefs: z.array(ImmutableArtifactRefSchema).max(100_000),
|
|
68
|
-
}).strict();
|
|
1
|
+
import { HarnessTraceSchema, ModelActionSchema, ToolObservationSchema, contentHash, } from "@openpond/harness";
|
|
2
|
+
import { createAttemptReceipt, } from "./runs.js";
|
|
3
|
+
export * from "@openpond/harness/harness";
|
|
69
4
|
export async function executeRuntimeProtocol(input) {
|
|
70
5
|
const now = input.now ?? (() => new Date().toISOString());
|
|
71
6
|
const startedAt = now();
|
|
@@ -84,12 +19,25 @@ export async function executeRuntimeProtocol(input) {
|
|
|
84
19
|
let output = {};
|
|
85
20
|
let artifactRefs = [];
|
|
86
21
|
const event = (type, payload, metadata = {}) => {
|
|
87
|
-
events.push({
|
|
22
|
+
events.push({
|
|
23
|
+
sequence: sequence++,
|
|
24
|
+
type,
|
|
25
|
+
payloadHash: contentHash(payload),
|
|
26
|
+
metadata,
|
|
27
|
+
});
|
|
88
28
|
};
|
|
89
29
|
try {
|
|
90
|
-
lease = await input.runtime.create({
|
|
30
|
+
lease = await input.runtime.create({
|
|
31
|
+
manifest: input.manifest,
|
|
32
|
+
taskId: input.taskId,
|
|
33
|
+
seed: input.seed,
|
|
34
|
+
signal: controller.signal,
|
|
35
|
+
});
|
|
91
36
|
event("created", { leaseId: lease.id });
|
|
92
|
-
await input.runtime.reset(lease, {
|
|
37
|
+
await input.runtime.reset(lease, {
|
|
38
|
+
seed: input.seed,
|
|
39
|
+
signal: controller.signal,
|
|
40
|
+
});
|
|
93
41
|
event("reset", { seed: input.seed });
|
|
94
42
|
for (const candidate of input.actions.slice(0, input.manifest.limits.maxTurns)) {
|
|
95
43
|
if (controller.signal.aborted)
|
|
@@ -152,12 +100,23 @@ export async function executeRuntimeProtocol(input) {
|
|
|
152
100
|
failureClass,
|
|
153
101
|
output,
|
|
154
102
|
};
|
|
155
|
-
const trace = HarnessTraceSchema.parse({
|
|
103
|
+
const trace = HarnessTraceSchema.parse({
|
|
104
|
+
...traceContent,
|
|
105
|
+
contentHash: contentHash(traceContent),
|
|
106
|
+
});
|
|
156
107
|
const completedAt = now();
|
|
157
108
|
const receipt = createAttemptReceipt({
|
|
158
109
|
schemaVersion: "openpond.attemptReceipt.v1",
|
|
159
|
-
id: `attempt-${contentHash([
|
|
160
|
-
|
|
110
|
+
id: `attempt-${contentHash([
|
|
111
|
+
input.manifest.contentHash,
|
|
112
|
+
input.taskId,
|
|
113
|
+
input.seed,
|
|
114
|
+
trace.contentHash,
|
|
115
|
+
]).slice(0, 24)}`,
|
|
116
|
+
runManifest: {
|
|
117
|
+
id: input.manifest.id,
|
|
118
|
+
contentHash: input.manifest.contentHash,
|
|
119
|
+
},
|
|
161
120
|
taskId: input.taskId,
|
|
162
121
|
seed: input.seed,
|
|
163
122
|
terminal,
|
|
@@ -175,30 +134,3 @@ export async function executeRuntimeProtocol(input) {
|
|
|
175
134
|
});
|
|
176
135
|
return { receipt, trace, output };
|
|
177
136
|
}
|
|
178
|
-
export function createAgentSnapshot(input) {
|
|
179
|
-
const content = AgentSnapshotContentSchema.parse(input);
|
|
180
|
-
return AgentSnapshotSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
181
|
-
}
|
|
182
|
-
export function createHarnessRelease(input) {
|
|
183
|
-
const content = HarnessReleaseContentSchema.parse(input);
|
|
184
|
-
return HarnessReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
185
|
-
}
|
|
186
|
-
export const HarnessLifecycleEventSchema = z.object({
|
|
187
|
-
sequence: z.number().int().nonnegative(),
|
|
188
|
-
type: z.enum(["created", "reset", "action", "observation", "terminal", "failure", "collected", "destroyed"]),
|
|
189
|
-
payloadHash: ReleaseHashSchema,
|
|
190
|
-
metadata: MetadataSchema,
|
|
191
|
-
}).strict();
|
|
192
|
-
export const HarnessTraceSchema = z.object({
|
|
193
|
-
schemaVersion: z.literal("openpond.harnessTrace.v1"),
|
|
194
|
-
manifest: ImmutableReleaseRefSchema,
|
|
195
|
-
taskId: ReleaseIdSchema,
|
|
196
|
-
seed: z.string().trim().min(1).max(500),
|
|
197
|
-
events: z.array(HarnessLifecycleEventSchema).max(1_000_000),
|
|
198
|
-
actions: z.array(ModelActionSchema).max(100_000),
|
|
199
|
-
observations: z.array(ToolObservationSchema).max(100_000),
|
|
200
|
-
terminal: z.boolean(),
|
|
201
|
-
failureClass: FailureClassSchema.nullable(),
|
|
202
|
-
output: z.record(z.string(), z.unknown()),
|
|
203
|
-
contentHash: ReleaseHashSchema,
|
|
204
|
-
}).strict();
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
export * from "./common.js";
|
|
2
1
|
export * from "./compatibility.js";
|
|
3
2
|
export * from "./evidence/index.js";
|
|
4
3
|
export * from "./graders.js";
|
|
5
4
|
export * from "./harness.js";
|
|
6
|
-
export * from "./harness-improvements.js";
|
|
7
|
-
export * from "./harness-workspaces.js";
|
|
8
5
|
export * from "./runs.js";
|
|
6
|
+
export * from "./model-improvement-qualification.js";
|
|
7
|
+
export * from "./review-conformance.js";
|
|
9
8
|
export * from "./tasksets.js";
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { HarnessReviewSourcePolicyRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
|
|
3
|
+
const BoundedTextSchema = z.string().trim().min(1).max(100_000);
|
|
4
|
+
export const ModelImprovementDecisionSchema = z.enum([
|
|
5
|
+
"no_training",
|
|
6
|
+
"sft",
|
|
7
|
+
"preference",
|
|
8
|
+
"rl",
|
|
9
|
+
]);
|
|
10
|
+
export const ModelImprovementSignalSchema = z
|
|
11
|
+
.object({
|
|
12
|
+
kind: z.enum(["none", "demonstrations", "chosen_rejected", "scalar_reward"]),
|
|
13
|
+
strength: z.enum(["absent", "weak", "usable"]),
|
|
14
|
+
calibrated: z.boolean(),
|
|
15
|
+
confounded: z.boolean(),
|
|
16
|
+
variance: z.number().finite().nonnegative().nullable(),
|
|
17
|
+
evidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
|
|
18
|
+
})
|
|
19
|
+
.strict();
|
|
20
|
+
export const ModelImprovementQualificationReceiptContentSchema = z
|
|
21
|
+
.object({
|
|
22
|
+
schemaVersion: z.literal("openpond.modelImprovementQualificationReceipt.v1"),
|
|
23
|
+
id: ReleaseIdSchema,
|
|
24
|
+
review: ImmutableReleaseRefSchema,
|
|
25
|
+
harnessRelease: ImmutableReleaseRefSchema,
|
|
26
|
+
tasksetRelease: ImmutableReleaseRefSchema.nullable(),
|
|
27
|
+
baselineEvaluation: ImmutableReleaseRefSchema.nullable(),
|
|
28
|
+
model: ModelRefSchema,
|
|
29
|
+
environmentHash: ReleaseHashSchema.nullable(),
|
|
30
|
+
toolContractHash: ReleaseHashSchema.nullable(),
|
|
31
|
+
permissionContractHash: ReleaseHashSchema.nullable(),
|
|
32
|
+
policyHash: ReleaseHashSchema.nullable(),
|
|
33
|
+
verifierRef: ImmutableReleaseRefSchema.nullable(),
|
|
34
|
+
sourcePolicies: z.array(HarnessReviewSourcePolicyRefSchema).max(10_000),
|
|
35
|
+
trainingEvidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
|
|
36
|
+
frozenEvaluationEvidenceRefs: z.array(ImmutableReleaseRefSchema).max(100_000),
|
|
37
|
+
privacyApproval: ImmutableReleaseRefSchema.nullable(),
|
|
38
|
+
budgetApproval: ImmutableReleaseRefSchema.nullable(),
|
|
39
|
+
maximumCostUsd: z.number().finite().nonnegative(),
|
|
40
|
+
signal: ModelImprovementSignalSchema,
|
|
41
|
+
decision: ModelImprovementDecisionSchema,
|
|
42
|
+
reasons: z.array(BoundedTextSchema).min(1).max(100),
|
|
43
|
+
createdAt: ReleaseTimestampSchema,
|
|
44
|
+
metadata: MetadataSchema,
|
|
45
|
+
})
|
|
46
|
+
.strict()
|
|
47
|
+
.superRefine((receipt, context) => {
|
|
48
|
+
const trainingKeys = new Set(receipt.trainingEvidenceRefs.map(refKey));
|
|
49
|
+
if (receipt.frozenEvaluationEvidenceRefs.some((reference) => trainingKeys.has(refKey(reference)))) {
|
|
50
|
+
context.addIssue({
|
|
51
|
+
code: "custom",
|
|
52
|
+
message: "frozen Evaluation evidence cannot be used as training evidence",
|
|
53
|
+
path: ["frozenEvaluationEvidenceRefs"],
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
if (receipt.decision === "no_training")
|
|
57
|
+
return;
|
|
58
|
+
const missingGate = !receipt.tasksetRelease ||
|
|
59
|
+
!receipt.baselineEvaluation ||
|
|
60
|
+
!receipt.environmentHash ||
|
|
61
|
+
!receipt.toolContractHash ||
|
|
62
|
+
!receipt.permissionContractHash ||
|
|
63
|
+
!receipt.policyHash ||
|
|
64
|
+
!receipt.verifierRef ||
|
|
65
|
+
!receipt.privacyApproval ||
|
|
66
|
+
!receipt.budgetApproval ||
|
|
67
|
+
receipt.sourcePolicies.length === 0 ||
|
|
68
|
+
receipt.sourcePolicies.some((policy) => policy.state !== "authorized") ||
|
|
69
|
+
receipt.trainingEvidenceRefs.length === 0 ||
|
|
70
|
+
receipt.signal.strength !== "usable" ||
|
|
71
|
+
!receipt.signal.calibrated ||
|
|
72
|
+
receipt.signal.confounded;
|
|
73
|
+
if (missingGate) {
|
|
74
|
+
context.addIssue({
|
|
75
|
+
code: "custom",
|
|
76
|
+
message: "qualified model improvement requires frozen lineage, authorized signal, privacy, and budget gates",
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
if (receipt.decision === "sft" &&
|
|
80
|
+
receipt.signal.kind !== "demonstrations") {
|
|
81
|
+
context.addIssue({
|
|
82
|
+
code: "custom",
|
|
83
|
+
message: "SFT qualification requires demonstration signal",
|
|
84
|
+
path: ["signal", "kind"],
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
if (receipt.decision === "preference" &&
|
|
88
|
+
receipt.signal.kind !== "chosen_rejected") {
|
|
89
|
+
context.addIssue({
|
|
90
|
+
code: "custom",
|
|
91
|
+
message: "preference qualification requires chosen/rejected signal",
|
|
92
|
+
path: ["signal", "kind"],
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
if (receipt.decision === "rl" &&
|
|
96
|
+
(receipt.signal.kind !== "scalar_reward" ||
|
|
97
|
+
receipt.signal.variance === null ||
|
|
98
|
+
receipt.signal.variance <= 0)) {
|
|
99
|
+
context.addIssue({
|
|
100
|
+
code: "custom",
|
|
101
|
+
message: "RL qualification requires a usable scalar reward with variance",
|
|
102
|
+
path: ["signal"],
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
export const ModelImprovementQualificationReceiptSchema = ModelImprovementQualificationReceiptContentSchema.extend({
|
|
107
|
+
contentHash: ReleaseHashSchema,
|
|
108
|
+
}).strict();
|
|
109
|
+
export function createModelImprovementQualificationReceipt(input) {
|
|
110
|
+
const content = ModelImprovementQualificationReceiptContentSchema.parse(input);
|
|
111
|
+
return ModelImprovementQualificationReceiptSchema.parse({
|
|
112
|
+
...content,
|
|
113
|
+
contentHash: contentHash(content),
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export function verifyModelImprovementQualificationReceipt(value) {
|
|
117
|
+
const parsed = ModelImprovementQualificationReceiptSchema.safeParse(value);
|
|
118
|
+
if (!parsed.success)
|
|
119
|
+
return false;
|
|
120
|
+
const { contentHash: actual, ...content } = parsed.data;
|
|
121
|
+
return (contentHash(ModelImprovementQualificationReceiptContentSchema.parse(content)) === actual);
|
|
122
|
+
}
|
|
123
|
+
function refKey(reference) {
|
|
124
|
+
return `${reference.id}:${reference.contentHash}`;
|
|
125
|
+
}
|