@openpond/evals 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +50 -2
- package/README.md +38 -6
- package/RELEASING.md +30 -24
- package/dist/compatibility.js +45 -0
- package/dist/conformance.js +6 -9
- package/dist/evidence/authoring.js +40 -0
- package/dist/evidence/conformance.js +194 -0
- package/dist/evidence/contracts.js +407 -0
- package/dist/evidence/eligibility.js +168 -0
- package/dist/evidence/index.js +4 -0
- package/dist/harness-improvements.js +329 -0
- package/dist/harness-workspaces.js +368 -0
- package/dist/harness.js +4 -6
- package/dist/index.js +4 -0
- package/dist/runs.js +33 -5
- package/dist/tasksets.js +2 -3
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/compatibility.d.ts +11 -0
- package/dist/types/compatibility.d.ts.map +1 -0
- package/dist/types/conformance.d.ts +8 -48
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/authoring.d.ts +41 -0
- package/dist/types/evidence/authoring.d.ts.map +1 -0
- package/dist/types/evidence/conformance.d.ts +767 -0
- package/dist/types/evidence/conformance.d.ts.map +1 -0
- package/dist/types/evidence/contracts.d.ts +873 -0
- package/dist/types/evidence/contracts.d.ts.map +1 -0
- package/dist/types/evidence/eligibility.d.ts +348 -0
- package/dist/types/evidence/eligibility.d.ts.map +1 -0
- package/dist/types/evidence/index.d.ts +5 -0
- package/dist/types/evidence/index.d.ts.map +1 -0
- package/dist/types/graders.d.ts.map +1 -1
- package/dist/types/harness-improvements.d.ts +523 -0
- package/dist/types/harness-improvements.d.ts.map +1 -0
- package/dist/types/harness-workspaces.d.ts +802 -0
- package/dist/types/harness-workspaces.d.ts.map +1 -0
- package/dist/types/harness.d.ts +6 -68
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/runs.d.ts +46 -1
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/sha256.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -10
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +14 -2
package/CONTRACT.md
CHANGED
|
@@ -6,13 +6,16 @@ authorization, model streaming, artifact bytes, and runtime processes.
|
|
|
6
6
|
|
|
7
7
|
| Existing object | Portable object | Migration rule |
|
|
8
8
|
| --- | --- | --- |
|
|
9
|
-
| `Taskset` | `TasksetRelease` | Project only the released tasks, policy, environment, tools, capabilities, graders
|
|
9
|
+
| `Taskset` | `TasksetRelease` | Project only the released tasks, policy, environment, tools, capabilities, and graders. A Taskset is deliberately independent of any Harness so the same workload can run against local or hosted execution. Authoring state and UI readiness remain host state. |
|
|
10
10
|
| `HarnessRunManifest` (`openpond.harnessRunManifest.v1`) | `RunManifest` (`openpond.runManifest.v1`) | Treat the old object as a legacy training projection. Normalize its release/model/runtime identities into one new manifest; recipe, compute, engine, secret leases, and approval records remain host bindings referenced by hashes. |
|
|
11
11
|
| `HarnessRunTrace` | `HarnessTrace` | Preserve ordered actions, observations, lifecycle events, terminal state, failure class, and trace hash. Learning-signal envelopes remain a training projection of the trace and receipt. |
|
|
12
12
|
| `TaskAttemptResult` | `AttemptReceipt` | Preserve the old record for application persistence while adding a lossless receipt reference. Output becomes `outputHash`; trace and artifacts are separately hash-bound. |
|
|
13
13
|
| `GradeResult` | `GraderEvidence[]` | Preserve component score, pass, reward eligibility, failure class, feedback, and visible/private evidence references. Aggregate UI results remain host projections. |
|
|
14
14
|
| managed-RL local receipt | `AttemptReceipt` | Submit canonical manifest/task/trace/artifact/grader identities; policy token responses and provider request IDs remain host-private trace data. |
|
|
15
|
-
| resolved training bundle | `HarnessRelease` + host training binding |
|
|
15
|
+
| resolved training bundle | `HarnessRelease` + host training binding | Agent snapshot, program, lifecycle, tool declarations, files, and grader interface belong to the Harness. Taskset environment/policy/graders and dataset/evidence, recipe, compute, engine, approval, and opaque leases remain explicit host-side bindings. |
|
|
16
|
+
| completed Work or Development turn | `WorkEvidenceReceipt` | Project the authoritative terminal turn, immutable Agent snapshot when available, model/runtime identity, sanitized trace reference, exact output revisions, validation evidence, interventions, timing, usage, and explicit consent provenance. Keep the raw source and trace host-private. |
|
|
17
|
+
| Agent plus environment runtime events | `WorkProcessTrace` | Emit one ordered trace with `agent` and `environment` layers. Bind every environment step to its outer Agent tool call or stable Agent-turn receipt hash. Hash inputs/outputs and expose only enumerated, bounded attributes. |
|
|
18
|
+
| user feedback on Work output | `WorkFeedbackReceipt` | Append a new receipt bound to the evidence receipt and, when selected, the exact content-addressed output-revision descriptor. Corrections are separate artifacts and never mutate prior receipts. |
|
|
16
19
|
|
|
17
20
|
## Compatibility policy
|
|
18
21
|
|
|
@@ -23,12 +26,57 @@ authorization, model streaming, artifact bytes, and runtime processes.
|
|
|
23
26
|
- Compatible package releases may add optional helpers and exports. Changing a
|
|
24
27
|
required field, identity hash, or privacy boundary requires a new schema
|
|
25
28
|
literal and an explicit normalizer.
|
|
29
|
+
- `openpond.agentSnapshot.v2`, `openpond.harnessRelease.v2`, and
|
|
30
|
+
`openpond.tasksetRelease.v2` define the Harness-first boundary. The v2
|
|
31
|
+
contracts remove the Profile reference from the Agent snapshot and keep the
|
|
32
|
+
Taskset independent of a concrete Harness, environment, or policy binding.
|
|
33
|
+
- Runs with different Harness releases require an explicit
|
|
34
|
+
`HarnessCompatibilityReceipt` binding both Harnesses to the same Taskset and
|
|
35
|
+
recording environment, tool, policy, and grader-interface contract hashes.
|
|
36
|
+
Callers with materialized releases should use
|
|
37
|
+
`createVerifiedHarnessCompatibilityReceipt`; it derives those hashes from the
|
|
38
|
+
immutable objects and rejects lifecycle, tool, grader-interface, or required
|
|
39
|
+
Environment-tool drift before issuing the receipt.
|
|
26
40
|
- The initial support target is Node.js ESM on Node 22.14 through Node 24.
|
|
27
41
|
- Portable paths are relative and at most 2,000 characters. Individual assets
|
|
28
42
|
are at most 250 MB. Tasksets, traces, and evidence arrays have schema-level
|
|
29
43
|
upper bounds.
|
|
30
44
|
- Immutable content never contains secret values, opaque lease values, mutable
|
|
31
45
|
provider resource IDs, database keys, UI state, or process handles.
|
|
46
|
+
- The Work evidence schemas first ship as additive `0.2.x` package exports. The
|
|
47
|
+
package version does not replace the `openpond.workEvidenceReceipt.v1`,
|
|
48
|
+
`openpond.workProcessTrace.v1`, `openpond.workFeedbackReceipt.v1`, or
|
|
49
|
+
`openpond.workEvidenceEligibility.v1` schema literals.
|
|
50
|
+
|
|
51
|
+
## Work evidence boundary
|
|
52
|
+
|
|
53
|
+
- One `WorkEvidenceReceipt` spans the Agent and its environment or sandbox. A
|
|
54
|
+
later Taskset/Harness replay emits one `AttemptReceipt`, not separate Agent and
|
|
55
|
+
sandbox attempts.
|
|
56
|
+
- Portable traces contain no private-trace path or reference. Source identities
|
|
57
|
+
are opaque SHA-256 URNs; artifacts and output-revision descriptors are
|
|
58
|
+
content-addressed. Hidden reasoning, prompts, raw command arguments/output,
|
|
59
|
+
local paths, credentials, provider handles, and raw validation text do not
|
|
60
|
+
cross the portable boundary.
|
|
61
|
+
- Step attributes are closed and bounded: tool/validation/transition categories,
|
|
62
|
+
intervention outcomes, artifact counts, exit status, duration, available CPU
|
|
63
|
+
time and peak memory, and error class. Adding an arbitrary metadata bag would
|
|
64
|
+
be a privacy-boundary change.
|
|
65
|
+
- Explicit process-and-artifact consent is required. Existing transcript consent
|
|
66
|
+
is not upgraded implicitly. Workspace-owned or multi-participant evidence must
|
|
67
|
+
satisfy the host's ownership and participant policy before projection.
|
|
68
|
+
- Receipts are immutable observations; current consent status is not. Revoked,
|
|
69
|
+
expired, or source-deleted evidence is ineligible for all downstream uses.
|
|
70
|
+
Hosts enforce source-bound deletion and distribution withdrawal outside the
|
|
71
|
+
immutable receipt.
|
|
72
|
+
- An incomplete trace remains valid observational evidence but is blocked from
|
|
73
|
+
eval, training, reward, and qualitative use according to the eligibility
|
|
74
|
+
report. Reward candidacy additionally requires a verified and reward-eligible
|
|
75
|
+
`AttemptReceipt` bound by hash.
|
|
76
|
+
- Feedback is append-only. `outputRevisionRef` identifies an exact revision
|
|
77
|
+
descriptor even when two revisions contain identical bytes; correction
|
|
78
|
+
content remains a separate private artifact unless a later policy explicitly
|
|
79
|
+
permits disclosure.
|
|
32
80
|
|
|
33
81
|
## Runtime interfaces
|
|
34
82
|
|
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# `@openpond/evals`
|
|
2
2
|
|
|
3
3
|
Portable OpenPond contracts and reference helpers for immutable Agent snapshots,
|
|
4
|
-
Harness releases, Taskset releases, run manifests, attempt receipts,
|
|
5
|
-
graders, and conformance fixtures.
|
|
4
|
+
Harness releases, Taskset releases, run manifests, attempt receipts, observational
|
|
5
|
+
Work evidence, deterministic graders, and conformance fixtures.
|
|
6
6
|
|
|
7
7
|
```ts
|
|
8
8
|
import {
|
|
@@ -12,13 +12,38 @@ import {
|
|
|
12
12
|
validateTasksetRelease,
|
|
13
13
|
verifyAttemptReceipt,
|
|
14
14
|
} from "@openpond/evals";
|
|
15
|
+
|
|
16
|
+
import {
|
|
17
|
+
WorkEvidenceReceiptSchema,
|
|
18
|
+
classifyWorkEvidence,
|
|
19
|
+
workEvidenceConformance,
|
|
20
|
+
} from "@openpond/evals/evidence";
|
|
15
21
|
```
|
|
16
22
|
|
|
17
23
|
Subpath exports are available at `/harness`, `/tasksets`, `/graders`, `/runs`,
|
|
18
|
-
and `/
|
|
24
|
+
`/conformance`, and `/evidence`. The package is a protocol library, not a hosted client. It
|
|
19
25
|
does not execute OpenPond Desktop or Sandbox sessions, resolve credentials, or
|
|
20
26
|
persist artifacts.
|
|
21
27
|
|
|
28
|
+
## Work evidence
|
|
29
|
+
|
|
30
|
+
`WorkEvidenceReceipt` records one completed Work or Development turn as a single
|
|
31
|
+
observational evidence unit. Its sanitized `WorkProcessTrace` combines Agent
|
|
32
|
+
actions with environment or sandbox actions. Every environment step is bound to
|
|
33
|
+
the outer Agent tool-call receipt hash or, for automatic lifecycle work, the
|
|
34
|
+
stable Agent-turn receipt hash. Commands, exit status, lifecycle transitions,
|
|
35
|
+
output and validation references, timing, and available CPU or peak-memory
|
|
36
|
+
observations stay in that one trace.
|
|
37
|
+
|
|
38
|
+
This receipt is not an evaluation result. A Taskset/Harness replay still emits
|
|
39
|
+
exactly one `AttemptReceipt`; callers may classify Work evidence as an eval,
|
|
40
|
+
training, reward, or qualitative candidate only through the explicit eligibility
|
|
41
|
+
report. Reward candidacy requires a verified, reward-eligible Attempt receipt.
|
|
42
|
+
|
|
43
|
+
Feedback is append-only and can bind an exact content-addressed output revision.
|
|
44
|
+
Corrections remain separate artifacts and are valid only for
|
|
45
|
+
`needs_correction` feedback.
|
|
46
|
+
|
|
22
47
|
## Privacy boundary
|
|
23
48
|
|
|
24
49
|
Release content may include approved policy-visible assets and immutable hashes
|
|
@@ -28,6 +53,14 @@ mutable local UI state. Private graders receive only their declared evidence and
|
|
|
28
53
|
must execute outside the policy-controlled Agent environment when tamper
|
|
29
54
|
resistance matters.
|
|
30
55
|
|
|
56
|
+
Work evidence requires explicit `work_process_and_artifacts` consent; transcript
|
|
57
|
+
consent is not sufficient. Portable receipts expose only opaque source/workspace
|
|
58
|
+
URNs, content hashes, bounded enums and counters, and sanitized artifact
|
|
59
|
+
references. Raw traces and correction bytes stay host-private. Consent,
|
|
60
|
+
revocation, expiry, deletion, ownership, participant policy, and source-bound
|
|
61
|
+
retention remain authoritative host policy state; an immutable receipt never
|
|
62
|
+
overrides that state.
|
|
63
|
+
|
|
31
64
|
## Runtime adapters
|
|
32
65
|
|
|
33
66
|
Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
|
|
@@ -55,6 +88,5 @@ pnpm --dir packages/evals run check
|
|
|
55
88
|
pnpm run release:evals:patch -- --dry-run
|
|
56
89
|
```
|
|
57
90
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
provenance; see the release workflow for the exact one-time setup.
|
|
91
|
+
Publication uses package-specific `evals-vX.Y.Z` tags, npm trusted publishing,
|
|
92
|
+
and provenance. See the release workflow for release preparation and verification.
|
package/RELEASING.md
CHANGED
|
@@ -1,49 +1,55 @@
|
|
|
1
1
|
# Releasing `@openpond/evals`
|
|
2
2
|
|
|
3
3
|
The package version is independent from OpenPond application and schema
|
|
4
|
-
versions. Version `0.
|
|
4
|
+
versions. Version `0.3.0` is the next Harness-first release and supports
|
|
5
|
+
these initial schema literals:
|
|
5
6
|
|
|
6
|
-
- `openpond.agentSnapshot.
|
|
7
|
-
- `openpond.harnessRelease.
|
|
8
|
-
- `openpond.tasksetRelease.
|
|
7
|
+
- `openpond.agentSnapshot.v2`
|
|
8
|
+
- `openpond.harnessRelease.v2`
|
|
9
|
+
- `openpond.tasksetRelease.v2`
|
|
9
10
|
- `openpond.runManifest.v1`
|
|
10
11
|
- `openpond.attemptReceipt.v1`
|
|
11
12
|
- `openpond.harnessTrace.v1`
|
|
13
|
+
- `openpond.harnessCompatibility.v1`
|
|
12
14
|
- `openpond.graderEvidence.v1`
|
|
13
15
|
- `openpond.evaluationResult.v1`
|
|
14
16
|
|
|
15
|
-
|
|
17
|
+
The package also carries the Work evidence schemas introduced in `0.2.0`:
|
|
16
18
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
- `openpond.workEvidenceReceipt.v1`
|
|
20
|
+
- `openpond.workProcessTrace.v1`
|
|
21
|
+
- `openpond.workFeedbackReceipt.v1`
|
|
22
|
+
- `openpond.workEvidenceEligibility.v1`
|
|
23
|
+
|
|
24
|
+
## Trusted publishing
|
|
25
|
+
|
|
26
|
+
The package already exists on npm and the `npm-production` environment is
|
|
27
|
+
configured for trusted publishing through `release-evals.yml`. Do not run a
|
|
28
|
+
manual `npm publish` or repeat the original bootstrap. The workflow uses its own
|
|
29
|
+
pinned npm version for trusted publication and provenance.
|
|
30
|
+
|
|
31
|
+
To inspect an already-published version:
|
|
19
32
|
|
|
20
33
|
```bash
|
|
21
34
|
pnpm evals:check
|
|
22
|
-
npm login
|
|
23
|
-
npm publish ./packages/evals --access public --ignore-scripts --provenance=false
|
|
24
|
-
npm install --global npm@^11.15.0
|
|
25
|
-
npm trust github @openpond/evals \
|
|
26
|
-
--repo openpond/openpond \
|
|
27
|
-
--file release-evals.yml \
|
|
28
|
-
--environment npm-production \
|
|
29
|
-
--allow-publish
|
|
30
35
|
npm trust list @openpond/evals
|
|
31
|
-
|
|
36
|
+
npm view @openpond/evals version dist.integrity dist.attestations
|
|
32
37
|
```
|
|
33
38
|
|
|
34
|
-
The last command is idempotent recovery: because `0.1.0` is already on npm, the
|
|
35
|
-
workflow skips publication and creates the missing `evals-v0.1.0` GitHub release.
|
|
36
|
-
|
|
37
39
|
## Later releases
|
|
38
40
|
|
|
39
|
-
|
|
41
|
+
For releases that do not already carry an intentional version bump, merge
|
|
42
|
+
feature work without changing the package version. When the intended package
|
|
43
|
+
changes are on `master`, prepare a separate release from a clean current
|
|
44
|
+
`master` checkout:
|
|
40
45
|
|
|
41
46
|
```bash
|
|
42
47
|
pnpm release:evals:patch
|
|
43
48
|
# or release:evals:minor / release:evals:major
|
|
44
49
|
```
|
|
45
50
|
|
|
46
|
-
The release helper creates an Evals-only release PR
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
release notes after
|
|
51
|
+
The release helper creates an Evals-only release PR and updates the package
|
|
52
|
+
version and lockfile. Merging it triggers trusted publishing with provenance,
|
|
53
|
+
registry verification, and a package-specific tag. Record the registry
|
|
54
|
+
integrity/provenance evidence and merge commit in the release notes after
|
|
55
|
+
publication.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { assertContentHash, contentHash } from "./common.js";
|
|
2
|
+
import { HarnessReleaseSchema, } from "./harness.js";
|
|
3
|
+
import { createHarnessCompatibilityReceipt, } from "./runs.js";
|
|
4
|
+
import { TasksetReleaseSchema, } from "./tasksets.js";
|
|
5
|
+
export function createVerifiedHarnessCompatibilityReceipt(input) {
|
|
6
|
+
const base = HarnessReleaseSchema.parse(input.baseHarnessRelease);
|
|
7
|
+
const candidate = HarnessReleaseSchema.parse(input.candidateHarnessRelease);
|
|
8
|
+
const taskset = TasksetReleaseSchema.parse(input.tasksetRelease);
|
|
9
|
+
assertContentHash(base, "Base Harness release");
|
|
10
|
+
assertContentHash(candidate, "Candidate Harness release");
|
|
11
|
+
assertContentHash(taskset, "Taskset release");
|
|
12
|
+
requireSameContract("lifecycle", base.lifecycle, candidate.lifecycle);
|
|
13
|
+
requireSameContract("tool", base.tools, candidate.tools);
|
|
14
|
+
requireSameContract("grader interface", base.graderInterface, candidate.graderInterface);
|
|
15
|
+
const environmentTools = new Set(taskset.tools.map((tool) => tool.name));
|
|
16
|
+
const unsupportedTools = base.tools
|
|
17
|
+
.map((tool) => tool.name)
|
|
18
|
+
.filter((name) => !environmentTools.has(name));
|
|
19
|
+
if (unsupportedTools.length) {
|
|
20
|
+
throw new Error(`Harness compatibility failed: Taskset Environment does not provide ${unsupportedTools.join(", ")}.`);
|
|
21
|
+
}
|
|
22
|
+
return createHarnessCompatibilityReceipt({
|
|
23
|
+
schemaVersion: "openpond.harnessCompatibility.v1",
|
|
24
|
+
id: input.id,
|
|
25
|
+
baseHarnessRelease: { id: base.id, contentHash: base.contentHash },
|
|
26
|
+
candidateHarnessRelease: {
|
|
27
|
+
id: candidate.id,
|
|
28
|
+
contentHash: candidate.contentHash,
|
|
29
|
+
},
|
|
30
|
+
tasksetRelease: { id: taskset.id, contentHash: taskset.contentHash },
|
|
31
|
+
environmentHash: contentHash(taskset.environment),
|
|
32
|
+
toolContractHash: contentHash(taskset.tools),
|
|
33
|
+
policyHash: contentHash(taskset.policy),
|
|
34
|
+
graderInterfaceHash: contentHash({
|
|
35
|
+
harness: base.graderInterface,
|
|
36
|
+
taskset: taskset.graders,
|
|
37
|
+
}),
|
|
38
|
+
metadata: input.metadata ?? {},
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
function requireSameContract(label, base, candidate) {
|
|
42
|
+
if (contentHash(base) !== contentHash(candidate)) {
|
|
43
|
+
throw new Error(`Harness compatibility failed: ${label} contract changed.`);
|
|
44
|
+
}
|
|
45
|
+
}
|
package/dist/conformance.js
CHANGED
|
@@ -14,32 +14,29 @@ export const marketingPortfolioConformance = fixture("marketing-portfolio-v1", [
|
|
|
14
14
|
]);
|
|
15
15
|
function fixture(id, tools) {
|
|
16
16
|
const snapshot = createAgentSnapshot({
|
|
17
|
-
schemaVersion: "openpond.agentSnapshot.
|
|
17
|
+
schemaVersion: "openpond.agentSnapshot.v2",
|
|
18
18
|
id: `${id}-agent`,
|
|
19
|
-
|
|
19
|
+
sourceRelease: null,
|
|
20
20
|
instructions: [], skills: [], agents: [], toolDeclarations: tools,
|
|
21
21
|
capabilityRequirements: [], dependencyLock,
|
|
22
22
|
portability: { portable: true, blockers: [], localOnlyAssetRefs: [], hostPrivateAssetRefs: [] },
|
|
23
23
|
metadata: { conformanceFixture: id },
|
|
24
24
|
});
|
|
25
25
|
const harness = createHarnessRelease({
|
|
26
|
-
schemaVersion: "openpond.harnessRelease.
|
|
26
|
+
schemaVersion: "openpond.harnessRelease.v2",
|
|
27
27
|
id: `${id}-harness`,
|
|
28
28
|
agentSnapshot: { id: snapshot.id, contentHash: snapshot.contentHash },
|
|
29
29
|
program,
|
|
30
|
-
environment: { protocolVersion: "openpond.environment.v1", kind: "agent", entrypoint: id, stateful: true, deterministicSeeds: true, lifecycle: ["create", "reset", "step", "collect", "destroy"], networkPolicy: "none", defaultTimeoutMs: 5_000 },
|
|
31
30
|
tools,
|
|
32
31
|
lifecycle: { create: true, reset: true, step: true, collect: true, destroy: true, resetScope: "attempt" },
|
|
33
32
|
graderInterface: { visibleEvidence: ["output"], privilegedEvidence: ["expected"], privateVerifierIsolation: true },
|
|
34
|
-
policy: { policyVisibleFields: ["input"], privilegedFields: ["expectedOutput"], hiddenGraderRefs: [], connectedAppScopes: [] },
|
|
35
33
|
files: [], metadata: { conformanceFixture: id },
|
|
36
34
|
});
|
|
37
35
|
const tasksetContent = {
|
|
38
|
-
schemaVersion: "openpond.tasksetRelease.
|
|
36
|
+
schemaVersion: "openpond.tasksetRelease.v2",
|
|
39
37
|
id: `${id}-taskset`, revision: 1,
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
environment: harness.environment,
|
|
38
|
+
policy: { policyVisibleFields: ["input"], privilegedFields: ["expectedOutput"], hiddenGraderRefs: [], connectedAppScopes: [] },
|
|
39
|
+
environment: { protocolVersion: "openpond.environment.v1", kind: "agent", entrypoint: id, stateful: true, deterministicSeeds: true, lifecycle: ["create", "reset", "step", "collect", "destroy"], networkPolicy: "none", defaultTimeoutMs: 5_000 },
|
|
43
40
|
tools,
|
|
44
41
|
capabilities: [],
|
|
45
42
|
tasks: [
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ImmutableReleaseRefSchema, ReleaseHashSchema } from "../common.js";
|
|
3
|
+
import { EvidenceArtifactRefSchema, WorkEvidenceReceiptSchema, evidenceArtifactRef, } from "./contracts.js";
|
|
4
|
+
import { WorkEvidenceEligibilitySchema } from "./eligibility.js";
|
|
5
|
+
export const WorkEvidenceAuthoringInputSchema = z.object({
|
|
6
|
+
schemaVersion: z.literal("openpond.workEvidenceAuthoringInput.v1"),
|
|
7
|
+
evidenceReceiptRef: EvidenceArtifactRefSchema,
|
|
8
|
+
inputHash: ReleaseHashSchema,
|
|
9
|
+
agentSnapshot: ImmutableReleaseRefSchema.nullable(),
|
|
10
|
+
sanitizedTraceRef: EvidenceArtifactRefSchema,
|
|
11
|
+
outputRefs: z.array(EvidenceArtifactRefSchema).max(10_000),
|
|
12
|
+
validationEvidenceRefs: z.array(EvidenceArtifactRefSchema).max(10_000),
|
|
13
|
+
incomplete: z.boolean(),
|
|
14
|
+
evalCandidate: z.boolean(),
|
|
15
|
+
blockerCodes: z.array(z.string().trim().min(1).max(120)).max(32),
|
|
16
|
+
}).strict();
|
|
17
|
+
export function toWorkEvidenceAuthoringInput(evidenceInput, eligibilityInput) {
|
|
18
|
+
const evidence = WorkEvidenceReceiptSchema.parse(evidenceInput);
|
|
19
|
+
const eligibility = WorkEvidenceEligibilitySchema.parse(eligibilityInput);
|
|
20
|
+
if (eligibility.evidenceReceiptHash !== evidence.contentHash) {
|
|
21
|
+
throw new Error("Eligibility report belongs to a different Work evidence receipt.");
|
|
22
|
+
}
|
|
23
|
+
const receiptRef = evidenceArtifactRef({
|
|
24
|
+
contentHash: evidence.contentHash,
|
|
25
|
+
mediaType: "application/vnd.openpond.work-evidence+json",
|
|
26
|
+
sizeBytes: null,
|
|
27
|
+
});
|
|
28
|
+
return WorkEvidenceAuthoringInputSchema.parse({
|
|
29
|
+
schemaVersion: "openpond.workEvidenceAuthoringInput.v1",
|
|
30
|
+
evidenceReceiptRef: receiptRef,
|
|
31
|
+
inputHash: evidence.inputHash,
|
|
32
|
+
agentSnapshot: evidence.agentSnapshot,
|
|
33
|
+
sanitizedTraceRef: evidence.trace.sanitizedRef,
|
|
34
|
+
outputRefs: evidence.outputRefs,
|
|
35
|
+
validationEvidenceRefs: evidence.validationEvidenceRefs,
|
|
36
|
+
incomplete: evidence.trace.incomplete,
|
|
37
|
+
evalCandidate: eligibility.decisions.eval_candidate.eligible,
|
|
38
|
+
blockerCodes: eligibility.decisions.eval_candidate.blockers,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { canonicalJson, contentHash } from "../common.js";
|
|
2
|
+
import { createWorkEvidenceReceipt, createWorkFeedbackReceipt, createWorkProcessTrace, evidenceArtifactRef, workEvidenceReceiptRef, workSourceOpaqueRef, } from "./contracts.js";
|
|
3
|
+
import { classifyWorkEvidence } from "./eligibility.js";
|
|
4
|
+
const startedAt = "2026-08-04T12:00:00.000Z";
|
|
5
|
+
const completedAt = "2026-08-04T12:00:02.500Z";
|
|
6
|
+
const sourceRevisionHash = contentHash({ fixture: "desktop-work", revision: 1 });
|
|
7
|
+
const output = evidenceArtifactRef({
|
|
8
|
+
contentHash: contentHash("fixture output"),
|
|
9
|
+
mediaType: "text/plain",
|
|
10
|
+
sizeBytes: 14,
|
|
11
|
+
});
|
|
12
|
+
const validation = evidenceArtifactRef({
|
|
13
|
+
contentHash: contentHash({ kind: "test", status: "passed" }),
|
|
14
|
+
mediaType: "application/json",
|
|
15
|
+
sizeBytes: null,
|
|
16
|
+
});
|
|
17
|
+
const consent = evidenceArtifactRef({
|
|
18
|
+
contentHash: contentHash({ scope: "work_process_and_artifacts", grantedAt: startedAt }),
|
|
19
|
+
mediaType: "application/json",
|
|
20
|
+
sizeBytes: null,
|
|
21
|
+
});
|
|
22
|
+
export const completeWorkProcessTraceFixture = createWorkProcessTrace({
|
|
23
|
+
schemaVersion: "openpond.workProcessTrace.v1",
|
|
24
|
+
sourceRevisionHash,
|
|
25
|
+
sanitationPolicyVersion: "openpond.desktop-work-sanitizer.v1",
|
|
26
|
+
incomplete: false,
|
|
27
|
+
incompleteReasons: [],
|
|
28
|
+
droppedEventCount: 1,
|
|
29
|
+
steps: [
|
|
30
|
+
step(0, startedAt, "agent", "state_transition", "turn_started", "started", {
|
|
31
|
+
transitionState: "running",
|
|
32
|
+
}),
|
|
33
|
+
step(1, "2026-08-04T12:00:00.500Z", "agent", "tool", "tool_invoked", "started", {
|
|
34
|
+
toolCategory: "filesystem",
|
|
35
|
+
inputHash: contentHash({ path: "hashed-only" }),
|
|
36
|
+
receiptHash: contentHash("fixture-tool-call"),
|
|
37
|
+
}),
|
|
38
|
+
step(2, "2026-08-04T12:00:01.000Z", "environment", "artifact", "artifact_created", "completed", {
|
|
39
|
+
outputHash: output.contentHash,
|
|
40
|
+
artifacts: [output],
|
|
41
|
+
receiptHash: contentHash("fixture-environment-operation"),
|
|
42
|
+
parentReceiptHash: contentHash("fixture-tool-call"),
|
|
43
|
+
}),
|
|
44
|
+
step(3, "2026-08-04T12:00:01.500Z", "environment", "validation", "validation_completed", "completed", {
|
|
45
|
+
validationKind: "test",
|
|
46
|
+
outputHash: validation.contentHash,
|
|
47
|
+
receiptHash: contentHash("fixture-validation"),
|
|
48
|
+
parentReceiptHash: contentHash("fixture-tool-call"),
|
|
49
|
+
}),
|
|
50
|
+
step(4, completedAt, "agent", "state_transition", "turn_completed", "completed", {
|
|
51
|
+
transitionState: "completed",
|
|
52
|
+
}),
|
|
53
|
+
],
|
|
54
|
+
});
|
|
55
|
+
const traceBytes = new TextEncoder().encode(canonicalJson(completeWorkProcessTraceFixture)).byteLength;
|
|
56
|
+
const traceRef = evidenceArtifactRef({
|
|
57
|
+
contentHash: completeWorkProcessTraceFixture.contentHash,
|
|
58
|
+
mediaType: "application/vnd.openpond.work-process-trace+json",
|
|
59
|
+
sizeBytes: traceBytes,
|
|
60
|
+
});
|
|
61
|
+
export const completeWorkEvidenceFixture = createWorkEvidenceReceipt({
|
|
62
|
+
schemaVersion: "openpond.workEvidenceReceipt.v1",
|
|
63
|
+
id: `work-evidence-${contentHash([sourceRevisionHash, traceRef.contentHash]).slice(0, 24)}`,
|
|
64
|
+
source: {
|
|
65
|
+
surface: "desktop",
|
|
66
|
+
experience: "work",
|
|
67
|
+
opaqueRef: workSourceOpaqueRef(["fixture-session", "fixture-turn"]),
|
|
68
|
+
revisionHash: sourceRevisionHash,
|
|
69
|
+
},
|
|
70
|
+
agentSnapshot: { id: "fixture-agent-snapshot", contentHash: contentHash("fixture-agent") },
|
|
71
|
+
model: {
|
|
72
|
+
provider: "fixture",
|
|
73
|
+
model: "scripted",
|
|
74
|
+
revision: "1",
|
|
75
|
+
artifactHash: null,
|
|
76
|
+
tokenizerRevision: null,
|
|
77
|
+
chatTemplateHash: null,
|
|
78
|
+
},
|
|
79
|
+
runtime: {
|
|
80
|
+
adapterId: "desktop-work",
|
|
81
|
+
adapterVersion: "1",
|
|
82
|
+
capabilityRef: null,
|
|
83
|
+
},
|
|
84
|
+
inputHash: contentHash("fixture prompt"),
|
|
85
|
+
terminal: { status: "completed", failureClass: null },
|
|
86
|
+
trace: {
|
|
87
|
+
sanitizedRef: traceRef,
|
|
88
|
+
traceHash: traceRef.contentHash,
|
|
89
|
+
sanitationPolicyVersion: completeWorkProcessTraceFixture.sanitationPolicyVersion,
|
|
90
|
+
incomplete: false,
|
|
91
|
+
},
|
|
92
|
+
outputRefs: [output],
|
|
93
|
+
artifactRefs: [output],
|
|
94
|
+
validationEvidenceRefs: [validation],
|
|
95
|
+
interventions: { approvals: 0, questions: 0, steeringEvents: 0, otherUserInterventions: 0 },
|
|
96
|
+
timing: { startedAt, completedAt, latencyMs: 2_500 },
|
|
97
|
+
usage: { promptTokens: 12, completionTokens: 8, totalTokens: 20 },
|
|
98
|
+
costUsd: 0,
|
|
99
|
+
provenance: {
|
|
100
|
+
consentReceiptRef: consent,
|
|
101
|
+
consentScope: "work_process_and_artifacts",
|
|
102
|
+
consentGrantedAt: startedAt,
|
|
103
|
+
policyVersion: "openpond.work-evidence-policy.v1",
|
|
104
|
+
projectorVersion: "openpond.desktop-work-evidence-projector.v1",
|
|
105
|
+
disclosure: "portable_sanitized",
|
|
106
|
+
ownershipScope: "personal",
|
|
107
|
+
workspaceRef: null,
|
|
108
|
+
participantPolicy: "creator_only",
|
|
109
|
+
retention: { policy: "source_bound", deleteWithSource: true, expiresAt: null },
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
export const acceptedWorkFeedbackFixture = createWorkFeedbackReceipt({
|
|
113
|
+
schemaVersion: "openpond.workFeedbackReceipt.v1",
|
|
114
|
+
id: `work-feedback-${contentHash([completeWorkEvidenceFixture.contentHash, "accepted"]).slice(0, 24)}`,
|
|
115
|
+
evidenceReceiptRef: workEvidenceReceiptRef(completeWorkEvidenceFixture),
|
|
116
|
+
outputRevisionRef: output,
|
|
117
|
+
verdict: "accepted",
|
|
118
|
+
reasonCodes: ["correct", "complete"],
|
|
119
|
+
correctionRef: null,
|
|
120
|
+
correctedOutputRevisionRef: null,
|
|
121
|
+
actor: "user",
|
|
122
|
+
createdAt: "2026-08-04T12:01:00.000Z",
|
|
123
|
+
}, completeWorkEvidenceFixture);
|
|
124
|
+
export const incompleteWorkProcessTraceFixture = createWorkProcessTrace({
|
|
125
|
+
schemaVersion: "openpond.workProcessTrace.v1",
|
|
126
|
+
sourceRevisionHash: contentHash({ fixture: "incomplete", revision: 1 }),
|
|
127
|
+
sanitationPolicyVersion: "openpond.desktop-work-sanitizer.v1",
|
|
128
|
+
incomplete: true,
|
|
129
|
+
incompleteReasons: ["missing_terminal", "unsupported_events_dropped"],
|
|
130
|
+
droppedEventCount: 2,
|
|
131
|
+
steps: [step(0, startedAt, "agent", "state_transition", "turn_started", "started", {
|
|
132
|
+
transitionState: "running",
|
|
133
|
+
})],
|
|
134
|
+
});
|
|
135
|
+
export const activeWorkEvidenceEligibilityFixture = classifyWorkEvidence({
|
|
136
|
+
evidence: completeWorkEvidenceFixture,
|
|
137
|
+
feedback: [acceptedWorkFeedbackFixture],
|
|
138
|
+
policyState: "active",
|
|
139
|
+
reconstructability: { input: true, environment: true, verifier: true },
|
|
140
|
+
});
|
|
141
|
+
export const revokedWorkEvidenceEligibilityFixture = classifyWorkEvidence({
|
|
142
|
+
evidence: completeWorkEvidenceFixture,
|
|
143
|
+
feedback: [acceptedWorkFeedbackFixture],
|
|
144
|
+
policyState: "revoked",
|
|
145
|
+
reconstructability: { input: true, environment: true, verifier: true },
|
|
146
|
+
});
|
|
147
|
+
export const invalidRawEvidenceFixture = {
|
|
148
|
+
...completeWorkEvidenceFixture,
|
|
149
|
+
source: {
|
|
150
|
+
...completeWorkEvidenceFixture.source,
|
|
151
|
+
opaqueRef: "raw-database-turn-id",
|
|
152
|
+
},
|
|
153
|
+
privateRef: "/private/runtime/trace.json",
|
|
154
|
+
publicData: {
|
|
155
|
+
reasoning: "hidden model reasoning",
|
|
156
|
+
apiKey: "secret-like-value",
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
export const workEvidenceConformance = {
|
|
160
|
+
trace: completeWorkProcessTraceFixture,
|
|
161
|
+
receipt: completeWorkEvidenceFixture,
|
|
162
|
+
feedback: acceptedWorkFeedbackFixture,
|
|
163
|
+
activeEligibility: activeWorkEvidenceEligibilityFixture,
|
|
164
|
+
revokedEligibility: revokedWorkEvidenceEligibilityFixture,
|
|
165
|
+
incompleteTrace: incompleteWorkProcessTraceFixture,
|
|
166
|
+
invalidRawEvidence: invalidRawEvidenceFixture,
|
|
167
|
+
};
|
|
168
|
+
function step(sequence, timestamp, layer, kind, action, status, input) {
|
|
169
|
+
return {
|
|
170
|
+
sequence,
|
|
171
|
+
timestamp,
|
|
172
|
+
layer,
|
|
173
|
+
kind,
|
|
174
|
+
action,
|
|
175
|
+
status,
|
|
176
|
+
inputHash: input.inputHash ?? null,
|
|
177
|
+
outputHash: input.outputHash ?? null,
|
|
178
|
+
receiptHash: input.receiptHash ?? null,
|
|
179
|
+
parentReceiptHash: input.parentReceiptHash ?? null,
|
|
180
|
+
artifactRefs: input.artifacts ?? [],
|
|
181
|
+
attributes: {
|
|
182
|
+
toolCategory: input.toolCategory ?? null,
|
|
183
|
+
validationKind: input.validationKind ?? null,
|
|
184
|
+
transitionState: input.transitionState ?? null,
|
|
185
|
+
interventionOutcome: null,
|
|
186
|
+
artifactCount: input.artifacts?.length ?? 0,
|
|
187
|
+
exitCode: null,
|
|
188
|
+
durationMs: null,
|
|
189
|
+
cpuTimeMs: null,
|
|
190
|
+
memoryPeakBytes: null,
|
|
191
|
+
errorClass: null,
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|