@openpond/evals 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +24 -5
- package/README.md +27 -8
- package/RELEASING.md +13 -45
- package/dist/compatibility.js +45 -0
- package/dist/conformance.js +7 -11
- package/dist/evidence/authoring.js +1 -1
- package/dist/evidence/conformance.js +1 -1
- package/dist/evidence/contracts.js +1 -2
- package/dist/evidence/eligibility.js +1 -1
- package/dist/graders.js +1 -1
- package/dist/harness.js +33 -103
- package/dist/index.js +3 -1
- package/dist/model-improvement-qualification.js +125 -0
- package/dist/review-conformance.js +209 -0
- package/dist/runs.js +33 -13
- package/dist/tasksets.js +3 -17
- package/dist/types/compatibility.d.ts +11 -0
- package/dist/types/compatibility.d.ts.map +1 -0
- package/dist/types/conformance.d.ts +10 -48
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +8 -8
- package/dist/types/evidence/contracts.d.ts +8 -8
- package/dist/types/harness.d.ts +3 -471
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/model-improvement-qualification.d.ts +222 -0
- package/dist/types/model-improvement-qualification.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +607 -0
- package/dist/types/review-conformance.d.ts.map +1 -0
- package/dist/types/runs.d.ts +47 -10
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +4 -35
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +14 -2
- package/dist/common.js +0 -67
- package/dist/sha256.js +0 -91
- package/dist/types/common.d.ts +0 -51
- package/dist/types/common.d.ts.map +0 -1
- package/dist/types/sha256.d.ts +0 -2
- package/dist/types/sha256.d.ts.map +0 -1
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { contentHash, createHarnessEvaluationReviewReceipt, } from "@openpond/harness";
|
|
2
|
+
import { createModelImprovementQualificationReceipt } from "./model-improvement-qualification.js";
|
|
3
|
+
const createdAt = "2026-08-08T12:00:00.000Z";
|
|
4
|
+
const harnessRelease = ref("harness-release");
|
|
5
|
+
const sourcePolicy = {
|
|
6
|
+
policy: ref("source-policy"),
|
|
7
|
+
state: "authorized",
|
|
8
|
+
checkedAt: createdAt,
|
|
9
|
+
};
|
|
10
|
+
const evidence = (id, kind) => ({
|
|
11
|
+
evidence: ref(id),
|
|
12
|
+
kind,
|
|
13
|
+
sourceRef: `source-${id}`,
|
|
14
|
+
sourcePolicy,
|
|
15
|
+
occurrenceKey: contentHash(`occurrence-${id}`),
|
|
16
|
+
occurredAt: createdAt,
|
|
17
|
+
});
|
|
18
|
+
const watermark = {
|
|
19
|
+
cursor: contentHash("review-watermark"),
|
|
20
|
+
throughCreatedAt: createdAt,
|
|
21
|
+
};
|
|
22
|
+
const claim = (family, count = 3) => ({
|
|
23
|
+
fingerprint: contentHash(`claim-${family}`),
|
|
24
|
+
recurrenceFamily: family,
|
|
25
|
+
statement: `The ${family} behavior remains unresolved after smaller-layer triage.`,
|
|
26
|
+
independentOccurrences: count,
|
|
27
|
+
unresolvedOccurrences: count,
|
|
28
|
+
});
|
|
29
|
+
const noAction = createHarnessEvaluationReviewReceipt({
|
|
30
|
+
schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1",
|
|
31
|
+
id: "review-no-action",
|
|
32
|
+
ownerScope: { kind: "personal", id: "owner-1" },
|
|
33
|
+
workspaceRef: "workspace-1",
|
|
34
|
+
harnessRelease,
|
|
35
|
+
previousWatermark: null,
|
|
36
|
+
nextWatermark: watermark,
|
|
37
|
+
selectedEvidence: [],
|
|
38
|
+
excludedEvidence: [],
|
|
39
|
+
claim: null,
|
|
40
|
+
classification: "no_action",
|
|
41
|
+
triage: [],
|
|
42
|
+
reason: "The bounded evidence window contains no unresolved reusable claim.",
|
|
43
|
+
nextAuthority: "none",
|
|
44
|
+
maxEstimatedCostUsd: 0,
|
|
45
|
+
tasksetProposal: null,
|
|
46
|
+
evaluation: null,
|
|
47
|
+
trainingQualification: null,
|
|
48
|
+
policyVersion: "harness-review-policy-v1",
|
|
49
|
+
createdAt,
|
|
50
|
+
metadata: {},
|
|
51
|
+
});
|
|
52
|
+
const runtime = createHarnessEvaluationReviewReceipt({
|
|
53
|
+
...withoutHash(noAction),
|
|
54
|
+
id: "review-runtime",
|
|
55
|
+
selectedEvidence: [evidence("runtime-failure", "observation")],
|
|
56
|
+
claim: claim("runtime-transport-failure", 1),
|
|
57
|
+
classification: "runtime",
|
|
58
|
+
triage: [
|
|
59
|
+
{
|
|
60
|
+
layer: "runtime",
|
|
61
|
+
status: "unresolved",
|
|
62
|
+
reason: "The adapter failed before model policy could affect the result.",
|
|
63
|
+
evidenceRefs: [ref("runtime-failure")],
|
|
64
|
+
},
|
|
65
|
+
],
|
|
66
|
+
reason: "A deterministic runtime regression is the smallest correct fix.",
|
|
67
|
+
nextAuthority: "runtime_service",
|
|
68
|
+
});
|
|
69
|
+
const product = createHarnessEvaluationReviewReceipt({
|
|
70
|
+
...withoutHash(noAction),
|
|
71
|
+
id: "review-product",
|
|
72
|
+
selectedEvidence: [evidence("product-routing", "route_decision")],
|
|
73
|
+
claim: claim("product-routing-defect", 1),
|
|
74
|
+
classification: "product",
|
|
75
|
+
triage: [
|
|
76
|
+
{
|
|
77
|
+
layer: "product",
|
|
78
|
+
status: "unresolved",
|
|
79
|
+
reason: "The product selected an unrelated Skill for an ordinary Work turn.",
|
|
80
|
+
evidenceRefs: [ref("product-routing")],
|
|
81
|
+
},
|
|
82
|
+
],
|
|
83
|
+
reason: "Product routing must be corrected before behavioral Evaluation.",
|
|
84
|
+
nextAuthority: "product_team",
|
|
85
|
+
});
|
|
86
|
+
const taskset = createHarnessEvaluationReviewReceipt({
|
|
87
|
+
...withoutHash(noAction),
|
|
88
|
+
id: "review-taskset",
|
|
89
|
+
selectedEvidence: [
|
|
90
|
+
evidence("failure-1", "work_outcome"),
|
|
91
|
+
evidence("failure-2", "work_outcome"),
|
|
92
|
+
evidence("failure-3", "work_outcome"),
|
|
93
|
+
],
|
|
94
|
+
claim: claim("search-budget-allocation"),
|
|
95
|
+
classification: "taskset",
|
|
96
|
+
triage: [
|
|
97
|
+
{
|
|
98
|
+
layer: "harness",
|
|
99
|
+
status: "unresolved",
|
|
100
|
+
reason: "The active research Skill did not resolve three independent failures.",
|
|
101
|
+
evidenceRefs: [ref("failure-1"), ref("failure-2"), ref("failure-3")],
|
|
102
|
+
},
|
|
103
|
+
],
|
|
104
|
+
reason: "The repeated behavioral claim now needs controlled measurement.",
|
|
105
|
+
nextAuthority: "human_review",
|
|
106
|
+
tasksetProposal: ref("taskset-proposal"),
|
|
107
|
+
maxEstimatedCostUsd: 2,
|
|
108
|
+
});
|
|
109
|
+
const blockedRl = createModelImprovementQualificationReceipt({
|
|
110
|
+
schemaVersion: "openpond.modelImprovementQualificationReceipt.v1",
|
|
111
|
+
id: "qualification-rl-blocked",
|
|
112
|
+
review: reference(taskset),
|
|
113
|
+
harnessRelease,
|
|
114
|
+
tasksetRelease: ref("taskset-release"),
|
|
115
|
+
baselineEvaluation: ref("baseline-evaluation"),
|
|
116
|
+
model: modelRef(),
|
|
117
|
+
environmentHash: contentHash("environment"),
|
|
118
|
+
toolContractHash: contentHash("tools"),
|
|
119
|
+
permissionContractHash: contentHash("permissions"),
|
|
120
|
+
policyHash: contentHash("policy"),
|
|
121
|
+
verifierRef: ref("verifier"),
|
|
122
|
+
sourcePolicies: [sourcePolicy],
|
|
123
|
+
trainingEvidenceRefs: [ref("training-evidence")],
|
|
124
|
+
frozenEvaluationEvidenceRefs: [ref("frozen-evidence")],
|
|
125
|
+
privacyApproval: ref("privacy-approval"),
|
|
126
|
+
budgetApproval: ref("budget-approval"),
|
|
127
|
+
maximumCostUsd: 25,
|
|
128
|
+
signal: {
|
|
129
|
+
kind: "scalar_reward",
|
|
130
|
+
strength: "weak",
|
|
131
|
+
calibrated: true,
|
|
132
|
+
confounded: false,
|
|
133
|
+
variance: 0,
|
|
134
|
+
evidenceRefs: [ref("reward-audit")],
|
|
135
|
+
},
|
|
136
|
+
decision: "no_training",
|
|
137
|
+
reasons: ["The observed reward is constant and cannot support RL."],
|
|
138
|
+
createdAt,
|
|
139
|
+
metadata: { blockedMethod: "rl" },
|
|
140
|
+
});
|
|
141
|
+
const qualifiedRl = createModelImprovementQualificationReceipt({
|
|
142
|
+
...withoutHash(blockedRl),
|
|
143
|
+
id: "qualification-rl-qualified",
|
|
144
|
+
signal: {
|
|
145
|
+
kind: "scalar_reward",
|
|
146
|
+
strength: "usable",
|
|
147
|
+
calibrated: true,
|
|
148
|
+
confounded: false,
|
|
149
|
+
variance: 0.18,
|
|
150
|
+
evidenceRefs: [ref("reward-audit")],
|
|
151
|
+
},
|
|
152
|
+
decision: "rl",
|
|
153
|
+
reasons: [
|
|
154
|
+
"The frozen baseline has usable variance and a calibrated sequential reward.",
|
|
155
|
+
],
|
|
156
|
+
});
|
|
157
|
+
const modelImprovement = createHarnessEvaluationReviewReceipt({
|
|
158
|
+
...withoutHash(noAction),
|
|
159
|
+
id: "review-model-improvement",
|
|
160
|
+
selectedEvidence: [
|
|
161
|
+
evidence("baseline-evaluation", "evaluation"),
|
|
162
|
+
evidence("qualification-rl-qualified", "training_qualification"),
|
|
163
|
+
],
|
|
164
|
+
claim: claim("search-budget-allocation"),
|
|
165
|
+
classification: "model_improvement",
|
|
166
|
+
triage: [
|
|
167
|
+
{
|
|
168
|
+
layer: "model",
|
|
169
|
+
status: "unresolved",
|
|
170
|
+
reason: "Harness, runtime, product, retrieval, and tool triage left a qualified sequential policy gap.",
|
|
171
|
+
evidenceRefs: [ref("baseline-evaluation"), reference(qualifiedRl)],
|
|
172
|
+
},
|
|
173
|
+
],
|
|
174
|
+
reason: "The qualified claim may proceed to a separately approved managed-training plan.",
|
|
175
|
+
nextAuthority: "training_system",
|
|
176
|
+
tasksetProposal: ref("taskset-proposal"),
|
|
177
|
+
evaluation: ref("baseline-evaluation"),
|
|
178
|
+
trainingQualification: reference(qualifiedRl),
|
|
179
|
+
maxEstimatedCostUsd: 25,
|
|
180
|
+
});
|
|
181
|
+
export const harnessEvaluationReviewConformance = {
|
|
182
|
+
noAction,
|
|
183
|
+
runtime,
|
|
184
|
+
product,
|
|
185
|
+
taskset,
|
|
186
|
+
blockedRl,
|
|
187
|
+
qualifiedRl,
|
|
188
|
+
modelImprovement,
|
|
189
|
+
};
|
|
190
|
+
function ref(id) {
|
|
191
|
+
return { id, contentHash: contentHash(id) };
|
|
192
|
+
}
|
|
193
|
+
function reference(value) {
|
|
194
|
+
return { id: value.id, contentHash: value.contentHash };
|
|
195
|
+
}
|
|
196
|
+
function withoutHash(value) {
|
|
197
|
+
const { contentHash: _contentHash, ...content } = value;
|
|
198
|
+
return content;
|
|
199
|
+
}
|
|
200
|
+
function modelRef() {
|
|
201
|
+
return {
|
|
202
|
+
provider: "openpond",
|
|
203
|
+
model: "openpond-chat",
|
|
204
|
+
revision: "model-revision-1",
|
|
205
|
+
artifactHash: contentHash("model-artifact"),
|
|
206
|
+
tokenizerRevision: "tokenizer-1",
|
|
207
|
+
chatTemplateHash: contentHash("chat-template"),
|
|
208
|
+
};
|
|
209
|
+
}
|
package/dist/runs.js
CHANGED
|
@@ -1,13 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "
|
|
3
|
-
export const ModelRefSchema = z.object({
|
|
4
|
-
provider: ReleaseIdSchema,
|
|
5
|
-
model: ReleaseIdSchema,
|
|
6
|
-
revision: z.string().trim().min(1).max(500).nullable().default(null),
|
|
7
|
-
artifactHash: ReleaseHashSchema.nullable().default(null),
|
|
8
|
-
tokenizerRevision: z.string().trim().min(1).max(500).nullable().default(null),
|
|
9
|
-
chatTemplateHash: ReleaseHashSchema.nullable().default(null),
|
|
10
|
-
}).strict();
|
|
2
|
+
import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
|
|
11
3
|
export const RuntimeTargetBindingSchema = z.object({
|
|
12
4
|
adapterId: ReleaseIdSchema,
|
|
13
5
|
placement: z.enum(["local", "remote", "colocated", "provider_native"]),
|
|
@@ -69,6 +61,21 @@ export const EvaluationResultContentSchema = z.object({
|
|
|
69
61
|
metadata: MetadataSchema,
|
|
70
62
|
}).strict();
|
|
71
63
|
export const EvaluationResultSchema = EvaluationResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
|
|
64
|
+
export const HarnessCompatibilityReceiptContentSchema = z.object({
|
|
65
|
+
schemaVersion: z.literal("openpond.harnessCompatibility.v1"),
|
|
66
|
+
id: ReleaseIdSchema,
|
|
67
|
+
baseHarnessRelease: ImmutableReleaseRefSchema,
|
|
68
|
+
candidateHarnessRelease: ImmutableReleaseRefSchema,
|
|
69
|
+
tasksetRelease: ImmutableReleaseRefSchema,
|
|
70
|
+
environmentHash: ReleaseHashSchema,
|
|
71
|
+
toolContractHash: ReleaseHashSchema,
|
|
72
|
+
policyHash: ReleaseHashSchema,
|
|
73
|
+
graderInterfaceHash: ReleaseHashSchema,
|
|
74
|
+
metadata: MetadataSchema,
|
|
75
|
+
}).strict();
|
|
76
|
+
export const HarnessCompatibilityReceiptSchema = HarnessCompatibilityReceiptContentSchema
|
|
77
|
+
.extend({ contentHash: ReleaseHashSchema })
|
|
78
|
+
.strict();
|
|
72
79
|
export function createRunManifest(input) {
|
|
73
80
|
const content = RunManifestContentSchema.parse(input);
|
|
74
81
|
return RunManifestSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
@@ -115,13 +122,26 @@ export function aggregateEvaluationReceipts(input) {
|
|
|
115
122
|
});
|
|
116
123
|
return EvaluationResultSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
117
124
|
}
|
|
118
|
-
export function
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
125
|
+
export function createHarnessCompatibilityReceipt(input) {
|
|
126
|
+
const content = HarnessCompatibilityReceiptContentSchema.parse(input);
|
|
127
|
+
return HarnessCompatibilityReceiptSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
128
|
+
}
|
|
129
|
+
export function assertComparableRunManifests(base, candidate, compatibility) {
|
|
122
130
|
if (base.tasksetRelease.contentHash !== candidate.tasksetRelease.contentHash) {
|
|
123
131
|
throw new Error("Evaluation runs use different Taskset Releases.");
|
|
124
132
|
}
|
|
133
|
+
if (base.harnessRelease.contentHash === candidate.harnessRelease.contentHash)
|
|
134
|
+
return;
|
|
135
|
+
if (!compatibility) {
|
|
136
|
+
throw new Error("Evaluation runs use different Harness Releases without a compatibility receipt.");
|
|
137
|
+
}
|
|
138
|
+
const receipt = HarnessCompatibilityReceiptSchema.parse(compatibility);
|
|
139
|
+
assertContentHash(receipt, "Harness compatibility receipt");
|
|
140
|
+
if (receipt.baseHarnessRelease.contentHash !== base.harnessRelease.contentHash
|
|
141
|
+
|| receipt.candidateHarnessRelease.contentHash !== candidate.harnessRelease.contentHash
|
|
142
|
+
|| receipt.tasksetRelease.contentHash !== base.tasksetRelease.contentHash) {
|
|
143
|
+
throw new Error("Harness compatibility receipt does not match the compared runs.");
|
|
144
|
+
}
|
|
125
145
|
}
|
|
126
146
|
export function rewardEligibleReceipts(receipts) {
|
|
127
147
|
return receipts.filter((receipt) => receipt.terminal
|
package/dist/tasksets.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import {
|
|
2
|
+
import { CapabilityRequirementSchema, ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ToolDeclarationSchema, assertContentHash, contentHash, } from "@openpond/harness";
|
|
3
3
|
export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
|
|
4
4
|
export const PolicyBoundarySchema = z.object({
|
|
5
5
|
policyVisibleFields: z.array(ReleaseIdSchema).max(1_000).default([]),
|
|
@@ -7,20 +7,6 @@ export const PolicyBoundarySchema = z.object({
|
|
|
7
7
|
hiddenGraderRefs: z.array(ReleaseIdSchema).max(1_000).default([]),
|
|
8
8
|
connectedAppScopes: z.array(ReleaseIdSchema).max(100).default([]),
|
|
9
9
|
}).strict();
|
|
10
|
-
export const ToolDeclarationSchema = z.object({
|
|
11
|
-
name: z.string().trim().min(1).max(64).regex(/^[a-zA-Z][a-zA-Z0-9_-]*$/),
|
|
12
|
-
description: z.string().trim().min(1).max(2_000),
|
|
13
|
-
inputSchema: z.record(z.string(), z.unknown()),
|
|
14
|
-
inputSchemaHash: ReleaseHashSchema,
|
|
15
|
-
sideEffect: z.enum(["read", "write"]),
|
|
16
|
-
timeoutMs: z.number().int().positive().max(3_600_000),
|
|
17
|
-
}).strict();
|
|
18
|
-
export const CapabilityRequirementSchema = z.object({
|
|
19
|
-
id: ReleaseIdSchema,
|
|
20
|
-
required: z.boolean(),
|
|
21
|
-
scopes: z.array(z.string().trim().min(1).max(500)).max(100).default([]),
|
|
22
|
-
portability: z.enum(["portable", "host_adapter", "local_only", "hosted_only"]),
|
|
23
|
-
}).strict();
|
|
24
10
|
export const EnvironmentContractSchema = z.object({
|
|
25
11
|
protocolVersion: z.literal("openpond.environment.v1"),
|
|
26
12
|
kind: z.enum(["text", "agent", "work", "custom_program"]),
|
|
@@ -77,10 +63,9 @@ export const TaskRecordSchema = z.object({
|
|
|
77
63
|
tags: z.array(ReleaseIdSchema).max(100).default([]),
|
|
78
64
|
}).strict();
|
|
79
65
|
export const TasksetReleaseContentSchema = z.object({
|
|
80
|
-
schemaVersion: z.literal("openpond.tasksetRelease.
|
|
66
|
+
schemaVersion: z.literal("openpond.tasksetRelease.v2"),
|
|
81
67
|
id: ReleaseIdSchema,
|
|
82
68
|
revision: z.number().int().positive(),
|
|
83
|
-
harnessRelease: ImmutableReleaseRefSchema,
|
|
84
69
|
policy: PolicyBoundarySchema,
|
|
85
70
|
environment: EnvironmentContractSchema,
|
|
86
71
|
tools: z.array(ToolDeclarationSchema).max(200),
|
|
@@ -161,3 +146,4 @@ export function trainingPolicyTaskViews(taskset) {
|
|
|
161
146
|
.filter((task) => task.split !== "frozen_eval")
|
|
162
147
|
.map(policyTaskView);
|
|
163
148
|
}
|
|
149
|
+
export { CapabilityRequirementSchema, ToolDeclarationSchema, } from "@openpond/harness";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type HarnessRelease } from "@openpond/harness";
|
|
2
|
+
import { type HarnessCompatibilityReceipt } from "./runs.js";
|
|
3
|
+
import { type TasksetRelease } from "./tasksets.js";
|
|
4
|
+
export declare function createVerifiedHarnessCompatibilityReceipt(input: {
|
|
5
|
+
id: string;
|
|
6
|
+
baseHarnessRelease: HarnessRelease;
|
|
7
|
+
candidateHarnessRelease: HarnessRelease;
|
|
8
|
+
tasksetRelease: TasksetRelease;
|
|
9
|
+
metadata?: Record<string, unknown>;
|
|
10
|
+
}): HarnessCompatibilityReceipt;
|
|
11
|
+
//# sourceMappingURL=compatibility.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,yCAAyC,CAAC,KAAK,EAAE;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,cAAc,CAAC;IACnC,uBAAuB,EAAE,cAAc,CAAC;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,2BAA2B,CA4C9B"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export declare const genericToolConformance: {
|
|
2
2
|
snapshot: {
|
|
3
|
-
schemaVersion: "openpond.agentSnapshot.
|
|
3
|
+
schemaVersion: "openpond.agentSnapshot.v2";
|
|
4
4
|
id: string;
|
|
5
|
-
|
|
5
|
+
sourceRelease: {
|
|
6
6
|
id: string;
|
|
7
7
|
contentHash: string;
|
|
8
8
|
} | null;
|
|
@@ -42,6 +42,7 @@ export declare const genericToolConformance: {
|
|
|
42
42
|
id: string;
|
|
43
43
|
required: boolean;
|
|
44
44
|
scopes: string[];
|
|
45
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
45
46
|
}[];
|
|
46
47
|
dependencyLock: {
|
|
47
48
|
id: string;
|
|
@@ -61,7 +62,7 @@ export declare const genericToolConformance: {
|
|
|
61
62
|
contentHash: string;
|
|
62
63
|
};
|
|
63
64
|
harness: {
|
|
64
|
-
schemaVersion: "openpond.harnessRelease.
|
|
65
|
+
schemaVersion: "openpond.harnessRelease.v2";
|
|
65
66
|
id: string;
|
|
66
67
|
agentSnapshot: {
|
|
67
68
|
id: string;
|
|
@@ -75,16 +76,6 @@ export declare const genericToolConformance: {
|
|
|
75
76
|
mediaType: string;
|
|
76
77
|
visibility: "policy" | "verifier" | "host_private";
|
|
77
78
|
};
|
|
78
|
-
environment: {
|
|
79
|
-
protocolVersion: "openpond.environment.v1";
|
|
80
|
-
kind: "text" | "agent" | "work" | "custom_program";
|
|
81
|
-
entrypoint: string;
|
|
82
|
-
stateful: boolean;
|
|
83
|
-
deterministicSeeds: boolean;
|
|
84
|
-
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
85
|
-
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
86
|
-
defaultTimeoutMs: number;
|
|
87
|
-
};
|
|
88
79
|
tools: {
|
|
89
80
|
name: string;
|
|
90
81
|
description: string;
|
|
@@ -106,12 +97,6 @@ export declare const genericToolConformance: {
|
|
|
106
97
|
privilegedEvidence: string[];
|
|
107
98
|
privateVerifierIsolation: boolean;
|
|
108
99
|
};
|
|
109
|
-
policy: {
|
|
110
|
-
policyVisibleFields: string[];
|
|
111
|
-
privilegedFields: string[];
|
|
112
|
-
hiddenGraderRefs: string[];
|
|
113
|
-
connectedAppScopes: string[];
|
|
114
|
-
};
|
|
115
100
|
files: {
|
|
116
101
|
id: string;
|
|
117
102
|
path: string;
|
|
@@ -124,13 +109,9 @@ export declare const genericToolConformance: {
|
|
|
124
109
|
contentHash: string;
|
|
125
110
|
};
|
|
126
111
|
taskset: {
|
|
127
|
-
schemaVersion: "openpond.tasksetRelease.
|
|
112
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
128
113
|
id: string;
|
|
129
114
|
revision: number;
|
|
130
|
-
harnessRelease: {
|
|
131
|
-
id: string;
|
|
132
|
-
contentHash: string;
|
|
133
|
-
};
|
|
134
115
|
policy: {
|
|
135
116
|
policyVisibleFields: string[];
|
|
136
117
|
privilegedFields: string[];
|
|
@@ -286,9 +267,9 @@ export declare const genericToolConformance: {
|
|
|
286
267
|
};
|
|
287
268
|
export declare const marketingPortfolioConformance: {
|
|
288
269
|
snapshot: {
|
|
289
|
-
schemaVersion: "openpond.agentSnapshot.
|
|
270
|
+
schemaVersion: "openpond.agentSnapshot.v2";
|
|
290
271
|
id: string;
|
|
291
|
-
|
|
272
|
+
sourceRelease: {
|
|
292
273
|
id: string;
|
|
293
274
|
contentHash: string;
|
|
294
275
|
} | null;
|
|
@@ -328,6 +309,7 @@ export declare const marketingPortfolioConformance: {
|
|
|
328
309
|
id: string;
|
|
329
310
|
required: boolean;
|
|
330
311
|
scopes: string[];
|
|
312
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
331
313
|
}[];
|
|
332
314
|
dependencyLock: {
|
|
333
315
|
id: string;
|
|
@@ -347,7 +329,7 @@ export declare const marketingPortfolioConformance: {
|
|
|
347
329
|
contentHash: string;
|
|
348
330
|
};
|
|
349
331
|
harness: {
|
|
350
|
-
schemaVersion: "openpond.harnessRelease.
|
|
332
|
+
schemaVersion: "openpond.harnessRelease.v2";
|
|
351
333
|
id: string;
|
|
352
334
|
agentSnapshot: {
|
|
353
335
|
id: string;
|
|
@@ -361,16 +343,6 @@ export declare const marketingPortfolioConformance: {
|
|
|
361
343
|
mediaType: string;
|
|
362
344
|
visibility: "policy" | "verifier" | "host_private";
|
|
363
345
|
};
|
|
364
|
-
environment: {
|
|
365
|
-
protocolVersion: "openpond.environment.v1";
|
|
366
|
-
kind: "text" | "agent" | "work" | "custom_program";
|
|
367
|
-
entrypoint: string;
|
|
368
|
-
stateful: boolean;
|
|
369
|
-
deterministicSeeds: boolean;
|
|
370
|
-
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
371
|
-
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
372
|
-
defaultTimeoutMs: number;
|
|
373
|
-
};
|
|
374
346
|
tools: {
|
|
375
347
|
name: string;
|
|
376
348
|
description: string;
|
|
@@ -392,12 +364,6 @@ export declare const marketingPortfolioConformance: {
|
|
|
392
364
|
privilegedEvidence: string[];
|
|
393
365
|
privateVerifierIsolation: boolean;
|
|
394
366
|
};
|
|
395
|
-
policy: {
|
|
396
|
-
policyVisibleFields: string[];
|
|
397
|
-
privilegedFields: string[];
|
|
398
|
-
hiddenGraderRefs: string[];
|
|
399
|
-
connectedAppScopes: string[];
|
|
400
|
-
};
|
|
401
367
|
files: {
|
|
402
368
|
id: string;
|
|
403
369
|
path: string;
|
|
@@ -410,13 +376,9 @@ export declare const marketingPortfolioConformance: {
|
|
|
410
376
|
contentHash: string;
|
|
411
377
|
};
|
|
412
378
|
taskset: {
|
|
413
|
-
schemaVersion: "openpond.tasksetRelease.
|
|
379
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
414
380
|
id: string;
|
|
415
381
|
revision: number;
|
|
416
|
-
harnessRelease: {
|
|
417
|
-
id: string;
|
|
418
|
-
contentHash: string;
|
|
419
|
-
};
|
|
420
382
|
policy: {
|
|
421
383
|
policyVisibleFields: string[];
|
|
422
384
|
privilegedFields: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
|
|
@@ -9,7 +9,7 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
9
9
|
sequence: number;
|
|
10
10
|
timestamp: string;
|
|
11
11
|
layer: "agent" | "environment";
|
|
12
|
-
kind: "
|
|
12
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
13
13
|
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
14
14
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
15
15
|
inputHash: string | null;
|
|
@@ -32,7 +32,7 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
32
32
|
durationMs: number | null;
|
|
33
33
|
cpuTimeMs: number | null;
|
|
34
34
|
memoryPeakBytes: number | null;
|
|
35
|
-
errorClass: "unknown" | "
|
|
35
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
36
36
|
};
|
|
37
37
|
}[];
|
|
38
38
|
contentHash: string;
|
|
@@ -186,7 +186,7 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
186
186
|
sequence: number;
|
|
187
187
|
timestamp: string;
|
|
188
188
|
layer: "agent" | "environment";
|
|
189
|
-
kind: "
|
|
189
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
190
190
|
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
191
191
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
192
192
|
inputHash: string | null;
|
|
@@ -209,7 +209,7 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
209
209
|
durationMs: number | null;
|
|
210
210
|
cpuTimeMs: number | null;
|
|
211
211
|
memoryPeakBytes: number | null;
|
|
212
|
-
errorClass: "unknown" | "
|
|
212
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
213
213
|
};
|
|
214
214
|
}[];
|
|
215
215
|
contentHash: string;
|
|
@@ -392,7 +392,7 @@ export declare const workEvidenceConformance: {
|
|
|
392
392
|
sequence: number;
|
|
393
393
|
timestamp: string;
|
|
394
394
|
layer: "agent" | "environment";
|
|
395
|
-
kind: "
|
|
395
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
396
396
|
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
397
397
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
398
398
|
inputHash: string | null;
|
|
@@ -415,7 +415,7 @@ export declare const workEvidenceConformance: {
|
|
|
415
415
|
durationMs: number | null;
|
|
416
416
|
cpuTimeMs: number | null;
|
|
417
417
|
memoryPeakBytes: number | null;
|
|
418
|
-
errorClass: "unknown" | "
|
|
418
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
419
419
|
};
|
|
420
420
|
}[];
|
|
421
421
|
contentHash: string;
|
|
@@ -625,7 +625,7 @@ export declare const workEvidenceConformance: {
|
|
|
625
625
|
sequence: number;
|
|
626
626
|
timestamp: string;
|
|
627
627
|
layer: "agent" | "environment";
|
|
628
|
-
kind: "
|
|
628
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
629
629
|
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
630
630
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
631
631
|
inputHash: string | null;
|
|
@@ -648,7 +648,7 @@ export declare const workEvidenceConformance: {
|
|
|
648
648
|
durationMs: number | null;
|
|
649
649
|
cpuTimeMs: number | null;
|
|
650
650
|
memoryPeakBytes: number | null;
|
|
651
|
-
errorClass: "unknown" | "
|
|
651
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
652
652
|
};
|
|
653
653
|
}[];
|
|
654
654
|
contentHash: string;
|
|
@@ -13,9 +13,9 @@ export declare const EvidenceArtifactRefSchema: z.ZodObject<{
|
|
|
13
13
|
id: z.ZodString;
|
|
14
14
|
}, z.core.$strict>;
|
|
15
15
|
export declare const WorkProcessStepKindSchema: z.ZodEnum<{
|
|
16
|
+
approval: "approval";
|
|
16
17
|
validation: "validation";
|
|
17
18
|
artifact: "artifact";
|
|
18
|
-
approval: "approval";
|
|
19
19
|
tool: "tool";
|
|
20
20
|
state_transition: "state_transition";
|
|
21
21
|
question: "question";
|
|
@@ -88,10 +88,10 @@ export declare const WorkInterventionOutcomeSchema: z.ZodEnum<{
|
|
|
88
88
|
}>;
|
|
89
89
|
export declare const WorkProcessErrorClassSchema: z.ZodEnum<{
|
|
90
90
|
unknown: "unknown";
|
|
91
|
-
policy: "policy";
|
|
92
91
|
timeout: "timeout";
|
|
93
92
|
cancelled: "cancelled";
|
|
94
93
|
validation: "validation";
|
|
94
|
+
policy: "policy";
|
|
95
95
|
environment: "environment";
|
|
96
96
|
infrastructure: "infrastructure";
|
|
97
97
|
}>;
|
|
@@ -111,9 +111,9 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
|
|
|
111
111
|
environment: "environment";
|
|
112
112
|
}>;
|
|
113
113
|
kind: z.ZodEnum<{
|
|
114
|
+
approval: "approval";
|
|
114
115
|
validation: "validation";
|
|
115
116
|
artifact: "artifact";
|
|
116
|
-
approval: "approval";
|
|
117
117
|
tool: "tool";
|
|
118
118
|
state_transition: "state_transition";
|
|
119
119
|
question: "question";
|
|
@@ -198,10 +198,10 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
|
|
|
198
198
|
memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
|
|
199
199
|
errorClass: z.ZodNullable<z.ZodEnum<{
|
|
200
200
|
unknown: "unknown";
|
|
201
|
-
policy: "policy";
|
|
202
201
|
timeout: "timeout";
|
|
203
202
|
cancelled: "cancelled";
|
|
204
203
|
validation: "validation";
|
|
204
|
+
policy: "policy";
|
|
205
205
|
environment: "environment";
|
|
206
206
|
infrastructure: "infrastructure";
|
|
207
207
|
}>>;
|
|
@@ -229,9 +229,9 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
|
|
|
229
229
|
environment: "environment";
|
|
230
230
|
}>;
|
|
231
231
|
kind: z.ZodEnum<{
|
|
232
|
+
approval: "approval";
|
|
232
233
|
validation: "validation";
|
|
233
234
|
artifact: "artifact";
|
|
234
|
-
approval: "approval";
|
|
235
235
|
tool: "tool";
|
|
236
236
|
state_transition: "state_transition";
|
|
237
237
|
question: "question";
|
|
@@ -316,10 +316,10 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
|
|
|
316
316
|
memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
|
|
317
317
|
errorClass: z.ZodNullable<z.ZodEnum<{
|
|
318
318
|
unknown: "unknown";
|
|
319
|
-
policy: "policy";
|
|
320
319
|
timeout: "timeout";
|
|
321
320
|
cancelled: "cancelled";
|
|
322
321
|
validation: "validation";
|
|
322
|
+
policy: "policy";
|
|
323
323
|
environment: "environment";
|
|
324
324
|
infrastructure: "infrastructure";
|
|
325
325
|
}>>;
|
|
@@ -348,9 +348,9 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
|
|
|
348
348
|
environment: "environment";
|
|
349
349
|
}>;
|
|
350
350
|
kind: z.ZodEnum<{
|
|
351
|
+
approval: "approval";
|
|
351
352
|
validation: "validation";
|
|
352
353
|
artifact: "artifact";
|
|
353
|
-
approval: "approval";
|
|
354
354
|
tool: "tool";
|
|
355
355
|
state_transition: "state_transition";
|
|
356
356
|
question: "question";
|
|
@@ -435,10 +435,10 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
|
|
|
435
435
|
memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
|
|
436
436
|
errorClass: z.ZodNullable<z.ZodEnum<{
|
|
437
437
|
unknown: "unknown";
|
|
438
|
-
policy: "policy";
|
|
439
438
|
timeout: "timeout";
|
|
440
439
|
cancelled: "cancelled";
|
|
441
440
|
validation: "validation";
|
|
441
|
+
policy: "policy";
|
|
442
442
|
environment: "environment";
|
|
443
443
|
infrastructure: "infrastructure";
|
|
444
444
|
}>>;
|