@openpond/evals 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +11 -3
- package/README.md +45 -9
- package/RELEASING.md +13 -46
- package/dist/benchmarks.js +218 -0
- package/dist/builtin-benchmarks/harness-refiner.js +1267 -0
- package/dist/compatibility.js +2 -2
- package/dist/conformance.js +1 -2
- package/dist/evidence/authoring.js +1 -1
- package/dist/evidence/conformance.js +1 -1
- package/dist/evidence/contracts.js +1 -2
- package/dist/evidence/eligibility.js +1 -1
- package/dist/graders.js +1 -1
- package/dist/harness.js +33 -101
- package/dist/index.js +3 -3
- package/dist/model-improvement-qualification.js +125 -0
- package/dist/review-conformance.js +209 -0
- package/dist/runs.js +1 -9
- package/dist/tasksets.js +2 -15
- package/dist/types/benchmarks.d.ts +293 -0
- package/dist/types/benchmarks.d.ts.map +1 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts +119 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -0
- package/dist/types/compatibility.d.ts +1 -1
- package/dist/types/compatibility.d.ts.map +1 -1
- package/dist/types/conformance.d.ts +2 -0
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +24 -24
- package/dist/types/evidence/contracts.d.ts +26 -26
- package/dist/types/harness.d.ts +3 -409
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/model-improvement-qualification.d.ts +222 -0
- package/dist/types/model-improvement-qualification.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +607 -0
- package/dist/types/review-conformance.d.ts.map +1 -0
- package/dist/types/runs.d.ts +1 -9
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -25
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +18 -10
- package/dist/common.js +0 -67
- package/dist/harness-improvements.js +0 -329
- package/dist/harness-workspaces.js +0 -368
- package/dist/sha256.js +0 -91
- package/dist/types/common.d.ts +0 -51
- package/dist/types/common.d.ts.map +0 -1
- package/dist/types/harness-improvements.d.ts +0 -523
- package/dist/types/harness-improvements.d.ts.map +0 -1
- package/dist/types/harness-workspaces.d.ts +0 -802
- package/dist/types/harness-workspaces.d.ts.map +0 -1
- package/dist/types/sha256.d.ts +0 -2
- package/dist/types/sha256.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,
|
|
1
|
+
{"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,uBAAuB,EAOxB,MAAM,mBAAmB,CAAC;AAE3B,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;kBAI5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;kBAKhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CA8CA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,GACrB,MAAM,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openpond/evals",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Portable
|
|
3
|
+
"version": "0.4.1",
|
|
4
|
+
"description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./dist/index.js",
|
|
@@ -22,18 +22,14 @@
|
|
|
22
22
|
"types": "./dist/types/harness.d.ts",
|
|
23
23
|
"import": "./dist/harness.js"
|
|
24
24
|
},
|
|
25
|
-
"./harness-improvements": {
|
|
26
|
-
"types": "./dist/types/harness-improvements.d.ts",
|
|
27
|
-
"import": "./dist/harness-improvements.js"
|
|
28
|
-
},
|
|
29
|
-
"./harness-workspaces": {
|
|
30
|
-
"types": "./dist/types/harness-workspaces.d.ts",
|
|
31
|
-
"import": "./dist/harness-workspaces.js"
|
|
32
|
-
},
|
|
33
25
|
"./tasksets": {
|
|
34
26
|
"types": "./dist/types/tasksets.d.ts",
|
|
35
27
|
"import": "./dist/tasksets.js"
|
|
36
28
|
},
|
|
29
|
+
"./benchmarks": {
|
|
30
|
+
"types": "./dist/types/benchmarks.d.ts",
|
|
31
|
+
"import": "./dist/benchmarks.js"
|
|
32
|
+
},
|
|
37
33
|
"./graders": {
|
|
38
34
|
"types": "./dist/types/graders.d.ts",
|
|
39
35
|
"import": "./dist/graders.js"
|
|
@@ -42,6 +38,14 @@
|
|
|
42
38
|
"types": "./dist/types/runs.d.ts",
|
|
43
39
|
"import": "./dist/runs.js"
|
|
44
40
|
},
|
|
41
|
+
"./review": {
|
|
42
|
+
"types": "./dist/types/review-conformance.d.ts",
|
|
43
|
+
"import": "./dist/review-conformance.js"
|
|
44
|
+
},
|
|
45
|
+
"./model-improvement-qualification": {
|
|
46
|
+
"types": "./dist/types/model-improvement-qualification.d.ts",
|
|
47
|
+
"import": "./dist/model-improvement-qualification.js"
|
|
48
|
+
},
|
|
45
49
|
"./conformance": {
|
|
46
50
|
"types": "./dist/types/conformance.d.ts",
|
|
47
51
|
"import": "./dist/conformance.js"
|
|
@@ -79,7 +83,11 @@
|
|
|
79
83
|
"dependencies": {
|
|
80
84
|
"zod": "^4.1.13"
|
|
81
85
|
},
|
|
86
|
+
"peerDependencies": {
|
|
87
|
+
"@openpond/harness": "^0.2.0"
|
|
88
|
+
},
|
|
82
89
|
"devDependencies": {
|
|
90
|
+
"@openpond/harness": "workspace:*",
|
|
83
91
|
"@types/node": "24.13.3",
|
|
84
92
|
"tsx": "4.23.1",
|
|
85
93
|
"typescript": "^5.9.3",
|
package/dist/common.js
DELETED
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { sha256Hex } from "./sha256.js";
|
|
3
|
-
export const MAX_PORTABLE_PATH_BYTES = 2_000;
|
|
4
|
-
export const MAX_PORTABLE_ASSET_BYTES = 250_000_000;
|
|
5
|
-
export const ReleaseIdSchema = z.string().trim().min(1).max(240);
|
|
6
|
-
export const ReleaseHashSchema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
7
|
-
export const ReleaseTimestampSchema = z.string().datetime({ offset: true });
|
|
8
|
-
export const MetadataSchema = z.record(z.string(), z.unknown()).default({});
|
|
9
|
-
export const ImmutableReleaseRefSchema = z.object({
|
|
10
|
-
id: ReleaseIdSchema,
|
|
11
|
-
contentHash: ReleaseHashSchema,
|
|
12
|
-
}).strict();
|
|
13
|
-
export const ImmutableAssetRefSchema = z.object({
|
|
14
|
-
id: ReleaseIdSchema,
|
|
15
|
-
path: z.string().trim().min(1).max(MAX_PORTABLE_PATH_BYTES).refine(safeRelativePath),
|
|
16
|
-
contentHash: ReleaseHashSchema,
|
|
17
|
-
sizeBytes: z.number().int().nonnegative().max(MAX_PORTABLE_ASSET_BYTES),
|
|
18
|
-
mediaType: z.string().trim().min(1).max(200),
|
|
19
|
-
visibility: z.enum(["policy", "verifier", "host_private"]),
|
|
20
|
-
}).strict();
|
|
21
|
-
export const ImmutableArtifactRefSchema = z.object({
|
|
22
|
-
id: ReleaseIdSchema,
|
|
23
|
-
contentHash: ReleaseHashSchema,
|
|
24
|
-
mediaType: z.string().trim().min(1).max(200).nullable().default(null),
|
|
25
|
-
sizeBytes: z.number().int().nonnegative().max(MAX_PORTABLE_ASSET_BYTES).nullable().default(null),
|
|
26
|
-
}).strict();
|
|
27
|
-
export const FailureClassSchema = z.enum([
|
|
28
|
-
"policy_failure",
|
|
29
|
-
"grader_failure",
|
|
30
|
-
"environment_failure",
|
|
31
|
-
"infrastructure_failure",
|
|
32
|
-
"timeout",
|
|
33
|
-
"cancelled",
|
|
34
|
-
]);
|
|
35
|
-
export function canonicalJson(value) {
|
|
36
|
-
return `${JSON.stringify(sortValue(value), null, 2)}\n`;
|
|
37
|
-
}
|
|
38
|
-
export function sha256(value) {
|
|
39
|
-
return sha256Hex(value);
|
|
40
|
-
}
|
|
41
|
-
export function contentHash(value) {
|
|
42
|
-
return sha256(canonicalJson(value));
|
|
43
|
-
}
|
|
44
|
-
export function withContentHash(value) {
|
|
45
|
-
return { ...value, contentHash: contentHash(value) };
|
|
46
|
-
}
|
|
47
|
-
export function assertContentHash(value, label) {
|
|
48
|
-
const { contentHash: actual, ...hashable } = value;
|
|
49
|
-
const expected = contentHash(hashable);
|
|
50
|
-
if (actual !== expected)
|
|
51
|
-
throw new Error(`${label} contentHash is ${actual}; expected ${expected}.`);
|
|
52
|
-
}
|
|
53
|
-
function safeRelativePath(value) {
|
|
54
|
-
const normalized = value.replaceAll("\\", "/");
|
|
55
|
-
if (!normalized || normalized.startsWith("/") || normalized.includes("\0"))
|
|
56
|
-
return false;
|
|
57
|
-
return !normalized.split("/").some((part) => !part || part === "." || part === "..");
|
|
58
|
-
}
|
|
59
|
-
function sortValue(value) {
|
|
60
|
-
if (Array.isArray(value))
|
|
61
|
-
return value.map(sortValue);
|
|
62
|
-
if (!value || typeof value !== "object")
|
|
63
|
-
return value;
|
|
64
|
-
return Object.fromEntries(Object.entries(value)
|
|
65
|
-
.sort(([left], [right]) => left.localeCompare(right))
|
|
66
|
-
.map(([key, child]) => [key, sortValue(child)]));
|
|
67
|
-
}
|
|
@@ -1,329 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { HarnessImprovementRouteSchema, HarnessOverlaySnapshotRefSchema, } from "./harness-workspaces.js";
|
|
3
|
-
import { contentHash, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "./common.js";
|
|
4
|
-
const BoundedTextSchema = z.string().trim().min(1).max(100_000);
|
|
5
|
-
const MetadataSchema = z.record(z.string(), z.unknown()).default({});
|
|
6
|
-
export const ImprovementSafeBoundaryKindSchema = z.enum([
|
|
7
|
-
"completed_tool_batch",
|
|
8
|
-
"before_model_step",
|
|
9
|
-
"turn_completed",
|
|
10
|
-
"turn_paused",
|
|
11
|
-
]);
|
|
12
|
-
export const ImprovementSafeBoundarySchema = z
|
|
13
|
-
.object({
|
|
14
|
-
kind: ImprovementSafeBoundaryKindSchema,
|
|
15
|
-
eventSequence: z.number().int().nonnegative(),
|
|
16
|
-
occurredAt: ReleaseTimestampSchema,
|
|
17
|
-
})
|
|
18
|
-
.strict();
|
|
19
|
-
export const ImprovementEventRefSchema = z
|
|
20
|
-
.object({
|
|
21
|
-
id: ReleaseIdSchema,
|
|
22
|
-
sequence: z.number().int().nonnegative().nullable(),
|
|
23
|
-
contentHash: ReleaseHashSchema,
|
|
24
|
-
})
|
|
25
|
-
.strict();
|
|
26
|
-
export const ImprovementObservationKindSchema = z.enum([
|
|
27
|
-
"tool_failure",
|
|
28
|
-
"retry",
|
|
29
|
-
"recovery",
|
|
30
|
-
"validation",
|
|
31
|
-
"user_correction",
|
|
32
|
-
"reusable_success",
|
|
33
|
-
"completion_detour",
|
|
34
|
-
]);
|
|
35
|
-
export const ImprovementObservationStateSchema = z.enum([
|
|
36
|
-
"open",
|
|
37
|
-
"recovered",
|
|
38
|
-
"terminal",
|
|
39
|
-
]);
|
|
40
|
-
export const ImprovementToolIdentitySchema = z
|
|
41
|
-
.object({
|
|
42
|
-
name: z.string().trim().min(1).max(500),
|
|
43
|
-
invocationKey: ReleaseHashSchema,
|
|
44
|
-
})
|
|
45
|
-
.strict();
|
|
46
|
-
export const ImprovementObservationContentSchema = z
|
|
47
|
-
.object({
|
|
48
|
-
schemaVersion: z.literal("openpond.improvementObservation.v1"),
|
|
49
|
-
id: ReleaseIdSchema,
|
|
50
|
-
runRef: ReleaseIdSchema,
|
|
51
|
-
turnId: ReleaseIdSchema,
|
|
52
|
-
harnessRelease: ImmutableReleaseRefSchema,
|
|
53
|
-
overlay: HarnessOverlaySnapshotRefSchema.nullable(),
|
|
54
|
-
eventRefs: z.array(ImprovementEventRefSchema).min(1).max(100),
|
|
55
|
-
kind: ImprovementObservationKindSchema,
|
|
56
|
-
state: ImprovementObservationStateSchema,
|
|
57
|
-
tool: ImprovementToolIdentitySchema.nullable(),
|
|
58
|
-
deterministicClass: z.string().trim().min(1).max(500).nullable(),
|
|
59
|
-
summary: BoundedTextSchema,
|
|
60
|
-
createdAt: ReleaseTimestampSchema,
|
|
61
|
-
metadata: MetadataSchema,
|
|
62
|
-
})
|
|
63
|
-
.strict()
|
|
64
|
-
.superRefine((observation, context) => {
|
|
65
|
-
if (new Set(observation.eventRefs.map((reference) => reference.id)).size !==
|
|
66
|
-
observation.eventRefs.length) {
|
|
67
|
-
context.addIssue({
|
|
68
|
-
code: "custom",
|
|
69
|
-
message: "observation event refs must be unique",
|
|
70
|
-
path: ["eventRefs"],
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
if (["tool_failure", "retry", "recovery", "completion_detour"].includes(observation.kind) &&
|
|
74
|
-
observation.tool === null) {
|
|
75
|
-
context.addIssue({
|
|
76
|
-
code: "custom",
|
|
77
|
-
message: `${observation.kind} observations require a tool identity`,
|
|
78
|
-
path: ["tool"],
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
export const ImprovementObservationSchema = ImprovementObservationContentSchema.extend({
|
|
83
|
-
contentHash: ReleaseHashSchema,
|
|
84
|
-
}).strict();
|
|
85
|
-
export const RefinementTriggerPolicySchema = z
|
|
86
|
-
.object({
|
|
87
|
-
schemaVersion: z.literal("openpond.refinementTriggerPolicy.v1"),
|
|
88
|
-
maxEstimatedCostUsd: z.number().finite().nonnegative(),
|
|
89
|
-
cooldownMs: z.number().int().nonnegative(),
|
|
90
|
-
maxPendingPlans: z.number().int().min(1).max(100),
|
|
91
|
-
maxEvidenceEvents: z.number().int().min(1).max(1_000),
|
|
92
|
-
maxProposalEdits: z.number().int().min(1).max(1_000),
|
|
93
|
-
maxProposalBytes: z.number().int().min(1).max(10_000_000),
|
|
94
|
-
})
|
|
95
|
-
.strict();
|
|
96
|
-
export const RefinementTriggerDecisionKindSchema = z.enum([
|
|
97
|
-
"no_action",
|
|
98
|
-
"route_deterministically",
|
|
99
|
-
"queue_refiner",
|
|
100
|
-
]);
|
|
101
|
-
export const RefinementTriggerDecisionContentSchema = z
|
|
102
|
-
.object({
|
|
103
|
-
schemaVersion: z.literal("openpond.refinementTriggerDecision.v1"),
|
|
104
|
-
id: ReleaseIdSchema,
|
|
105
|
-
runRef: ReleaseIdSchema,
|
|
106
|
-
turnId: ReleaseIdSchema,
|
|
107
|
-
harnessRelease: ImmutableReleaseRefSchema,
|
|
108
|
-
overlay: HarnessOverlaySnapshotRefSchema.nullable(),
|
|
109
|
-
observations: z.array(ImmutableReleaseRefSchema).max(100),
|
|
110
|
-
decision: RefinementTriggerDecisionKindSchema,
|
|
111
|
-
deterministicRoute: HarnessImprovementRouteSchema.nullable(),
|
|
112
|
-
suggestedRoutes: z.array(HarnessImprovementRouteSchema).max(8),
|
|
113
|
-
reason: BoundedTextSchema,
|
|
114
|
-
deduplicationKey: ReleaseHashSchema,
|
|
115
|
-
policy: RefinementTriggerPolicySchema,
|
|
116
|
-
estimatedMaxCostUsd: z.number().finite().nonnegative(),
|
|
117
|
-
pendingPlanCount: z.number().int().nonnegative(),
|
|
118
|
-
boundary: ImprovementSafeBoundarySchema,
|
|
119
|
-
cooldownUntil: ReleaseTimestampSchema.nullable(),
|
|
120
|
-
createdAt: ReleaseTimestampSchema,
|
|
121
|
-
metadata: MetadataSchema,
|
|
122
|
-
})
|
|
123
|
-
.strict()
|
|
124
|
-
.superRefine((trigger, context) => {
|
|
125
|
-
if (trigger.estimatedMaxCostUsd > trigger.policy.maxEstimatedCostUsd) {
|
|
126
|
-
context.addIssue({
|
|
127
|
-
code: "custom",
|
|
128
|
-
message: "estimated Refiner cost exceeds the trigger policy budget",
|
|
129
|
-
path: ["estimatedMaxCostUsd"],
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
if (trigger.pendingPlanCount > trigger.policy.maxPendingPlans) {
|
|
133
|
-
context.addIssue({
|
|
134
|
-
code: "custom",
|
|
135
|
-
message: "pending plan count exceeds the trigger policy limit",
|
|
136
|
-
path: ["pendingPlanCount"],
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
if (trigger.observations.length > trigger.policy.maxEvidenceEvents) {
|
|
140
|
-
context.addIssue({
|
|
141
|
-
code: "custom",
|
|
142
|
-
message: "trigger observations exceed the evidence budget",
|
|
143
|
-
path: ["observations"],
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
if (trigger.decision === "route_deterministically" &&
|
|
147
|
-
trigger.deterministicRoute === null) {
|
|
148
|
-
context.addIssue({
|
|
149
|
-
code: "custom",
|
|
150
|
-
message: "deterministic decisions require a route",
|
|
151
|
-
path: ["deterministicRoute"],
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
if (trigger.decision !== "route_deterministically" &&
|
|
155
|
-
trigger.deterministicRoute !== null) {
|
|
156
|
-
context.addIssue({
|
|
157
|
-
code: "custom",
|
|
158
|
-
message: "only deterministic decisions may declare a deterministic route",
|
|
159
|
-
path: ["deterministicRoute"],
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
if (trigger.decision === "no_action" && trigger.suggestedRoutes.length > 0) {
|
|
163
|
-
context.addIssue({
|
|
164
|
-
code: "custom",
|
|
165
|
-
message: "no-action decisions cannot suggest routes",
|
|
166
|
-
path: ["suggestedRoutes"],
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
if (trigger.decision !== "no_action" && trigger.observations.length === 0) {
|
|
170
|
-
context.addIssue({
|
|
171
|
-
code: "custom",
|
|
172
|
-
message: "actionable trigger decisions require observations",
|
|
173
|
-
path: ["observations"],
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
});
|
|
177
|
-
export const RefinementTriggerDecisionSchema = RefinementTriggerDecisionContentSchema.extend({
|
|
178
|
-
contentHash: ReleaseHashSchema,
|
|
179
|
-
}).strict();
|
|
180
|
-
export const ImprovementRouteAuthoritySchema = z.enum([
|
|
181
|
-
"runtime_service",
|
|
182
|
-
"refiner_model",
|
|
183
|
-
"human_review",
|
|
184
|
-
"evaluation_system",
|
|
185
|
-
"training_system",
|
|
186
|
-
]);
|
|
187
|
-
export const ImprovementRouteDecisionContentSchema = z
|
|
188
|
-
.object({
|
|
189
|
-
schemaVersion: z.literal("openpond.improvementRouteDecision.v1"),
|
|
190
|
-
id: ReleaseIdSchema,
|
|
191
|
-
trigger: ImmutableReleaseRefSchema,
|
|
192
|
-
route: HarnessImprovementRouteSchema,
|
|
193
|
-
authority: ImprovementRouteAuthoritySchema,
|
|
194
|
-
automatic: z.boolean(),
|
|
195
|
-
reason: BoundedTextSchema,
|
|
196
|
-
createdAt: ReleaseTimestampSchema,
|
|
197
|
-
metadata: MetadataSchema,
|
|
198
|
-
})
|
|
199
|
-
.strict()
|
|
200
|
-
.superRefine((decision, context) => {
|
|
201
|
-
if (decision.automatic &&
|
|
202
|
-
["human_review", "evaluation_system", "training_system"].includes(decision.authority)) {
|
|
203
|
-
context.addIssue({
|
|
204
|
-
code: "custom",
|
|
205
|
-
message: `${decision.authority} routes cannot be marked automatic`,
|
|
206
|
-
path: ["automatic"],
|
|
207
|
-
});
|
|
208
|
-
}
|
|
209
|
-
if (decision.route === "training" && decision.authority !== "training_system") {
|
|
210
|
-
context.addIssue({
|
|
211
|
-
code: "custom",
|
|
212
|
-
message: "training routes require training-system authority",
|
|
213
|
-
path: ["authority"],
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
|
-
});
|
|
217
|
-
export const ImprovementRouteDecisionSchema = ImprovementRouteDecisionContentSchema.extend({
|
|
218
|
-
contentHash: ReleaseHashSchema,
|
|
219
|
-
}).strict();
|
|
220
|
-
export const HarnessRefinerOutcomeContentSchema = z
|
|
221
|
-
.object({
|
|
222
|
-
schemaVersion: z.literal("openpond.harnessRefinerOutcome.v1"),
|
|
223
|
-
id: ReleaseIdSchema,
|
|
224
|
-
trigger: ImmutableReleaseRefSchema,
|
|
225
|
-
decision: z.enum(["no_action", "proposed"]),
|
|
226
|
-
proposal: ImmutableReleaseRefSchema.nullable(),
|
|
227
|
-
reason: BoundedTextSchema,
|
|
228
|
-
evidenceRefs: z.array(ImmutableReleaseRefSchema).max(100),
|
|
229
|
-
estimatedCostUsd: z.number().finite().nonnegative(),
|
|
230
|
-
createdAt: ReleaseTimestampSchema,
|
|
231
|
-
metadata: MetadataSchema,
|
|
232
|
-
})
|
|
233
|
-
.strict()
|
|
234
|
-
.superRefine((outcome, context) => {
|
|
235
|
-
if ((outcome.decision === "proposed") !== (outcome.proposal !== null)) {
|
|
236
|
-
context.addIssue({
|
|
237
|
-
code: "custom",
|
|
238
|
-
message: "proposed Refiner outcomes require a proposal; no-action outcomes cannot include one",
|
|
239
|
-
path: ["proposal"],
|
|
240
|
-
});
|
|
241
|
-
}
|
|
242
|
-
});
|
|
243
|
-
export const HarnessRefinerOutcomeSchema = HarnessRefinerOutcomeContentSchema.extend({
|
|
244
|
-
contentHash: ReleaseHashSchema,
|
|
245
|
-
}).strict();
|
|
246
|
-
export const ImprovementApplyDecisionSchema = z.enum([
|
|
247
|
-
"applied",
|
|
248
|
-
"retained",
|
|
249
|
-
"declined",
|
|
250
|
-
"conflict",
|
|
251
|
-
"rolled_back",
|
|
252
|
-
]);
|
|
253
|
-
export const ImprovementApplyReceiptContentSchema = z
|
|
254
|
-
.object({
|
|
255
|
-
schemaVersion: z.literal("openpond.improvementApplyReceipt.v1"),
|
|
256
|
-
id: ReleaseIdSchema,
|
|
257
|
-
proposal: ImmutableReleaseRefSchema,
|
|
258
|
-
beforeOverlay: HarnessOverlaySnapshotRefSchema,
|
|
259
|
-
afterOverlay: HarnessOverlaySnapshotRefSchema.nullable(),
|
|
260
|
-
decision: ImprovementApplyDecisionSchema,
|
|
261
|
-
boundary: ImprovementSafeBoundarySchema,
|
|
262
|
-
validationRefs: z.array(ImmutableReleaseRefSchema).max(100),
|
|
263
|
-
outcomeEvidenceRefs: z.array(ImprovementEventRefSchema).max(1_000),
|
|
264
|
-
rollbackOf: ImmutableReleaseRefSchema.nullable(),
|
|
265
|
-
createdAt: ReleaseTimestampSchema,
|
|
266
|
-
metadata: MetadataSchema,
|
|
267
|
-
})
|
|
268
|
-
.strict()
|
|
269
|
-
.superRefine((receipt, context) => {
|
|
270
|
-
if ((receipt.decision === "applied") !== (receipt.afterOverlay !== null)) {
|
|
271
|
-
context.addIssue({
|
|
272
|
-
code: "custom",
|
|
273
|
-
message: "applied receipts require an after-overlay; other decisions cannot include one",
|
|
274
|
-
path: ["afterOverlay"],
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
if ((receipt.decision === "rolled_back") !== (receipt.rollbackOf !== null)) {
|
|
278
|
-
context.addIssue({
|
|
279
|
-
code: "custom",
|
|
280
|
-
message: "only rollback receipts require rollbackOf",
|
|
281
|
-
path: ["rollbackOf"],
|
|
282
|
-
});
|
|
283
|
-
}
|
|
284
|
-
});
|
|
285
|
-
export const ImprovementApplyReceiptSchema = ImprovementApplyReceiptContentSchema.extend({
|
|
286
|
-
contentHash: ReleaseHashSchema,
|
|
287
|
-
}).strict();
|
|
288
|
-
function createHashedContract(input) {
|
|
289
|
-
const parsed = input.contentSchema.parse(input.content);
|
|
290
|
-
return input.resultSchema.parse({
|
|
291
|
-
...parsed,
|
|
292
|
-
contentHash: contentHash(parsed),
|
|
293
|
-
});
|
|
294
|
-
}
|
|
295
|
-
export function createImprovementObservation(content) {
|
|
296
|
-
return createHashedContract({
|
|
297
|
-
content,
|
|
298
|
-
contentSchema: ImprovementObservationContentSchema,
|
|
299
|
-
resultSchema: ImprovementObservationSchema,
|
|
300
|
-
});
|
|
301
|
-
}
|
|
302
|
-
export function createRefinementTriggerDecision(content) {
|
|
303
|
-
return createHashedContract({
|
|
304
|
-
content,
|
|
305
|
-
contentSchema: RefinementTriggerDecisionContentSchema,
|
|
306
|
-
resultSchema: RefinementTriggerDecisionSchema,
|
|
307
|
-
});
|
|
308
|
-
}
|
|
309
|
-
export function createImprovementRouteDecision(content) {
|
|
310
|
-
return createHashedContract({
|
|
311
|
-
content,
|
|
312
|
-
contentSchema: ImprovementRouteDecisionContentSchema,
|
|
313
|
-
resultSchema: ImprovementRouteDecisionSchema,
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
export function createHarnessRefinerOutcome(content) {
|
|
317
|
-
return createHashedContract({
|
|
318
|
-
content,
|
|
319
|
-
contentSchema: HarnessRefinerOutcomeContentSchema,
|
|
320
|
-
resultSchema: HarnessRefinerOutcomeSchema,
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
export function createImprovementApplyReceipt(content) {
|
|
324
|
-
return createHashedContract({
|
|
325
|
-
content,
|
|
326
|
-
contentSchema: ImprovementApplyReceiptContentSchema,
|
|
327
|
-
resultSchema: ImprovementApplyReceiptSchema,
|
|
328
|
-
});
|
|
329
|
-
}
|