@skill-harness/core 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/adapters/types.d.ts +0 -12
- package/dist/capture-trace-types.d.ts +0 -96
- package/dist/capture-trace-types.js +0 -4
- package/dist/command-cost.d.ts +1 -1
- package/dist/command-cost.js +0 -5
- package/dist/execution-trace.js +1 -1
- package/dist/index.d.ts +3 -23
- package/dist/index.js +3 -23
- package/dist/instruction-coverage.d.ts +0 -7
- package/dist/instruction-coverage.js +5 -7
- package/dist/journal.d.ts +0 -8
- package/dist/metrics.d.ts +18 -0
- package/dist/metrics.js +28 -0
- package/dist/redaction.d.ts +6 -0
- package/dist/redaction.js +55 -0
- package/dist/regate.js +6 -71
- package/dist/regrade.d.ts +1 -3
- package/dist/regrade.js +1 -21
- package/dist/report.d.ts +1 -15
- package/dist/report.js +1 -13
- package/dist/reps.d.ts +1 -2
- package/dist/reps.js +0 -2
- package/dist/results.d.ts +0 -9
- package/dist/results.js +2 -19
- package/dist/run.js +11 -72
- package/dist/seeded.d.ts +1 -7
- package/dist/seeded.js +5 -11
- package/dist/sources.js +4 -9
- package/dist/spec.d.ts +0 -13
- package/dist/spec.js +6 -48
- package/dist/trajectory-events.d.ts +58 -0
- package/dist/trajectory-events.js +119 -0
- package/package.json +1 -1
- package/schemas/results-v2.schema.json +376 -75
- package/schemas/specification-v1.schema.json +128 -174
- package/dist/adjudication.d.ts +0 -219
- package/dist/adjudication.js +0 -422
- package/dist/adoption.d.ts +0 -128
- package/dist/adoption.js +0 -102
- package/dist/affected.d.ts +0 -88
- package/dist/affected.js +0 -222
- package/dist/capture.d.ts +0 -193
- package/dist/capture.js +0 -344
- package/dist/comparison.d.ts +0 -97
- package/dist/comparison.js +0 -189
- package/dist/factory-calibration.d.ts +0 -63
- package/dist/factory-calibration.js +0 -115
- package/dist/intervention-results.d.ts +0 -6
- package/dist/intervention-results.js +0 -52
- package/dist/intervention.d.ts +0 -123
- package/dist/intervention.js +0 -218
- package/dist/investigation.d.ts +0 -94
- package/dist/investigation.js +0 -148
- package/dist/judge-agreement.d.ts +0 -19
- package/dist/judge-agreement.js +0 -31
- package/dist/qualification-capture.d.ts +0 -44
- package/dist/qualification-capture.js +0 -163
- package/dist/qualification-config.d.ts +0 -233
- package/dist/qualification-config.js +0 -694
- package/dist/qualification-lock.d.ts +0 -15
- package/dist/qualification-lock.js +0 -87
- package/dist/qualification-oauth-directory.d.ts +0 -79
- package/dist/qualification-oauth-directory.js +0 -444
- package/dist/qualification-panel-store.d.ts +0 -14
- package/dist/qualification-panel-store.js +0 -188
- package/dist/qualification-panels.d.ts +0 -99
- package/dist/qualification-panels.js +0 -91
- package/dist/qualification-process.d.ts +0 -13
- package/dist/qualification-process.js +0 -190
- package/dist/qualification-runner.d.ts +0 -203
- package/dist/qualification-runner.js +0 -2037
- package/dist/qualification-store.d.ts +0 -187
- package/dist/qualification-store.js +0 -833
- package/dist/qualification-validation.d.ts +0 -12
- package/dist/qualification-validation.js +0 -8
- package/dist/screen.d.ts +0 -31
- package/dist/screen.js +0 -105
- package/dist/trajectory-gates.d.ts +0 -137
- package/dist/trajectory-gates.js +0 -576
- package/dist/trusted-host-supervision.d.ts +0 -25
- package/dist/trusted-host-supervision.js +0 -94
- package/dist/work-capture-schema.d.ts +0 -38
- package/dist/work-capture-schema.js +0 -33
- package/dist/work-capture.d.ts +0 -43
- package/dist/work-capture.js +0 -81
- package/dist/work-signals.d.ts +0 -55
- package/dist/work-signals.js +0 -71
- package/schemas/qualification-cell-result-v1.schema.json +0 -32
- package/schemas/qualification-config-v1.schema.json +0 -244
- package/schemas/qualification-invocation-request-v1.schema.json +0 -59
- package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
- package/schemas/trajectory-event-v1.schema.json +0 -80
|
@@ -1,244 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "https://skill-harness.dev/schemas/qualification-config-v1.schema.json",
|
|
4
|
-
"title": "skill-harness qualification runner configuration v1",
|
|
5
|
-
"type": "object",
|
|
6
|
-
"additionalProperties": false,
|
|
7
|
-
"required": ["schema_version", "mode", "product", "engine", "producer", "runner", "accounting", "arms"],
|
|
8
|
-
"properties": {
|
|
9
|
-
"schema_version": { "const": "qualification-config-v1" },
|
|
10
|
-
"oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" },
|
|
11
|
-
"terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" },
|
|
12
|
-
"judge_panel": { "$ref": "#/$defs/judgePanel" },
|
|
13
|
-
"board": { "$ref": "#/$defs/board" },
|
|
14
|
-
"mode": { "enum": ["production", "test"] },
|
|
15
|
-
"product": { "$ref": "#/$defs/productPin" },
|
|
16
|
-
"engine": { "$ref": "#/$defs/enginePin" },
|
|
17
|
-
"producer": { "$ref": "#/$defs/producerPin" },
|
|
18
|
-
"runner": { "$ref": "#/$defs/runner" },
|
|
19
|
-
"accounting": { "oneOf": [{ "$ref": "#/$defs/accountingLegacy" }, { "$ref": "#/$defs/accountingPanel" }] },
|
|
20
|
-
"arms": { "type": "array", "minItems": 2, "uniqueItems": true, "items": { "$ref": "#/$defs/arm" } }
|
|
21
|
-
},
|
|
22
|
-
"allOf": [
|
|
23
|
-
{ "if": { "properties": { "judge_panel": { "$ref": "#/$defs/judgePanel" } }, "required": ["judge_panel"] }, "then": { "required": ["oauth_directory_policy", "terminal_receipt_version", "board"], "properties": { "accounting": { "$ref": "#/$defs/accountingPanel" }, "board": { "$ref": "#/$defs/board" }, "oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" }, "terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" } } }, "else": { "properties": { "accounting": { "$ref": "#/$defs/accountingLegacy" }, "board": false } } },
|
|
24
|
-
{ "if": { "properties": { "terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" } }, "required": ["terminal_receipt_version"] }, "then": { "properties": { "oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" } }, "required": ["oauth_directory_policy"] } },
|
|
25
|
-
{ "type": "object", "properties": { "arms": { "type": "array", "contains": { "type": "object", "properties": { "kind": { "const": "subject" } }, "required": ["kind"] }, "minContains": 1 } } },
|
|
26
|
-
{ "type": "object", "properties": { "arms": { "type": "array", "contains": { "type": "object", "properties": { "kind": { "const": "judge" } }, "required": ["kind"] }, "minContains": 1 } } },
|
|
27
|
-
{
|
|
28
|
-
"if": { "type": "object", "properties": { "mode": { "const": "production" } }, "required": ["mode"] },
|
|
29
|
-
"then": {
|
|
30
|
-
"type": "object", "properties": {
|
|
31
|
-
"product": { "type": "object", "properties": {
|
|
32
|
-
"repository": { "const": "https://github.com/mojomanyana/principal-pi-skills" },
|
|
33
|
-
"commit": { "const": "a6596950d64a3a525f95329d5dbd3e38948be408" },
|
|
34
|
-
"tree": { "const": "960359d69deb6f216724b86e13eef67e2f6a6aa1" },
|
|
35
|
-
"package_sha256": { "const": "3677f76fb31dcaf6a28c9e8b9cf9d6358f998b78bd370f0a0f250d05e136fcc8" },
|
|
36
|
-
"package_bytes": { "const": 110168 }
|
|
37
|
-
} },
|
|
38
|
-
"engine": { "type": "object", "properties": { "repository": { "const": "https://github.com/mojomanyana/skill-harness" } } },
|
|
39
|
-
"producer": { "type": "object", "properties": {
|
|
40
|
-
"repository": { "const": "https://github.com/mojomanyana/pi-daddy" },
|
|
41
|
-
"commit": { "const": "4a9524394ca995fd74ed9bbb836dc4e73cda3b8c" },
|
|
42
|
-
"tree": { "const": "7c006bff213142634f0f911ba9bd6add363ecaae" },
|
|
43
|
-
"version": { "const": "0.21.1" },
|
|
44
|
-
"ledger_version": { "const": 3 },
|
|
45
|
-
"ledger_schema_sha256": { "const": "64e3d875e74bc32fa43fb96892605548259cd16f6ed6678646d73cc56280c511" }
|
|
46
|
-
} },
|
|
47
|
-
"arms": {
|
|
48
|
-
"type": "array", "items": {
|
|
49
|
-
"type": "object", "allOf": [
|
|
50
|
-
{ "type": "object", "properties": { "provider": { "const": "openai-codex" }, "authentication": { "const": "chatgpt-oauth" } } },
|
|
51
|
-
{
|
|
52
|
-
"if": { "type": "object", "properties": { "kind": { "const": "subject" } }, "required": ["kind"] },
|
|
53
|
-
"then": { "type": "object", "properties": { "model": { "enum": ["gpt-5.6-luna", "gpt-5.6-terra"] } } },
|
|
54
|
-
"else": { "type": "object", "properties": { "model": { "const": "gpt-5.6-sol" } } }
|
|
55
|
-
},
|
|
56
|
-
{ "type": "object", "properties": { "allowed_environment_names": { "type": "array", "contains": { "enum": ["HOME", "PI_CODING_AGENT_DIR"] } } } }
|
|
57
|
-
]
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
},
|
|
62
|
-
"else": {
|
|
63
|
-
"type": "object", "properties": {
|
|
64
|
-
"arms": { "type": "array", "items": { "type": "object", "properties": { "provider": { "const": "fake" }, "authentication": { "const": "test-oauth" } } } }
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
],
|
|
69
|
-
"$defs": {
|
|
70
|
-
"sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
|
|
71
|
-
"gitSha": { "type": "string", "pattern": "^[a-fA-F0-9]{40}$" },
|
|
72
|
-
"absolutePath": { "type": "string", "pattern": "^(?:/|[A-Za-z]:[\\\\/])" },
|
|
73
|
-
"executable": {
|
|
74
|
-
"type": "object",
|
|
75
|
-
"additionalProperties": false,
|
|
76
|
-
"required": ["path", "sha256"],
|
|
77
|
-
"properties": { "path": { "$ref": "#/$defs/absolutePath" }, "sha256": { "$ref": "#/$defs/sha256" } }
|
|
78
|
-
},
|
|
79
|
-
"repositoryPin": {
|
|
80
|
-
"type": "object",
|
|
81
|
-
"required": ["repository", "commit", "tree"],
|
|
82
|
-
"properties": {
|
|
83
|
-
"repository": { "type": "string", "minLength": 1 },
|
|
84
|
-
"commit": { "$ref": "#/$defs/gitSha" },
|
|
85
|
-
"tree": { "$ref": "#/$defs/gitSha" }
|
|
86
|
-
}
|
|
87
|
-
},
|
|
88
|
-
"productPin": {
|
|
89
|
-
"allOf": [
|
|
90
|
-
{ "$ref": "#/$defs/repositoryPin" },
|
|
91
|
-
{
|
|
92
|
-
"type": "object", "additionalProperties": false,
|
|
93
|
-
"required": ["repository", "commit", "tree", "checkout_path", "package_path", "package_sha256", "package_bytes"],
|
|
94
|
-
"properties": {
|
|
95
|
-
"repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
|
|
96
|
-
"checkout_path": { "$ref": "#/$defs/absolutePath" }, "package_path": { "$ref": "#/$defs/absolutePath" },
|
|
97
|
-
"package_sha256": { "$ref": "#/$defs/sha256" }, "package_bytes": { "type": "integer", "minimum": 1 }
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
]
|
|
101
|
-
},
|
|
102
|
-
"enginePin": {
|
|
103
|
-
"type": "object", "additionalProperties": false,
|
|
104
|
-
"required": ["repository", "commit", "tree", "checkout_path", "package_paths", "package_sha256"],
|
|
105
|
-
"properties": {
|
|
106
|
-
"repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
|
|
107
|
-
"checkout_path": { "$ref": "#/$defs/absolutePath" },
|
|
108
|
-
"package_paths": {
|
|
109
|
-
"type": "object", "additionalProperties": false, "required": ["core", "adapters", "cli", "meta"],
|
|
110
|
-
"properties": { "core": { "$ref": "#/$defs/absolutePath" }, "adapters": { "$ref": "#/$defs/absolutePath" }, "cli": { "$ref": "#/$defs/absolutePath" }, "meta": { "$ref": "#/$defs/absolutePath" } }
|
|
111
|
-
},
|
|
112
|
-
"package_sha256": {
|
|
113
|
-
"type": "object", "additionalProperties": false, "required": ["core", "adapters", "cli", "meta"],
|
|
114
|
-
"properties": { "core": { "$ref": "#/$defs/sha256" }, "adapters": { "$ref": "#/$defs/sha256" }, "cli": { "$ref": "#/$defs/sha256" }, "meta": { "$ref": "#/$defs/sha256" } }
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
},
|
|
118
|
-
"producerPin": {
|
|
119
|
-
"type": "object", "additionalProperties": false,
|
|
120
|
-
"required": ["repository", "commit", "tree", "checkout_path", "version", "ledger_version", "ledger_schema_sha256"],
|
|
121
|
-
"properties": {
|
|
122
|
-
"repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
|
|
123
|
-
"checkout_path": { "$ref": "#/$defs/absolutePath" }, "version": { "type": "string", "minLength": 1 },
|
|
124
|
-
"ledger_version": { "const": 3 }, "ledger_schema_sha256": { "$ref": "#/$defs/sha256" }
|
|
125
|
-
}
|
|
126
|
-
},
|
|
127
|
-
"runner": {
|
|
128
|
-
"type": "object", "additionalProperties": false,
|
|
129
|
-
"required": ["version", "executable", "conflicting_parent_environment"],
|
|
130
|
-
"properties": {
|
|
131
|
-
"version": { "const": "qualification-runner-v1" }, "executable": { "$ref": "#/$defs/executable" },
|
|
132
|
-
"conflicting_parent_environment": { "enum": ["refuse", "remove-and-record"] }
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
"board": {
|
|
136
|
-
"type": "object", "additionalProperties": false, "required": ["schema_version", "read_only", "cells"],
|
|
137
|
-
"properties": {
|
|
138
|
-
"schema_version": { "const": "qualification-board-v1" }, "read_only": { "const": true },
|
|
139
|
-
"cells": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/boardCell" } }
|
|
140
|
-
}
|
|
141
|
-
},
|
|
142
|
-
"boardCell": {
|
|
143
|
-
"type": "object", "additionalProperties": false, "required": ["id", "scenario_id", "measurement_identity_sha256", "scenario_version", "stimulus_sha256", "rubric_sha256", "subject_input_sha256", "subject_arm", "judge_arms", "panels", "critical", "pass_threshold"],
|
|
144
|
-
"properties": {
|
|
145
|
-
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
146
|
-
"scenario_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
147
|
-
"measurement_identity_sha256": { "$ref": "#/$defs/sha256" }, "scenario_version": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
148
|
-
"stimulus_sha256": { "$ref": "#/$defs/sha256" }, "rubric_sha256": { "$ref": "#/$defs/sha256" }, "subject_input_sha256": { "$ref": "#/$defs/sha256" },
|
|
149
|
-
"subject_arm": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
150
|
-
"judge_arms": { "type": "array", "minItems": 3, "maxItems": 3, "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" } },
|
|
151
|
-
"panels": { "type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, "required": ["id", "repetition"], "properties": { "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" }, "repetition": { "type": "integer", "minimum": 0 } } } },
|
|
152
|
-
"critical": { "type": "boolean" }, "pass_threshold": { "type": "integer", "minimum": 1 }
|
|
153
|
-
}
|
|
154
|
-
},
|
|
155
|
-
"judgePanel": {
|
|
156
|
-
"type": "object", "additionalProperties": false,
|
|
157
|
-
"required": ["schema_version", "initial_judge_calls", "agreement", "split", "max_judge_calls", "error", "unresolved", "behavioral_failure", "integrity_failure", "wave_a_call_budget", "complete_program_call_budget"],
|
|
158
|
-
"properties": {
|
|
159
|
-
"schema_version": { "const": "qualification-judge-panel-policy-v1" },
|
|
160
|
-
"initial_judge_calls": { "const": 2 },
|
|
161
|
-
"agreement": { "const": "two-matching-clean-votes" },
|
|
162
|
-
"split": { "const": "one-conditional-tie-break" },
|
|
163
|
-
"max_judge_calls": { "const": 3 },
|
|
164
|
-
"error": { "const": "record-as-non-vote" },
|
|
165
|
-
"unresolved": { "const": "inconclusive-blocks-acceptance" },
|
|
166
|
-
"behavioral_failure": { "const": "record-and-continue-read-only-board" },
|
|
167
|
-
"integrity_failure": { "const": "halt" },
|
|
168
|
-
"wave_a_call_budget": {
|
|
169
|
-
"type": "object", "additionalProperties": false, "required": ["artifacts", "minimum", "maximum"],
|
|
170
|
-
"properties": { "artifacts": { "const": 54 }, "minimum": { "const": 108 }, "maximum": { "const": 162 } }
|
|
171
|
-
},
|
|
172
|
-
"complete_program_call_budget": {
|
|
173
|
-
"type": "object", "additionalProperties": false, "required": ["artifacts", "minimum", "maximum"],
|
|
174
|
-
"properties": { "artifacts": { "const": 642 }, "minimum": { "const": 1284 }, "maximum": { "const": 1926 } }
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
},
|
|
178
|
-
"artifact": {
|
|
179
|
-
"type": "object", "additionalProperties": false, "required": ["type", "relative_path_template"],
|
|
180
|
-
"properties": { "type": { "const": "pi-jsonl" }, "relative_path_template": { "const": "artifacts/{invocation_id}.jsonl" } }
|
|
181
|
-
},
|
|
182
|
-
"arm": {
|
|
183
|
-
"type": "object", "additionalProperties": false,
|
|
184
|
-
"required": ["id", "kind", "provider", "model", "authentication", "executable", "resources", "arguments", "allowed_environment_names", "timeout_ms", "output_limit_bytes", "artifact", "fallback", "metered_override"],
|
|
185
|
-
"properties": {
|
|
186
|
-
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
187
|
-
"kind": { "enum": ["subject", "judge"] }, "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string", "minLength": 1 },
|
|
188
|
-
"authentication": { "enum": ["chatgpt-oauth", "test-oauth"] }, "executable": { "$ref": "#/$defs/executable" },
|
|
189
|
-
"resources": {
|
|
190
|
-
"type": "array", "uniqueItems": true,
|
|
191
|
-
"items": {
|
|
192
|
-
"type": "object", "additionalProperties": false, "required": ["kind", "path", "sha256"],
|
|
193
|
-
"properties": { "kind": { "enum": ["extension", "skill", "system-prompt"] }, "path": { "$ref": "#/$defs/absolutePath" }, "sha256": { "$ref": "#/$defs/sha256" } }
|
|
194
|
-
}
|
|
195
|
-
},
|
|
196
|
-
"arguments": { "type": "array", "items": { "type": "string", "pattern": "^(?!-)(?:[^{}]|\\{(?:input_path|scenario_id|invocation_id|artifact_path)\\})+$" }, "contains": { "type": "string", "pattern": "\\{input_path\\}" } },
|
|
197
|
-
"allowed_environment_names": { "type": "array", "items": { "type": "string", "pattern": "^[A-Z_][A-Z0-9_]*$" }, "uniqueItems": true },
|
|
198
|
-
"timeout_ms": { "type": "integer", "minimum": 1, "maximum": 86400000 },
|
|
199
|
-
"output_limit_bytes": { "type": "integer", "minimum": 1, "maximum": 104857600 },
|
|
200
|
-
"artifact": { "$ref": "#/$defs/artifact" }, "fallback": { "const": false }, "metered_override": { "const": false }
|
|
201
|
-
}
|
|
202
|
-
},
|
|
203
|
-
"accountingLegacy": {
|
|
204
|
-
"type": "object", "additionalProperties": false, "required": ["wave_a", "complete_program", "ceilings", "initial"],
|
|
205
|
-
"properties": { "wave_a": { "$ref": "#/$defs/waveA" }, "complete_program": { "$ref": "#/$defs/completeProgram" }, "ceilings": { "$ref": "#/$defs/ceilings" }, "initial": { "$ref": "#/$defs/initial" } }
|
|
206
|
-
},
|
|
207
|
-
"accountingPanel": {
|
|
208
|
-
"type": "object", "additionalProperties": false, "required": ["wave_a", "complete_program", "ceilings", "initial"],
|
|
209
|
-
"properties": { "wave_a": { "$ref": "#/$defs/waveAPanel" }, "complete_program": { "$ref": "#/$defs/completeProgramPanel" }, "ceilings": { "$ref": "#/$defs/ceilingsPanel" }, "initial": { "$ref": "#/$defs/initial" } }
|
|
210
|
-
},
|
|
211
|
-
"pair": {
|
|
212
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
213
|
-
"properties": { "subject": { "type": "integer" }, "judge": { "type": "integer" } }
|
|
214
|
-
},
|
|
215
|
-
"waveA": {
|
|
216
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
217
|
-
"properties": { "subject": { "const": 54 }, "judge": { "const": 54 } }
|
|
218
|
-
},
|
|
219
|
-
"completeProgram": {
|
|
220
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
221
|
-
"properties": { "subject": { "const": 642 }, "judge": { "const": 642 } }
|
|
222
|
-
},
|
|
223
|
-
"waveAPanel": {
|
|
224
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
225
|
-
"properties": { "subject": { "const": 54 }, "judge": { "const": 162 } }
|
|
226
|
-
},
|
|
227
|
-
"completeProgramPanel": {
|
|
228
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
229
|
-
"properties": { "subject": { "const": 642 }, "judge": { "const": 1926 } }
|
|
230
|
-
},
|
|
231
|
-
"ceilingsPanel": {
|
|
232
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
233
|
-
"properties": { "subject": { "const": 700 }, "judge": { "const": 2100 } }
|
|
234
|
-
},
|
|
235
|
-
"ceilings": {
|
|
236
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
237
|
-
"properties": { "subject": { "const": 700 }, "judge": { "const": 700 } }
|
|
238
|
-
},
|
|
239
|
-
"initial": {
|
|
240
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
241
|
-
"properties": { "subject": { "const": 0 }, "judge": { "const": 0 } }
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "https://skill-harness.dev/schemas/qualification-invocation-request-v1.schema.json",
|
|
4
|
-
"title": "skill-harness qualification invocation request v1",
|
|
5
|
-
"type": "object",
|
|
6
|
-
"additionalProperties": false,
|
|
7
|
-
"required": ["schema_version", "measurement_identity_sha256", "invocation_id", "continuation_authority_sha256", "continuation_authority_expires_at", "scenario", "role", "counts_as_measurement", "arms", "selected_arm", "repetition"],
|
|
8
|
-
"properties": {
|
|
9
|
-
"schema_version": { "const": "qualification-invocation-request-v1" },
|
|
10
|
-
"measurement_identity_sha256": { "$ref": "#/$defs/sha256" },
|
|
11
|
-
"invocation_id": { "$ref": "#/$defs/id" },
|
|
12
|
-
"continuation_authority_sha256": { "$ref": "#/$defs/sha256" },
|
|
13
|
-
"continuation_authority_expires_at": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z$" },
|
|
14
|
-
"scenario": {
|
|
15
|
-
"type": "object", "additionalProperties": false,
|
|
16
|
-
"required": ["id", "version", "stimulus_sha256", "rubric_sha256", "input_path", "input_sha256", "working_directory"],
|
|
17
|
-
"properties": {
|
|
18
|
-
"id": { "$ref": "#/$defs/id" }, "version": { "$ref": "#/$defs/id" },
|
|
19
|
-
"stimulus_sha256": { "$ref": "#/$defs/sha256" }, "rubric_sha256": { "$ref": "#/$defs/sha256" },
|
|
20
|
-
"input_path": { "$ref": "#/$defs/absolutePath" }, "input_sha256": { "$ref": "#/$defs/sha256" },
|
|
21
|
-
"working_directory": { "$ref": "#/$defs/absolutePath" }
|
|
22
|
-
}
|
|
23
|
-
},
|
|
24
|
-
"role": { "enum": ["holdout-author", "holdout-reviewer", "subject", "judge", "calibration", "canary"] },
|
|
25
|
-
"counts_as_measurement": { "type": "boolean" },
|
|
26
|
-
"arms": {
|
|
27
|
-
"type": "object", "additionalProperties": false, "required": ["subject", "judge"],
|
|
28
|
-
"properties": { "subject": { "$ref": "#/$defs/id" }, "judge": { "$ref": "#/$defs/id" } }
|
|
29
|
-
},
|
|
30
|
-
"selected_arm": { "$ref": "#/$defs/id" },
|
|
31
|
-
"repetition": { "type": "integer", "minimum": 0 },
|
|
32
|
-
"panel": {
|
|
33
|
-
"type": "object", "additionalProperties": false,
|
|
34
|
-
"required": ["id", "member_ordinal", "subject_invocation_id", "subject_artifact_sha256"],
|
|
35
|
-
"properties": {
|
|
36
|
-
"id": { "$ref": "#/$defs/id" },
|
|
37
|
-
"member_ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
|
|
38
|
-
"subject_invocation_id": { "$ref": "#/$defs/id" },
|
|
39
|
-
"subject_artifact_sha256": { "$ref": "#/$defs/sha256" }
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
},
|
|
43
|
-
"allOf": [
|
|
44
|
-
{
|
|
45
|
-
"if": { "properties": { "panel": { "type": "object" } }, "required": ["panel"] },
|
|
46
|
-
"then": { "properties": { "role": { "const": "judge" }, "selected_arm": { "$ref": "#/$defs/id" } }, "required": ["role", "selected_arm"] }
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
"if": { "type": "object", "properties": { "role": { "enum": ["subject", "judge"] } }, "required": ["role"] },
|
|
50
|
-
"then": { "type": "object", "properties": { "counts_as_measurement": { "const": true } } },
|
|
51
|
-
"else": { "type": "object", "properties": { "counts_as_measurement": { "const": false } } }
|
|
52
|
-
}
|
|
53
|
-
],
|
|
54
|
-
"$defs": {
|
|
55
|
-
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
56
|
-
"absolutePath": { "type": "string", "pattern": "^(?:/|[A-Za-z]:[\\\\/])" },
|
|
57
|
-
"sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" }
|
|
58
|
-
}
|
|
59
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "https://skill-harness.dev/schemas/qualification-judge-panel-result-v1.schema.json",
|
|
4
|
-
"title": "skill-harness qualification judge panel result v1",
|
|
5
|
-
"type": "object",
|
|
6
|
-
"additionalProperties": false,
|
|
7
|
-
"required": ["schema_version", "panel_id", "scenario_id", "subject_arm", "repetition", "critical", "state", "members", "judge_calls", "clean_votes", "disagreement", "acceptance", "collection"],
|
|
8
|
-
"properties": {
|
|
9
|
-
"schema_version": { "const": "qualification-judge-panel-result-v1" },
|
|
10
|
-
"panel_id": { "$ref": "#/$defs/id" }, "scenario_id": { "$ref": "#/$defs/id" }, "subject_arm": { "$ref": "#/$defs/id" },
|
|
11
|
-
"repetition": { "type": "integer", "minimum": 0 }, "critical": { "type": "boolean" },
|
|
12
|
-
"state": { "enum": ["confirmed", "tie_broken", "unresolved"] }, "verdict": { "enum": ["PASS", "FAIL"] },
|
|
13
|
-
"members": { "type": "array", "minItems": 2, "maxItems": 3, "items": { "$ref": "#/$defs/member" } },
|
|
14
|
-
"judge_calls": { "type": "integer", "minimum": 2, "maximum": 3 }, "clean_votes": { "type": "integer", "minimum": 0, "maximum": 3 },
|
|
15
|
-
"disagreement": { "type": "object", "additionalProperties": false, "required": ["initial_split", "minority_rate"], "properties": { "initial_split": { "type": "boolean" }, "minority_rate": { "type": "number", "minimum": 0, "maximum": 0.5 } } },
|
|
16
|
-
"acceptance": { "enum": ["pass", "fail", "inconclusive"] }, "collection": { "const": "continue" }
|
|
17
|
-
},
|
|
18
|
-
"allOf": [
|
|
19
|
-
{ "if": { "properties": { "state": { "const": "unresolved" } }, "required": ["state"] }, "then": { "properties": { "acceptance": { "const": "inconclusive" } }, "not": { "properties": { "verdict": {} }, "required": ["verdict"] } }, "else": { "required": ["verdict"], "properties": { "acceptance": { "enum": ["pass", "fail"] } } } },
|
|
20
|
-
{ "if": { "properties": { "state": { "const": "tie_broken" } }, "required": ["state"] }, "then": { "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 3 }, "disagreement": { "type": "object", "properties": { "initial_split": { "const": true } } } } } },
|
|
21
|
-
{ "if": { "properties": { "verdict": { "const": "PASS" } }, "required": ["verdict"] }, "then": { "properties": { "acceptance": { "const": "pass" } } } },
|
|
22
|
-
{ "if": { "properties": { "verdict": { "const": "FAIL" } }, "required": ["verdict"] }, "then": { "properties": { "acceptance": { "const": "fail" } } } },
|
|
23
|
-
{ "if": { "properties": { "judge_calls": { "const": 2 } }, "required": ["judge_calls"] }, "then": { "properties": { "members": { "type": "array", "minItems": 2, "maxItems": 2 } } } },
|
|
24
|
-
{ "if": { "properties": { "judge_calls": { "const": 3 } }, "required": ["judge_calls"] }, "then": { "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 3 } } } },
|
|
25
|
-
{ "if": { "properties": { "disagreement": { "type": "object", "properties": { "initial_split": { "const": true } }, "required": ["initial_split"] } }, "required": ["disagreement"] }, "then": { "properties": { "state": { "const": "tie_broken" }, "members": { "type": "array", "minItems": 3, "maxItems": 3 } } } }
|
|
26
|
-
],
|
|
27
|
-
"$defs": {
|
|
28
|
-
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
29
|
-
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
30
|
-
"member": {
|
|
31
|
-
"type": "object", "additionalProperties": false,
|
|
32
|
-
"required": ["invocation_id", "ordinal", "judge", "verdict", "reason", "suspect", "artifact", "terminal_receipt_sha256"],
|
|
33
|
-
"properties": {
|
|
34
|
-
"invocation_id": { "$ref": "#/$defs/id" }, "ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
|
|
35
|
-
"judge": { "type": "object", "additionalProperties": false, "required": ["provider", "model"], "properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string", "minLength": 1 } } },
|
|
36
|
-
"verdict": { "enum": ["PASS", "FAIL", "ERROR", "JUDGE-AMBIGUOUS"] }, "reason": { "type": "string" }, "suspect": { "type": "boolean" },
|
|
37
|
-
"artifact": { "oneOf": [{ "type": "null" }, { "type": "object", "additionalProperties": false, "required": ["sha256", "bytes"], "properties": { "sha256": { "$ref": "#/$defs/sha256" }, "bytes": { "type": "integer", "minimum": 0 } } }] },
|
|
38
|
-
"terminal_receipt_sha256": { "$ref": "#/$defs/sha256" }
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "https://skill-harness.dev/schemas/trajectory-event-v1.schema.json",
|
|
4
|
-
"title": "skill-harness normalized trajectory event v1",
|
|
5
|
-
"type": "object",
|
|
6
|
-
"additionalProperties": false,
|
|
7
|
-
"required": ["event_version", "seq", "type", "source"],
|
|
8
|
-
"properties": {
|
|
9
|
-
"event_version": { "enum": ["1.0", "1.1"] },
|
|
10
|
-
"seq": { "type": "integer", "minimum": 1 },
|
|
11
|
-
"type": { "type": "string", "minLength": 1 },
|
|
12
|
-
"source": { "type": "string", "minLength": 1 },
|
|
13
|
-
"at": { "type": "string", "format": "date-time" },
|
|
14
|
-
"run_id": { "$ref": "#/$defs/id" },
|
|
15
|
-
"task_id": { "$ref": "#/$defs/id" },
|
|
16
|
-
"workspace_id": { "$ref": "#/$defs/id" },
|
|
17
|
-
"context_id": { "$ref": "#/$defs/id" },
|
|
18
|
-
"finding_id": { "$ref": "#/$defs/id" },
|
|
19
|
-
"parent_id": { "$ref": "#/$defs/id" },
|
|
20
|
-
"child_id": { "$ref": "#/$defs/id" },
|
|
21
|
-
"execution_id": { "$ref": "#/$defs/id" },
|
|
22
|
-
"parent_execution_id": { "oneOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] },
|
|
23
|
-
"task_from_execution_id": { "$ref": "#/$defs/id" },
|
|
24
|
-
"workflow_fact_id": { "$ref": "#/$defs/id" },
|
|
25
|
-
"deadline_at": { "type": "string", "format": "date-time" },
|
|
26
|
-
"phase": { "type": "string", "minLength": 1 },
|
|
27
|
-
"tool": { "type": "string", "minLength": 1 },
|
|
28
|
-
"capability": { "type": "string", "minLength": 1 },
|
|
29
|
-
"requested_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
|
|
30
|
-
"effective_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
|
|
31
|
-
"refusal_code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" },
|
|
32
|
-
"exit_code": { "type": "integer" },
|
|
33
|
-
"digests": {
|
|
34
|
-
"type": "object",
|
|
35
|
-
"properties": {
|
|
36
|
-
"plan": { "$ref": "#/$defs/sha256" },
|
|
37
|
-
"task": { "$ref": "#/$defs/sha256" },
|
|
38
|
-
"definition": { "$ref": "#/$defs/sha256" },
|
|
39
|
-
"head": { "$ref": "#/$defs/gitSha" },
|
|
40
|
-
"tree": { "$ref": "#/$defs/gitSha" }
|
|
41
|
-
},
|
|
42
|
-
"additionalProperties": { "type": "string" }
|
|
43
|
-
},
|
|
44
|
-
"approval": {
|
|
45
|
-
"type": "object",
|
|
46
|
-
"additionalProperties": false,
|
|
47
|
-
"properties": {
|
|
48
|
-
"id": { "$ref": "#/$defs/id" },
|
|
49
|
-
"capability": { "type": "string", "minLength": 1 },
|
|
50
|
-
"subject": { "type": "string", "minLength": 1 },
|
|
51
|
-
"source": { "type": "string", "minLength": 1 },
|
|
52
|
-
"scope": { "type": "string", "minLength": 1 },
|
|
53
|
-
"approved_at": { "type": "string", "format": "date-time" },
|
|
54
|
-
"expires_at": { "type": "string", "format": "date-time" },
|
|
55
|
-
"used_at": { "type": "string", "format": "date-time" }
|
|
56
|
-
}
|
|
57
|
-
},
|
|
58
|
-
"requirements": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
|
|
59
|
-
"attributes": { "type": "object" }
|
|
60
|
-
},
|
|
61
|
-
"allOf": [
|
|
62
|
-
{
|
|
63
|
-
"if": { "properties": { "event_version": { "const": "1.0" } }, "required": ["event_version"] },
|
|
64
|
-
"then": {
|
|
65
|
-
"properties": {
|
|
66
|
-
"execution_id": false,
|
|
67
|
-
"parent_execution_id": false,
|
|
68
|
-
"task_from_execution_id": false,
|
|
69
|
-
"workflow_fact_id": false,
|
|
70
|
-
"deadline_at": false
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
],
|
|
75
|
-
"$defs": {
|
|
76
|
-
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
|
|
77
|
-
"sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
|
|
78
|
-
"gitSha": { "type": "string", "pattern": "^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$" }
|
|
79
|
-
}
|
|
80
|
-
}
|