@skill-harness/core 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +2 -12
  3. package/dist/capture-trace-types.d.ts +8 -102
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.d.ts +0 -16
  8. package/dist/execution-trace.js +31 -42
  9. package/dist/index.d.ts +3 -23
  10. package/dist/index.js +3 -23
  11. package/dist/instruction-coverage.d.ts +0 -7
  12. package/dist/instruction-coverage.js +5 -7
  13. package/dist/journal.d.ts +0 -8
  14. package/dist/metrics.d.ts +19 -0
  15. package/dist/metrics.js +28 -0
  16. package/dist/redaction.d.ts +6 -0
  17. package/dist/redaction.js +55 -0
  18. package/dist/regate.js +6 -71
  19. package/dist/regrade.d.ts +1 -3
  20. package/dist/regrade.js +1 -21
  21. package/dist/report.d.ts +1 -15
  22. package/dist/report.js +1 -13
  23. package/dist/reps.d.ts +1 -2
  24. package/dist/reps.js +33 -10
  25. package/dist/results.d.ts +10 -9
  26. package/dist/results.js +3 -19
  27. package/dist/run.js +15 -74
  28. package/dist/seeded.d.ts +1 -7
  29. package/dist/seeded.js +5 -11
  30. package/dist/sources.js +4 -9
  31. package/dist/spec.d.ts +0 -13
  32. package/dist/spec.js +6 -48
  33. package/dist/trajectory-events.d.ts +58 -0
  34. package/dist/trajectory-events.js +119 -0
  35. package/package.json +1 -1
  36. package/schemas/results-v2.schema.json +394 -75
  37. package/schemas/results-v3.schema.json +18 -0
  38. package/schemas/specification-v1.schema.json +128 -174
  39. package/dist/adjudication.d.ts +0 -219
  40. package/dist/adjudication.js +0 -422
  41. package/dist/adoption.d.ts +0 -128
  42. package/dist/adoption.js +0 -102
  43. package/dist/affected.d.ts +0 -88
  44. package/dist/affected.js +0 -222
  45. package/dist/capture.d.ts +0 -193
  46. package/dist/capture.js +0 -344
  47. package/dist/comparison.d.ts +0 -97
  48. package/dist/comparison.js +0 -189
  49. package/dist/factory-calibration.d.ts +0 -63
  50. package/dist/factory-calibration.js +0 -115
  51. package/dist/intervention-results.d.ts +0 -6
  52. package/dist/intervention-results.js +0 -52
  53. package/dist/intervention.d.ts +0 -123
  54. package/dist/intervention.js +0 -218
  55. package/dist/investigation.d.ts +0 -94
  56. package/dist/investigation.js +0 -148
  57. package/dist/judge-agreement.d.ts +0 -19
  58. package/dist/judge-agreement.js +0 -31
  59. package/dist/qualification-capture.d.ts +0 -44
  60. package/dist/qualification-capture.js +0 -163
  61. package/dist/qualification-config.d.ts +0 -233
  62. package/dist/qualification-config.js +0 -694
  63. package/dist/qualification-lock.d.ts +0 -15
  64. package/dist/qualification-lock.js +0 -87
  65. package/dist/qualification-oauth-directory.d.ts +0 -79
  66. package/dist/qualification-oauth-directory.js +0 -444
  67. package/dist/qualification-panel-store.d.ts +0 -14
  68. package/dist/qualification-panel-store.js +0 -188
  69. package/dist/qualification-panels.d.ts +0 -99
  70. package/dist/qualification-panels.js +0 -91
  71. package/dist/qualification-process.d.ts +0 -13
  72. package/dist/qualification-process.js +0 -190
  73. package/dist/qualification-runner.d.ts +0 -203
  74. package/dist/qualification-runner.js +0 -2037
  75. package/dist/qualification-store.d.ts +0 -187
  76. package/dist/qualification-store.js +0 -833
  77. package/dist/qualification-validation.d.ts +0 -12
  78. package/dist/qualification-validation.js +0 -8
  79. package/dist/screen.d.ts +0 -31
  80. package/dist/screen.js +0 -105
  81. package/dist/trajectory-gates.d.ts +0 -137
  82. package/dist/trajectory-gates.js +0 -576
  83. package/dist/trusted-host-supervision.d.ts +0 -25
  84. package/dist/trusted-host-supervision.js +0 -94
  85. package/dist/work-capture-schema.d.ts +0 -38
  86. package/dist/work-capture-schema.js +0 -33
  87. package/dist/work-capture.d.ts +0 -43
  88. package/dist/work-capture.js +0 -81
  89. package/dist/work-signals.d.ts +0 -55
  90. package/dist/work-signals.js +0 -71
  91. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  92. package/schemas/qualification-config-v1.schema.json +0 -244
  93. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  94. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  95. package/schemas/trajectory-event-v1.schema.json +0 -80
@@ -1,244 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://skill-harness.dev/schemas/qualification-config-v1.schema.json",
4
- "title": "skill-harness qualification runner configuration v1",
5
- "type": "object",
6
- "additionalProperties": false,
7
- "required": ["schema_version", "mode", "product", "engine", "producer", "runner", "accounting", "arms"],
8
- "properties": {
9
- "schema_version": { "const": "qualification-config-v1" },
10
- "oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" },
11
- "terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" },
12
- "judge_panel": { "$ref": "#/$defs/judgePanel" },
13
- "board": { "$ref": "#/$defs/board" },
14
- "mode": { "enum": ["production", "test"] },
15
- "product": { "$ref": "#/$defs/productPin" },
16
- "engine": { "$ref": "#/$defs/enginePin" },
17
- "producer": { "$ref": "#/$defs/producerPin" },
18
- "runner": { "$ref": "#/$defs/runner" },
19
- "accounting": { "oneOf": [{ "$ref": "#/$defs/accountingLegacy" }, { "$ref": "#/$defs/accountingPanel" }] },
20
- "arms": { "type": "array", "minItems": 2, "uniqueItems": true, "items": { "$ref": "#/$defs/arm" } }
21
- },
22
- "allOf": [
23
- { "if": { "properties": { "judge_panel": { "$ref": "#/$defs/judgePanel" } }, "required": ["judge_panel"] }, "then": { "required": ["oauth_directory_policy", "terminal_receipt_version", "board"], "properties": { "accounting": { "$ref": "#/$defs/accountingPanel" }, "board": { "$ref": "#/$defs/board" }, "oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" }, "terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" } } }, "else": { "properties": { "accounting": { "$ref": "#/$defs/accountingLegacy" }, "board": false } } },
24
- { "if": { "properties": { "terminal_receipt_version": { "const": "qualification-terminal-receipt-v3" } }, "required": ["terminal_receipt_version"] }, "then": { "properties": { "oauth_directory_policy": { "const": "qualification-oauth-directory-policy-v2" } }, "required": ["oauth_directory_policy"] } },
25
- { "type": "object", "properties": { "arms": { "type": "array", "contains": { "type": "object", "properties": { "kind": { "const": "subject" } }, "required": ["kind"] }, "minContains": 1 } } },
26
- { "type": "object", "properties": { "arms": { "type": "array", "contains": { "type": "object", "properties": { "kind": { "const": "judge" } }, "required": ["kind"] }, "minContains": 1 } } },
27
- {
28
- "if": { "type": "object", "properties": { "mode": { "const": "production" } }, "required": ["mode"] },
29
- "then": {
30
- "type": "object", "properties": {
31
- "product": { "type": "object", "properties": {
32
- "repository": { "const": "https://github.com/mojomanyana/principal-pi-skills" },
33
- "commit": { "const": "a6596950d64a3a525f95329d5dbd3e38948be408" },
34
- "tree": { "const": "960359d69deb6f216724b86e13eef67e2f6a6aa1" },
35
- "package_sha256": { "const": "3677f76fb31dcaf6a28c9e8b9cf9d6358f998b78bd370f0a0f250d05e136fcc8" },
36
- "package_bytes": { "const": 110168 }
37
- } },
38
- "engine": { "type": "object", "properties": { "repository": { "const": "https://github.com/mojomanyana/skill-harness" } } },
39
- "producer": { "type": "object", "properties": {
40
- "repository": { "const": "https://github.com/mojomanyana/pi-daddy" },
41
- "commit": { "const": "4a9524394ca995fd74ed9bbb836dc4e73cda3b8c" },
42
- "tree": { "const": "7c006bff213142634f0f911ba9bd6add363ecaae" },
43
- "version": { "const": "0.21.1" },
44
- "ledger_version": { "const": 3 },
45
- "ledger_schema_sha256": { "const": "64e3d875e74bc32fa43fb96892605548259cd16f6ed6678646d73cc56280c511" }
46
- } },
47
- "arms": {
48
- "type": "array", "items": {
49
- "type": "object", "allOf": [
50
- { "type": "object", "properties": { "provider": { "const": "openai-codex" }, "authentication": { "const": "chatgpt-oauth" } } },
51
- {
52
- "if": { "type": "object", "properties": { "kind": { "const": "subject" } }, "required": ["kind"] },
53
- "then": { "type": "object", "properties": { "model": { "enum": ["gpt-5.6-luna", "gpt-5.6-terra"] } } },
54
- "else": { "type": "object", "properties": { "model": { "const": "gpt-5.6-sol" } } }
55
- },
56
- { "type": "object", "properties": { "allowed_environment_names": { "type": "array", "contains": { "enum": ["HOME", "PI_CODING_AGENT_DIR"] } } } }
57
- ]
58
- }
59
- }
60
- }
61
- },
62
- "else": {
63
- "type": "object", "properties": {
64
- "arms": { "type": "array", "items": { "type": "object", "properties": { "provider": { "const": "fake" }, "authentication": { "const": "test-oauth" } } } }
65
- }
66
- }
67
- }
68
- ],
69
- "$defs": {
70
- "sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
71
- "gitSha": { "type": "string", "pattern": "^[a-fA-F0-9]{40}$" },
72
- "absolutePath": { "type": "string", "pattern": "^(?:/|[A-Za-z]:[\\\\/])" },
73
- "executable": {
74
- "type": "object",
75
- "additionalProperties": false,
76
- "required": ["path", "sha256"],
77
- "properties": { "path": { "$ref": "#/$defs/absolutePath" }, "sha256": { "$ref": "#/$defs/sha256" } }
78
- },
79
- "repositoryPin": {
80
- "type": "object",
81
- "required": ["repository", "commit", "tree"],
82
- "properties": {
83
- "repository": { "type": "string", "minLength": 1 },
84
- "commit": { "$ref": "#/$defs/gitSha" },
85
- "tree": { "$ref": "#/$defs/gitSha" }
86
- }
87
- },
88
- "productPin": {
89
- "allOf": [
90
- { "$ref": "#/$defs/repositoryPin" },
91
- {
92
- "type": "object", "additionalProperties": false,
93
- "required": ["repository", "commit", "tree", "checkout_path", "package_path", "package_sha256", "package_bytes"],
94
- "properties": {
95
- "repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
96
- "checkout_path": { "$ref": "#/$defs/absolutePath" }, "package_path": { "$ref": "#/$defs/absolutePath" },
97
- "package_sha256": { "$ref": "#/$defs/sha256" }, "package_bytes": { "type": "integer", "minimum": 1 }
98
- }
99
- }
100
- ]
101
- },
102
- "enginePin": {
103
- "type": "object", "additionalProperties": false,
104
- "required": ["repository", "commit", "tree", "checkout_path", "package_paths", "package_sha256"],
105
- "properties": {
106
- "repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
107
- "checkout_path": { "$ref": "#/$defs/absolutePath" },
108
- "package_paths": {
109
- "type": "object", "additionalProperties": false, "required": ["core", "adapters", "cli", "meta"],
110
- "properties": { "core": { "$ref": "#/$defs/absolutePath" }, "adapters": { "$ref": "#/$defs/absolutePath" }, "cli": { "$ref": "#/$defs/absolutePath" }, "meta": { "$ref": "#/$defs/absolutePath" } }
111
- },
112
- "package_sha256": {
113
- "type": "object", "additionalProperties": false, "required": ["core", "adapters", "cli", "meta"],
114
- "properties": { "core": { "$ref": "#/$defs/sha256" }, "adapters": { "$ref": "#/$defs/sha256" }, "cli": { "$ref": "#/$defs/sha256" }, "meta": { "$ref": "#/$defs/sha256" } }
115
- }
116
- }
117
- },
118
- "producerPin": {
119
- "type": "object", "additionalProperties": false,
120
- "required": ["repository", "commit", "tree", "checkout_path", "version", "ledger_version", "ledger_schema_sha256"],
121
- "properties": {
122
- "repository": { "type": "string", "minLength": 1 }, "commit": { "$ref": "#/$defs/gitSha" }, "tree": { "$ref": "#/$defs/gitSha" },
123
- "checkout_path": { "$ref": "#/$defs/absolutePath" }, "version": { "type": "string", "minLength": 1 },
124
- "ledger_version": { "const": 3 }, "ledger_schema_sha256": { "$ref": "#/$defs/sha256" }
125
- }
126
- },
127
- "runner": {
128
- "type": "object", "additionalProperties": false,
129
- "required": ["version", "executable", "conflicting_parent_environment"],
130
- "properties": {
131
- "version": { "const": "qualification-runner-v1" }, "executable": { "$ref": "#/$defs/executable" },
132
- "conflicting_parent_environment": { "enum": ["refuse", "remove-and-record"] }
133
- }
134
- },
135
- "board": {
136
- "type": "object", "additionalProperties": false, "required": ["schema_version", "read_only", "cells"],
137
- "properties": {
138
- "schema_version": { "const": "qualification-board-v1" }, "read_only": { "const": true },
139
- "cells": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/boardCell" } }
140
- }
141
- },
142
- "boardCell": {
143
- "type": "object", "additionalProperties": false, "required": ["id", "scenario_id", "measurement_identity_sha256", "scenario_version", "stimulus_sha256", "rubric_sha256", "subject_input_sha256", "subject_arm", "judge_arms", "panels", "critical", "pass_threshold"],
144
- "properties": {
145
- "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
146
- "scenario_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
147
- "measurement_identity_sha256": { "$ref": "#/$defs/sha256" }, "scenario_version": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
148
- "stimulus_sha256": { "$ref": "#/$defs/sha256" }, "rubric_sha256": { "$ref": "#/$defs/sha256" }, "subject_input_sha256": { "$ref": "#/$defs/sha256" },
149
- "subject_arm": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
150
- "judge_arms": { "type": "array", "minItems": 3, "maxItems": 3, "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" } },
151
- "panels": { "type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, "required": ["id", "repetition"], "properties": { "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" }, "repetition": { "type": "integer", "minimum": 0 } } } },
152
- "critical": { "type": "boolean" }, "pass_threshold": { "type": "integer", "minimum": 1 }
153
- }
154
- },
155
- "judgePanel": {
156
- "type": "object", "additionalProperties": false,
157
- "required": ["schema_version", "initial_judge_calls", "agreement", "split", "max_judge_calls", "error", "unresolved", "behavioral_failure", "integrity_failure", "wave_a_call_budget", "complete_program_call_budget"],
158
- "properties": {
159
- "schema_version": { "const": "qualification-judge-panel-policy-v1" },
160
- "initial_judge_calls": { "const": 2 },
161
- "agreement": { "const": "two-matching-clean-votes" },
162
- "split": { "const": "one-conditional-tie-break" },
163
- "max_judge_calls": { "const": 3 },
164
- "error": { "const": "record-as-non-vote" },
165
- "unresolved": { "const": "inconclusive-blocks-acceptance" },
166
- "behavioral_failure": { "const": "record-and-continue-read-only-board" },
167
- "integrity_failure": { "const": "halt" },
168
- "wave_a_call_budget": {
169
- "type": "object", "additionalProperties": false, "required": ["artifacts", "minimum", "maximum"],
170
- "properties": { "artifacts": { "const": 54 }, "minimum": { "const": 108 }, "maximum": { "const": 162 } }
171
- },
172
- "complete_program_call_budget": {
173
- "type": "object", "additionalProperties": false, "required": ["artifacts", "minimum", "maximum"],
174
- "properties": { "artifacts": { "const": 642 }, "minimum": { "const": 1284 }, "maximum": { "const": 1926 } }
175
- }
176
- }
177
- },
178
- "artifact": {
179
- "type": "object", "additionalProperties": false, "required": ["type", "relative_path_template"],
180
- "properties": { "type": { "const": "pi-jsonl" }, "relative_path_template": { "const": "artifacts/{invocation_id}.jsonl" } }
181
- },
182
- "arm": {
183
- "type": "object", "additionalProperties": false,
184
- "required": ["id", "kind", "provider", "model", "authentication", "executable", "resources", "arguments", "allowed_environment_names", "timeout_ms", "output_limit_bytes", "artifact", "fallback", "metered_override"],
185
- "properties": {
186
- "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
187
- "kind": { "enum": ["subject", "judge"] }, "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string", "minLength": 1 },
188
- "authentication": { "enum": ["chatgpt-oauth", "test-oauth"] }, "executable": { "$ref": "#/$defs/executable" },
189
- "resources": {
190
- "type": "array", "uniqueItems": true,
191
- "items": {
192
- "type": "object", "additionalProperties": false, "required": ["kind", "path", "sha256"],
193
- "properties": { "kind": { "enum": ["extension", "skill", "system-prompt"] }, "path": { "$ref": "#/$defs/absolutePath" }, "sha256": { "$ref": "#/$defs/sha256" } }
194
- }
195
- },
196
- "arguments": { "type": "array", "items": { "type": "string", "pattern": "^(?!-)(?:[^{}]|\\{(?:input_path|scenario_id|invocation_id|artifact_path)\\})+$" }, "contains": { "type": "string", "pattern": "\\{input_path\\}" } },
197
- "allowed_environment_names": { "type": "array", "items": { "type": "string", "pattern": "^[A-Z_][A-Z0-9_]*$" }, "uniqueItems": true },
198
- "timeout_ms": { "type": "integer", "minimum": 1, "maximum": 86400000 },
199
- "output_limit_bytes": { "type": "integer", "minimum": 1, "maximum": 104857600 },
200
- "artifact": { "$ref": "#/$defs/artifact" }, "fallback": { "const": false }, "metered_override": { "const": false }
201
- }
202
- },
203
- "accountingLegacy": {
204
- "type": "object", "additionalProperties": false, "required": ["wave_a", "complete_program", "ceilings", "initial"],
205
- "properties": { "wave_a": { "$ref": "#/$defs/waveA" }, "complete_program": { "$ref": "#/$defs/completeProgram" }, "ceilings": { "$ref": "#/$defs/ceilings" }, "initial": { "$ref": "#/$defs/initial" } }
206
- },
207
- "accountingPanel": {
208
- "type": "object", "additionalProperties": false, "required": ["wave_a", "complete_program", "ceilings", "initial"],
209
- "properties": { "wave_a": { "$ref": "#/$defs/waveAPanel" }, "complete_program": { "$ref": "#/$defs/completeProgramPanel" }, "ceilings": { "$ref": "#/$defs/ceilingsPanel" }, "initial": { "$ref": "#/$defs/initial" } }
210
- },
211
- "pair": {
212
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
213
- "properties": { "subject": { "type": "integer" }, "judge": { "type": "integer" } }
214
- },
215
- "waveA": {
216
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
217
- "properties": { "subject": { "const": 54 }, "judge": { "const": 54 } }
218
- },
219
- "completeProgram": {
220
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
221
- "properties": { "subject": { "const": 642 }, "judge": { "const": 642 } }
222
- },
223
- "waveAPanel": {
224
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
225
- "properties": { "subject": { "const": 54 }, "judge": { "const": 162 } }
226
- },
227
- "completeProgramPanel": {
228
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
229
- "properties": { "subject": { "const": 642 }, "judge": { "const": 1926 } }
230
- },
231
- "ceilingsPanel": {
232
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
233
- "properties": { "subject": { "const": 700 }, "judge": { "const": 2100 } }
234
- },
235
- "ceilings": {
236
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
237
- "properties": { "subject": { "const": 700 }, "judge": { "const": 700 } }
238
- },
239
- "initial": {
240
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
241
- "properties": { "subject": { "const": 0 }, "judge": { "const": 0 } }
242
- }
243
- }
244
- }
@@ -1,59 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://skill-harness.dev/schemas/qualification-invocation-request-v1.schema.json",
4
- "title": "skill-harness qualification invocation request v1",
5
- "type": "object",
6
- "additionalProperties": false,
7
- "required": ["schema_version", "measurement_identity_sha256", "invocation_id", "continuation_authority_sha256", "continuation_authority_expires_at", "scenario", "role", "counts_as_measurement", "arms", "selected_arm", "repetition"],
8
- "properties": {
9
- "schema_version": { "const": "qualification-invocation-request-v1" },
10
- "measurement_identity_sha256": { "$ref": "#/$defs/sha256" },
11
- "invocation_id": { "$ref": "#/$defs/id" },
12
- "continuation_authority_sha256": { "$ref": "#/$defs/sha256" },
13
- "continuation_authority_expires_at": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z$" },
14
- "scenario": {
15
- "type": "object", "additionalProperties": false,
16
- "required": ["id", "version", "stimulus_sha256", "rubric_sha256", "input_path", "input_sha256", "working_directory"],
17
- "properties": {
18
- "id": { "$ref": "#/$defs/id" }, "version": { "$ref": "#/$defs/id" },
19
- "stimulus_sha256": { "$ref": "#/$defs/sha256" }, "rubric_sha256": { "$ref": "#/$defs/sha256" },
20
- "input_path": { "$ref": "#/$defs/absolutePath" }, "input_sha256": { "$ref": "#/$defs/sha256" },
21
- "working_directory": { "$ref": "#/$defs/absolutePath" }
22
- }
23
- },
24
- "role": { "enum": ["holdout-author", "holdout-reviewer", "subject", "judge", "calibration", "canary"] },
25
- "counts_as_measurement": { "type": "boolean" },
26
- "arms": {
27
- "type": "object", "additionalProperties": false, "required": ["subject", "judge"],
28
- "properties": { "subject": { "$ref": "#/$defs/id" }, "judge": { "$ref": "#/$defs/id" } }
29
- },
30
- "selected_arm": { "$ref": "#/$defs/id" },
31
- "repetition": { "type": "integer", "minimum": 0 },
32
- "panel": {
33
- "type": "object", "additionalProperties": false,
34
- "required": ["id", "member_ordinal", "subject_invocation_id", "subject_artifact_sha256"],
35
- "properties": {
36
- "id": { "$ref": "#/$defs/id" },
37
- "member_ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
38
- "subject_invocation_id": { "$ref": "#/$defs/id" },
39
- "subject_artifact_sha256": { "$ref": "#/$defs/sha256" }
40
- }
41
- }
42
- },
43
- "allOf": [
44
- {
45
- "if": { "properties": { "panel": { "type": "object" } }, "required": ["panel"] },
46
- "then": { "properties": { "role": { "const": "judge" }, "selected_arm": { "$ref": "#/$defs/id" } }, "required": ["role", "selected_arm"] }
47
- },
48
- {
49
- "if": { "type": "object", "properties": { "role": { "enum": ["subject", "judge"] } }, "required": ["role"] },
50
- "then": { "type": "object", "properties": { "counts_as_measurement": { "const": true } } },
51
- "else": { "type": "object", "properties": { "counts_as_measurement": { "const": false } } }
52
- }
53
- ],
54
- "$defs": {
55
- "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
56
- "absolutePath": { "type": "string", "pattern": "^(?:/|[A-Za-z]:[\\\\/])" },
57
- "sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" }
58
- }
59
- }
@@ -1,42 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://skill-harness.dev/schemas/qualification-judge-panel-result-v1.schema.json",
4
- "title": "skill-harness qualification judge panel result v1",
5
- "type": "object",
6
- "additionalProperties": false,
7
- "required": ["schema_version", "panel_id", "scenario_id", "subject_arm", "repetition", "critical", "state", "members", "judge_calls", "clean_votes", "disagreement", "acceptance", "collection"],
8
- "properties": {
9
- "schema_version": { "const": "qualification-judge-panel-result-v1" },
10
- "panel_id": { "$ref": "#/$defs/id" }, "scenario_id": { "$ref": "#/$defs/id" }, "subject_arm": { "$ref": "#/$defs/id" },
11
- "repetition": { "type": "integer", "minimum": 0 }, "critical": { "type": "boolean" },
12
- "state": { "enum": ["confirmed", "tie_broken", "unresolved"] }, "verdict": { "enum": ["PASS", "FAIL"] },
13
- "members": { "type": "array", "minItems": 2, "maxItems": 3, "items": { "$ref": "#/$defs/member" } },
14
- "judge_calls": { "type": "integer", "minimum": 2, "maximum": 3 }, "clean_votes": { "type": "integer", "minimum": 0, "maximum": 3 },
15
- "disagreement": { "type": "object", "additionalProperties": false, "required": ["initial_split", "minority_rate"], "properties": { "initial_split": { "type": "boolean" }, "minority_rate": { "type": "number", "minimum": 0, "maximum": 0.5 } } },
16
- "acceptance": { "enum": ["pass", "fail", "inconclusive"] }, "collection": { "const": "continue" }
17
- },
18
- "allOf": [
19
- { "if": { "properties": { "state": { "const": "unresolved" } }, "required": ["state"] }, "then": { "properties": { "acceptance": { "const": "inconclusive" } }, "not": { "properties": { "verdict": {} }, "required": ["verdict"] } }, "else": { "required": ["verdict"], "properties": { "acceptance": { "enum": ["pass", "fail"] } } } },
20
- { "if": { "properties": { "state": { "const": "tie_broken" } }, "required": ["state"] }, "then": { "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 3 }, "disagreement": { "type": "object", "properties": { "initial_split": { "const": true } } } } } },
21
- { "if": { "properties": { "verdict": { "const": "PASS" } }, "required": ["verdict"] }, "then": { "properties": { "acceptance": { "const": "pass" } } } },
22
- { "if": { "properties": { "verdict": { "const": "FAIL" } }, "required": ["verdict"] }, "then": { "properties": { "acceptance": { "const": "fail" } } } },
23
- { "if": { "properties": { "judge_calls": { "const": 2 } }, "required": ["judge_calls"] }, "then": { "properties": { "members": { "type": "array", "minItems": 2, "maxItems": 2 } } } },
24
- { "if": { "properties": { "judge_calls": { "const": 3 } }, "required": ["judge_calls"] }, "then": { "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 3 } } } },
25
- { "if": { "properties": { "disagreement": { "type": "object", "properties": { "initial_split": { "const": true } }, "required": ["initial_split"] } }, "required": ["disagreement"] }, "then": { "properties": { "state": { "const": "tie_broken" }, "members": { "type": "array", "minItems": 3, "maxItems": 3 } } } }
26
- ],
27
- "$defs": {
28
- "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
29
- "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
30
- "member": {
31
- "type": "object", "additionalProperties": false,
32
- "required": ["invocation_id", "ordinal", "judge", "verdict", "reason", "suspect", "artifact", "terminal_receipt_sha256"],
33
- "properties": {
34
- "invocation_id": { "$ref": "#/$defs/id" }, "ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
35
- "judge": { "type": "object", "additionalProperties": false, "required": ["provider", "model"], "properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string", "minLength": 1 } } },
36
- "verdict": { "enum": ["PASS", "FAIL", "ERROR", "JUDGE-AMBIGUOUS"] }, "reason": { "type": "string" }, "suspect": { "type": "boolean" },
37
- "artifact": { "oneOf": [{ "type": "null" }, { "type": "object", "additionalProperties": false, "required": ["sha256", "bytes"], "properties": { "sha256": { "$ref": "#/$defs/sha256" }, "bytes": { "type": "integer", "minimum": 0 } } }] },
38
- "terminal_receipt_sha256": { "$ref": "#/$defs/sha256" }
39
- }
40
- }
41
- }
42
- }
@@ -1,80 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "https://skill-harness.dev/schemas/trajectory-event-v1.schema.json",
4
- "title": "skill-harness normalized trajectory event v1",
5
- "type": "object",
6
- "additionalProperties": false,
7
- "required": ["event_version", "seq", "type", "source"],
8
- "properties": {
9
- "event_version": { "enum": ["1.0", "1.1"] },
10
- "seq": { "type": "integer", "minimum": 1 },
11
- "type": { "type": "string", "minLength": 1 },
12
- "source": { "type": "string", "minLength": 1 },
13
- "at": { "type": "string", "format": "date-time" },
14
- "run_id": { "$ref": "#/$defs/id" },
15
- "task_id": { "$ref": "#/$defs/id" },
16
- "workspace_id": { "$ref": "#/$defs/id" },
17
- "context_id": { "$ref": "#/$defs/id" },
18
- "finding_id": { "$ref": "#/$defs/id" },
19
- "parent_id": { "$ref": "#/$defs/id" },
20
- "child_id": { "$ref": "#/$defs/id" },
21
- "execution_id": { "$ref": "#/$defs/id" },
22
- "parent_execution_id": { "oneOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] },
23
- "task_from_execution_id": { "$ref": "#/$defs/id" },
24
- "workflow_fact_id": { "$ref": "#/$defs/id" },
25
- "deadline_at": { "type": "string", "format": "date-time" },
26
- "phase": { "type": "string", "minLength": 1 },
27
- "tool": { "type": "string", "minLength": 1 },
28
- "capability": { "type": "string", "minLength": 1 },
29
- "requested_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
30
- "effective_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
31
- "refusal_code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" },
32
- "exit_code": { "type": "integer" },
33
- "digests": {
34
- "type": "object",
35
- "properties": {
36
- "plan": { "$ref": "#/$defs/sha256" },
37
- "task": { "$ref": "#/$defs/sha256" },
38
- "definition": { "$ref": "#/$defs/sha256" },
39
- "head": { "$ref": "#/$defs/gitSha" },
40
- "tree": { "$ref": "#/$defs/gitSha" }
41
- },
42
- "additionalProperties": { "type": "string" }
43
- },
44
- "approval": {
45
- "type": "object",
46
- "additionalProperties": false,
47
- "properties": {
48
- "id": { "$ref": "#/$defs/id" },
49
- "capability": { "type": "string", "minLength": 1 },
50
- "subject": { "type": "string", "minLength": 1 },
51
- "source": { "type": "string", "minLength": 1 },
52
- "scope": { "type": "string", "minLength": 1 },
53
- "approved_at": { "type": "string", "format": "date-time" },
54
- "expires_at": { "type": "string", "format": "date-time" },
55
- "used_at": { "type": "string", "format": "date-time" }
56
- }
57
- },
58
- "requirements": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
59
- "attributes": { "type": "object" }
60
- },
61
- "allOf": [
62
- {
63
- "if": { "properties": { "event_version": { "const": "1.0" } }, "required": ["event_version"] },
64
- "then": {
65
- "properties": {
66
- "execution_id": false,
67
- "parent_execution_id": false,
68
- "task_from_execution_id": false,
69
- "workflow_fact_id": false,
70
- "deadline_at": false
71
- }
72
- }
73
- }
74
- ],
75
- "$defs": {
76
- "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
77
- "sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
78
- "gitSha": { "type": "string", "pattern": "^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$" }
79
- }
80
- }