@dzhechkov/harness-core 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/dist/create-skill.d.ts +2 -0
  2. package/dist/create-skill.d.ts.map +1 -1
  3. package/dist/create-skill.js +189 -4
  4. package/dist/create-skill.js.map +1 -1
  5. package/package.json +4 -4
  6. package/src/create-skill.ts +193 -4
package/dist/create-skill.d.ts CHANGED
@@ -20,6 +20,8 @@ export interface CreateSkillOptions {
20
20
  readonly withReferences?: boolean | undefined;
21
21
  /** Trust tier (1-3). Default: 1. */
22
22
  readonly trustTier?: number | undefined;
23
+ /** Generate BTO-compatible eval templates with 3-layer benchmarks. */
24
+ readonly bto?: boolean | undefined;
23
25
  }
24
26
  /** Result of skill creation. */
25
27
  export interface CreateSkillResult {
package/dist/create-skill.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"create-skill.d.ts","sourceRoot":"","sources":["../src/create-skill.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,kCAAkC;AAClC,MAAM,WAAW,kBAAkB;IACjC,4CAA4C;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,mFAAmF;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,8CAA8C;IAC9C,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACzC,qCAAqC;IACrC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,oCAAoC;IACpC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC;AAED,gCAAgC;AAChC,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC;AA4ED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,iBAAiB,CAyCvE"}
1
+ {"version":3,"file":"create-skill.d.ts","sourceRoot":"","sources":["../src/create-skill.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,kCAAkC;AAClC,MAAM,WAAW,kBAAkB;IACjC,4CAA4C;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,mFAAmF;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,8CAA8C;IAC9C,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACzC,qCAAqC;IACrC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,oCAAoC;IACpC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,sEAAsE;IACtE,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACpC;AAED,gCAAgC;AAChC,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC;AAiQD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,iBAAiB,CA+CvE"}
package/dist/create-skill.js CHANGED
@@ -78,6 +78,184 @@ test_cases:
78
78
  summary_contains: "${name}"
79
79
  `;
80
80
  }
81
+ /** Generate BTO-compatible eval template with 3-layer benchmarks. */
82
+ function generateBtoEval(name) {
83
+ return `# ${name} — BTO evaluation
84
+ # Run: /bto-test .claude/skills/${name}
85
+ # Layers: L0 (deterministic) → L1 (single judge) → L2 (3-judge panel)
86
+
87
+ artifact_type: skill
88
+ bto_version: "1.0"
89
+
90
+ # ── Layer 0: Deterministic Pre-checks (free, always runs) ──────────
91
+ layer_0:
92
+ gate_threshold: 80 # percent pass rate to proceed
93
+ checks:
94
+ # Universal checks (U1-U5)
95
+ - id: U1
96
+ name: "file exists"
97
+ check: "SKILL.md exists in skill directory"
98
+ - id: U2
99
+ name: "valid UTF-8"
100
+ check: "SKILL.md is valid UTF-8"
101
+ - id: U3
102
+ name: "has headings"
103
+ check: "SKILL.md contains at least one markdown heading"
104
+ - id: U4
105
+ name: "no excessive blanks"
106
+ check: "no more than 3 consecutive blank lines"
107
+ - id: U5
108
+ name: "size bounds"
109
+ check: "file size between 100 bytes and 50KB"
110
+ # Skill-specific checks (S1-S10)
111
+ - id: S1
112
+ name: "frontmatter present"
113
+ check: "SKILL.md starts with YAML frontmatter (---)"
114
+ - id: S2
115
+ name: "name field"
116
+ check: "frontmatter contains name field"
117
+ - id: S3
118
+ name: "description field"
119
+ check: "frontmatter contains description field"
120
+ - id: S4
121
+ name: "trust_tier field"
122
+ check: "frontmatter contains trust_tier (1-3)"
123
+ - id: S5
124
+ name: "protocol section"
125
+ check: "SKILL.md contains ## Protocol or ## Steps"
126
+ - id: S6
127
+ name: "output section"
128
+ check: "SKILL.md contains ## Output"
129
+ - id: S7
130
+ name: "schema exists"
131
+ check: "schemas/output.json exists and is valid JSON"
132
+ - id: S8
133
+ name: "validator exists"
134
+ check: "scripts/validate-config.json exists"
135
+ - id: S9
136
+ name: "when to use"
137
+ check: "SKILL.md contains ## When to use"
138
+ - id: S10
139
+ name: "no TODO placeholders"
140
+ check: "no [TODO] or [PLACEHOLDER] markers in SKILL.md"
141
+
142
+ # ── Layer 1: Single LLM Judge (Haiku, quick) ──────────────────────
143
+ layer_1:
144
+ model: haiku
145
+ pass_threshold: 7.0
146
+ dimensions:
147
+ - name: CLARITY
148
+ weight: 1.0
149
+ anchors:
150
+ "9-10": "Instructions unambiguous, no interpretation needed"
151
+ "5-6": "Mostly clear but some steps need clarification"
152
+ "1-3": "Confusing, contradictory, or missing instructions"
153
+ - name: COMPLETENESS
154
+ weight: 1.0
155
+ anchors:
156
+ "9-10": "All sections filled, protocol covers edge cases"
157
+ "5-6": "Main path covered, edge cases missing"
158
+ "1-3": "Stub-level, most sections empty"
159
+ - name: ACTIONABILITY
160
+ weight: 1.0
161
+ anchors:
162
+ "9-10": "Each step produces a concrete, verifiable output"
163
+ "5-6": "Some steps vague or unmeasurable"
164
+ "1-3": "Steps are descriptions, not actions"
165
+ - name: QUALITY
166
+ weight: 1.0
167
+ anchors:
168
+ "9-10": "Production-ready, well-structured, follows conventions"
169
+ "5-6": "Functional but needs polish"
170
+ "1-3": "Draft quality, significant issues"
171
+ - name: ANTI_PATTERNS
172
+ weight: 1.0
173
+ anchors:
174
+ "9-10": "No anti-patterns detected"
175
+ "5-6": "Minor anti-patterns (e.g., wall of text)"
176
+ "1-3": "Major anti-patterns (e.g., no error handling, no abort)"
177
+
178
+ # ── Layer 2: Full Judge Panel (3 × Sonnet, deep) ──────────────────
179
+ layer_2:
180
+ judges:
181
+ - role: expert
182
+ weight: 0.40
183
+ focus: "methodology, depth, correctness, domain fit"
184
+ - role: critic
185
+ weight: 0.30
186
+ focus: "gaps, weaknesses, anti-patterns, failure modes"
187
+ - role: auditor
188
+ weight: 0.30
189
+ focus: "structure, coverage, cross-references, completeness"
190
+ model: sonnet
191
+ pass_threshold: 7.0
192
+ dimensions:
193
+ - METHODOLOGY
194
+ - DEPTH
195
+ - CORRECTNESS
196
+ - USABILITY
197
+ - ROBUSTNESS
198
+ disagreement_threshold: 3 # max-min > 3 triggers meta-judge
199
+
200
+ # ── Quality Gates ──────────────────────────────────────────────────
201
+ gates:
202
+ layer_0: "pass_rate >= 80%"
203
+ layer_1: "average >= 7.0"
204
+ layer_2: "weighted_average >= 7.0"
205
+ optimization_delta: 0.5 # min improvement per iteration
206
+ max_iterations: 10
207
+
208
+ # ── Test Cases ─────────────────────────────────────────────────────
209
+ test_cases:
210
+ - name: "basic invocation"
211
+ input: "Run ${name}"
212
+ expected:
213
+ status: success
214
+ summary_contains: "${name}"
215
+ - name: "edge case — empty input"
216
+ input: ""
217
+ expected:
218
+ status: failure
219
+ summary_contains: "input required"
220
+ - name: "edge case — malformed input"
221
+ input: "{{RANDOM_GARBAGE}}"
222
+ expected:
223
+ status: failure
224
+ summary_contains: "invalid"
225
+ `;
226
+ }
227
+ /** Generate BTO judge rubrics reference file. */
228
+ function generateBtoRubrics(name) {
229
+ return `# ${name} — Judge Rubrics
230
+
231
+ ## Evaluation Dimensions (Layer 2)
232
+
233
+ | Dimension | Expert (0.40) | Critic (0.30) | Auditor (0.30) |
234
+ |-----------|--------------|---------------|----------------|
235
+ | METHODOLOGY | Multi-step protocol with decision points | Missing decision branches | Steps are numbered and sequential |
236
+ | DEPTH | Detailed modules, references, examples | Stub sections, no examples | All sections present and non-empty |
237
+ | CORRECTNESS | Instructions produce expected output | Wrong output or side effects | Schema matches actual output |
238
+ | USABILITY | Quick start, clear navigation | Wall of text, no structure | Cross-references work, paths valid |
239
+ | ROBUSTNESS | Anti-patterns, failure modes, abort | No error handling | Failure modes documented |
240
+
241
+ ## Scoring Anchors
242
+
243
+ | Score | Label | Description |
244
+ |-------|-------|-------------|
245
+ | 9-10 | Excellent | Production-ready, no changes needed |
246
+ | 7-8 | Good | Minor improvements possible |
247
+ | 5-6 | Needs work | Functional but gaps exist |
248
+ | 3-4 | Poor | Significant rework required |
249
+ | 1-2 | Failed | Does not meet minimum standards |
250
+
251
+ ## Anti-Patterns to Flag
252
+
253
+ - Score inflation: all judges score >8.5 on first attempt
254
+ - Conformity collapse: identical scores across all judges
255
+ - Missing rejection log: failed checks silently skipped
256
+ - Phantom improvement: score delta >0.5 but no content change
257
+ `;
258
+ }
81
259
  /**
82
260
  * Create a new skill directory with all required files.
83
261
  *
@@ -107,14 +285,21 @@ export function createSkill(opts) {
107
285
  // Optional: evals/
108
286
  if (opts.withEvals !== false) {
109
287
  mkdirSync(join(skillDir, 'evals'), { recursive: true });
110
- writeFileSync(join(skillDir, 'evals', `${opts.name}.yaml`), generateEval(opts.name));
288
+ const evalContent = opts.bto ? generateBtoEval(opts.name) : generateEval(opts.name);
289
+ writeFileSync(join(skillDir, 'evals', `${opts.name}.yaml`), evalContent);
111
290
  filesCreated.push(`evals/${opts.name}.yaml`);
112
291
  }
113
292
  // Optional: references/
114
- if (opts.withReferences) {
293
+ if (opts.withReferences || opts.bto) {
115
294
  mkdirSync(join(skillDir, 'references'), { recursive: true });
116
- writeFileSync(join(skillDir, 'references', '.gitkeep'), '');
117
- filesCreated.push('references/.gitkeep');
295
+ if (opts.bto) {
296
+ writeFileSync(join(skillDir, 'references', 'judge-rubrics.md'), generateBtoRubrics(opts.name));
297
+ filesCreated.push('references/judge-rubrics.md');
298
+ }
299
+ else {
300
+ writeFileSync(join(skillDir, 'references', '.gitkeep'), '');
301
+ filesCreated.push('references/.gitkeep');
302
+ }
118
303
  }
119
304
  return { skillDir, filesCreated, alreadyExists: false };
120
305
  }
package/dist/create-skill.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"create-skill.js","sourceRoot":"","sources":["../src/create-skill.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAyBjC,iEAAiE;AACjE,SAAS,eAAe,CAAC,IAAwB;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;IACjC,OAAO;SACA,IAAI,CAAC,IAAI;gBACF,IAAI,CAAC,WAAW;cAClB,IAAI;qBACG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY;;;;;;IAMlF,IAAI,CAAC,IAAI;;EAEX,IAAI,CAAC,WAAW;;;;;;;;;;;;;;;CAejB,CAAC;AACF,CAAC;AAED,uCAAuC;AACvC,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,yCAAyC;QAClD,KAAK,EAAE,GAAG,IAAI,SAAS;QACvB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;YACnE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;SACxD;QACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;KAChC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACd,CAAC;AAED,8CAA8C;AAC9C,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,yCAAyC;QAClD,KAAK,EAAE,GAAG,IAAI,oBAAoB;QAClC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE;SAC5C;KACF,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACd,CAAC;AAED,8BAA8B;AAC9B,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,KAAK,IAAI;kBACA,IAAI;;;;kBAIJ,IAAI;;;2BAGK,IAAI;CAC9B,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAwB;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,gBAAgB,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED,qBAAqB;IACrB,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,WAAW;IACX,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,aAAa,CAAC,WAAW,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE9B,sBAAsB;IACtB,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEzC,+BAA+B;IAC/B,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpG,YAAY,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAElD,mBAAmB;IACnB,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACrF,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC1D,CAAC"}
1
+ {"version":3,"file":"create-skill.js","sourceRoot":"","sources":["../src/create-skill.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA2BjC,iEAAiE;AACjE,SAAS,eAAe,CAAC,IAAwB;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;IACjC,OAAO;SACA,IAAI,CAAC,IAAI;gBACF,IAAI,CAAC,WAAW;cAClB,IAAI;qBACG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY;;;;;;IAMlF,IAAI,CAAC,IAAI;;EAEX,IAAI,CAAC,WAAW;;;;;;;;;;;;;;;CAejB,CAAC;AACF,CAAC;AAED,uCAAuC;AACvC,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,yCAAyC;QAClD,KAAK,EAAE,GAAG,IAAI,SAAS;QACvB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;YACnE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;SACxD;QACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;KAChC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACd,CAAC;AAED,8CAA8C;AAC9C,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,yCAAyC;QAClD,KAAK,EAAE,GAAG,IAAI,oBAAoB;QAClC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE;SAC5C;KACF,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACd,CAAC;AAED,8BAA8B;AAC9B,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,KAAK,IAAI;kBACA,IAAI;;;;kBAIJ,IAAI;;;2BAGK,IAAI;CAC9B,CAAC;AACF,CAAC;AAED,qEAAqE;AACrE,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,KAAK,IAAI;kCACgB,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA+HpB,IAAI;;;2BAGK,IAAI;;;;;;;;;;;CAW9B,CAAC;AACF,CAAC;AAED,iDAAiD;AACjD,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BjB,CAAC;AACF,CAAC;AAGD;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAwB;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,gBAAgB,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED,qBAAqB;IACrB,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,WAAW;IACX,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,aAAa,CAAC,WAAW,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE9B,sBAAsB;IACtB,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEzC,+BAA+B;IAC/B,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpG,YAAY,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAElD,mBAAmB;IACnB,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpF,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC;QACzE,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/F,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC1D,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dzhechkov/harness-core",
3
- "version": "0.3.0",
3
+ "version": "0.3.1",
4
4
  "description": "Shared harness logic - skill loading, additive apply, and the init/sync/verify/doctor operations.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -25,11 +25,11 @@
25
25
  ],
26
26
  "dependencies": {
27
27
  "yaml": "^2.0.0",
28
- "@dzhechkov/adapter-claude": "0.2.0",
29
28
  "@dzhechkov/adapter-codex": "0.2.0",
30
- "@dzhechkov/adapter-hermes": "0.2.0",
31
29
  "@dzhechkov/adapter-opencode": "0.2.0",
32
- "@dzhechkov/core": "0.2.0"
30
+ "@dzhechkov/core": "0.2.0",
31
+ "@dzhechkov/adapter-claude": "0.2.0",
32
+ "@dzhechkov/adapter-hermes": "0.2.0"
33
33
  },
34
34
  "devDependencies": {
35
35
  "@types/node": "^25.6.0",
package/src/create-skill.ts CHANGED
@@ -24,6 +24,8 @@ export interface CreateSkillOptions {
24
24
  readonly withReferences?: boolean | undefined;
25
25
  /** Trust tier (1-3). Default: 1. */
26
26
  readonly trustTier?: number | undefined;
27
+ /** Generate BTO-compatible eval templates with 3-layer benchmarks. */
28
+ readonly bto?: boolean | undefined;
27
29
  }
28
30
 
29
31
  /** Result of skill creation. */
@@ -107,6 +109,187 @@ test_cases:
107
109
  `;
108
110
  }
109
111
 
112
+ /** Generate BTO-compatible eval template with 3-layer benchmarks. */
113
+ function generateBtoEval(name: string): string {
114
+ return `# ${name} — BTO evaluation
115
+ # Run: /bto-test .claude/skills/${name}
116
+ # Layers: L0 (deterministic) → L1 (single judge) → L2 (3-judge panel)
117
+
118
+ artifact_type: skill
119
+ bto_version: "1.0"
120
+
121
+ # ── Layer 0: Deterministic Pre-checks (free, always runs) ──────────
122
+ layer_0:
123
+ gate_threshold: 80 # percent pass rate to proceed
124
+ checks:
125
+ # Universal checks (U1-U5)
126
+ - id: U1
127
+ name: "file exists"
128
+ check: "SKILL.md exists in skill directory"
129
+ - id: U2
130
+ name: "valid UTF-8"
131
+ check: "SKILL.md is valid UTF-8"
132
+ - id: U3
133
+ name: "has headings"
134
+ check: "SKILL.md contains at least one markdown heading"
135
+ - id: U4
136
+ name: "no excessive blanks"
137
+ check: "no more than 3 consecutive blank lines"
138
+ - id: U5
139
+ name: "size bounds"
140
+ check: "file size between 100 bytes and 50KB"
141
+ # Skill-specific checks (S1-S10)
142
+ - id: S1
143
+ name: "frontmatter present"
144
+ check: "SKILL.md starts with YAML frontmatter (---)"
145
+ - id: S2
146
+ name: "name field"
147
+ check: "frontmatter contains name field"
148
+ - id: S3
149
+ name: "description field"
150
+ check: "frontmatter contains description field"
151
+ - id: S4
152
+ name: "trust_tier field"
153
+ check: "frontmatter contains trust_tier (1-3)"
154
+ - id: S5
155
+ name: "protocol section"
156
+ check: "SKILL.md contains ## Protocol or ## Steps"
157
+ - id: S6
158
+ name: "output section"
159
+ check: "SKILL.md contains ## Output"
160
+ - id: S7
161
+ name: "schema exists"
162
+ check: "schemas/output.json exists and is valid JSON"
163
+ - id: S8
164
+ name: "validator exists"
165
+ check: "scripts/validate-config.json exists"
166
+ - id: S9
167
+ name: "when to use"
168
+ check: "SKILL.md contains ## When to use"
169
+ - id: S10
170
+ name: "no TODO placeholders"
171
+ check: "no [TODO] or [PLACEHOLDER] markers in SKILL.md"
172
+
173
+ # ── Layer 1: Single LLM Judge (Haiku, quick) ──────────────────────
174
+ layer_1:
175
+ model: haiku
176
+ pass_threshold: 7.0
177
+ dimensions:
178
+ - name: CLARITY
179
+ weight: 1.0
180
+ anchors:
181
+ "9-10": "Instructions unambiguous, no interpretation needed"
182
+ "5-6": "Mostly clear but some steps need clarification"
183
+ "1-3": "Confusing, contradictory, or missing instructions"
184
+ - name: COMPLETENESS
185
+ weight: 1.0
186
+ anchors:
187
+ "9-10": "All sections filled, protocol covers edge cases"
188
+ "5-6": "Main path covered, edge cases missing"
189
+ "1-3": "Stub-level, most sections empty"
190
+ - name: ACTIONABILITY
191
+ weight: 1.0
192
+ anchors:
193
+ "9-10": "Each step produces a concrete, verifiable output"
194
+ "5-6": "Some steps vague or unmeasurable"
195
+ "1-3": "Steps are descriptions, not actions"
196
+ - name: QUALITY
197
+ weight: 1.0
198
+ anchors:
199
+ "9-10": "Production-ready, well-structured, follows conventions"
200
+ "5-6": "Functional but needs polish"
201
+ "1-3": "Draft quality, significant issues"
202
+ - name: ANTI_PATTERNS
203
+ weight: 1.0
204
+ anchors:
205
+ "9-10": "No anti-patterns detected"
206
+ "5-6": "Minor anti-patterns (e.g., wall of text)"
207
+ "1-3": "Major anti-patterns (e.g., no error handling, no abort)"
208
+
209
+ # ── Layer 2: Full Judge Panel (3 × Sonnet, deep) ──────────────────
210
+ layer_2:
211
+ judges:
212
+ - role: expert
213
+ weight: 0.40
214
+ focus: "methodology, depth, correctness, domain fit"
215
+ - role: critic
216
+ weight: 0.30
217
+ focus: "gaps, weaknesses, anti-patterns, failure modes"
218
+ - role: auditor
219
+ weight: 0.30
220
+ focus: "structure, coverage, cross-references, completeness"
221
+ model: sonnet
222
+ pass_threshold: 7.0
223
+ dimensions:
224
+ - METHODOLOGY
225
+ - DEPTH
226
+ - CORRECTNESS
227
+ - USABILITY
228
+ - ROBUSTNESS
229
+ disagreement_threshold: 3 # max-min > 3 triggers meta-judge
230
+
231
+ # ── Quality Gates ──────────────────────────────────────────────────
232
+ gates:
233
+ layer_0: "pass_rate >= 80%"
234
+ layer_1: "average >= 7.0"
235
+ layer_2: "weighted_average >= 7.0"
236
+ optimization_delta: 0.5 # min improvement per iteration
237
+ max_iterations: 10
238
+
239
+ # ── Test Cases ─────────────────────────────────────────────────────
240
+ test_cases:
241
+ - name: "basic invocation"
242
+ input: "Run ${name}"
243
+ expected:
244
+ status: success
245
+ summary_contains: "${name}"
246
+ - name: "edge case — empty input"
247
+ input: ""
248
+ expected:
249
+ status: failure
250
+ summary_contains: "input required"
251
+ - name: "edge case — malformed input"
252
+ input: "{{RANDOM_GARBAGE}}"
253
+ expected:
254
+ status: failure
255
+ summary_contains: "invalid"
256
+ `;
257
+ }
258
+
259
+ /** Generate BTO judge rubrics reference file. */
260
+ function generateBtoRubrics(name: string): string {
261
+ return `# ${name} — Judge Rubrics
262
+
263
+ ## Evaluation Dimensions (Layer 2)
264
+
265
+ | Dimension | Expert (0.40) | Critic (0.30) | Auditor (0.30) |
266
+ |-----------|--------------|---------------|----------------|
267
+ | METHODOLOGY | Multi-step protocol with decision points | Missing decision branches | Steps are numbered and sequential |
268
+ | DEPTH | Detailed modules, references, examples | Stub sections, no examples | All sections present and non-empty |
269
+ | CORRECTNESS | Instructions produce expected output | Wrong output or side effects | Schema matches actual output |
270
+ | USABILITY | Quick start, clear navigation | Wall of text, no structure | Cross-references work, paths valid |
271
+ | ROBUSTNESS | Anti-patterns, failure modes, abort | No error handling | Failure modes documented |
272
+
273
+ ## Scoring Anchors
274
+
275
+ | Score | Label | Description |
276
+ |-------|-------|-------------|
277
+ | 9-10 | Excellent | Production-ready, no changes needed |
278
+ | 7-8 | Good | Minor improvements possible |
279
+ | 5-6 | Needs work | Functional but gaps exist |
280
+ | 3-4 | Poor | Significant rework required |
281
+ | 1-2 | Failed | Does not meet minimum standards |
282
+
283
+ ## Anti-Patterns to Flag
284
+
285
+ - Score inflation: all judges score >8.5 on first attempt
286
+ - Conformity collapse: identical scores across all judges
287
+ - Missing rejection log: failed checks silently skipped
288
+ - Phantom improvement: score delta >0.5 but no content change
289
+ `;
290
+ }
291
+
292
+
110
293
  /**
111
294
  * Create a new skill directory with all required files.
112
295
  *
@@ -142,15 +325,21 @@ export function createSkill(opts: CreateSkillOptions): CreateSkillResult {
142
325
  // Optional: evals/
143
326
  if (opts.withEvals !== false) {
144
327
  mkdirSync(join(skillDir, 'evals'), { recursive: true });
145
- writeFileSync(join(skillDir, 'evals', `${opts.name}.yaml`), generateEval(opts.name));
328
+ const evalContent = opts.bto ? generateBtoEval(opts.name) : generateEval(opts.name);
329
+ writeFileSync(join(skillDir, 'evals', `${opts.name}.yaml`), evalContent);
146
330
  filesCreated.push(`evals/${opts.name}.yaml`);
147
331
  }
148
332
 
149
333
  // Optional: references/
150
- if (opts.withReferences) {
334
+ if (opts.withReferences || opts.bto) {
151
335
  mkdirSync(join(skillDir, 'references'), { recursive: true });
152
- writeFileSync(join(skillDir, 'references', '.gitkeep'), '');
153
- filesCreated.push('references/.gitkeep');
336
+ if (opts.bto) {
337
+ writeFileSync(join(skillDir, 'references', 'judge-rubrics.md'), generateBtoRubrics(opts.name));
338
+ filesCreated.push('references/judge-rubrics.md');
339
+ } else {
340
+ writeFileSync(join(skillDir, 'references', '.gitkeep'), '');
341
+ filesCreated.push('references/.gitkeep');
342
+ }
154
343
  }
155
344
 
156
345
  return { skillDir, filesCreated, alreadyExists: false };