@dzhechkov/skills-bto 1.3.2 → 1.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dzhechkov/skills-bto",
-  "version": "1.3.2",
+  "version": "1.3.4",
   "description": "Build-Benchmark-Test-Optimize skill pack for Claude Code — deterministic benchmarking, quality gates, witness chain, judge attestation, and optimization",
   "main": "src/cli.js",
   "bin": {
@@ -50,4 +50,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/cli.js

@@ -84,8 +84,14 @@ function parseArgs(argv) {
         command = 'version';
         break;
       default:
-        if (!arg.startsWith('-') && command === null) {
+        if (arg.startsWith('-')) {
+          error(`Unknown option: ${arg}`);
+          process.exit(1);
+        } else if (command === null) {
           command = arg;
+        } else {
+          error(`Unexpected argument: ${arg}`);
+          process.exit(1);
         }
         break;
     }

package/src/commands/init.js

@@ -64,7 +64,7 @@ function installComponent(key, comp, templatesDir, targetDir) {
   if (filterFn) {
     // Filtered component — only copy matching files from shared directory
     copyDirFiltered(src, dest, filterFn);
-    return getRelativePathsFiltered(dest, filterFn).map((rel) => path.join(comp.src, rel));
+    return getRelativePathsFiltered(src, filterFn).map((rel) => path.join(comp.src, rel));
   }
 
   // Non-filtered component — copy entire directory or file
@@ -75,7 +75,7 @@ function installComponent(key, comp, templatesDir, targetDir) {
   }
 
   copyDirRecursive(src, dest);
-  return getRelativePaths(dest).map((rel) => path.join(comp.src, rel));
+  return getRelativePaths(src).map((rel) => path.join(comp.src, rel));
 }
 
 // ---------------------------------------------------------------------------

package/src/commands/update.js

@@ -162,16 +162,19 @@ async function run(options) {
     if (!comp) continue;
 
     const destPath = path.join(targetDir, comp.src);
+    const srcPath = path.join(templatesDir, comp.src);
     const filterFn = getComponentFilter(comp);
+    const scanBase = fileExists(srcPath) ? srcPath : destPath;
 
     if (comp.isFile) {
       if (fileExists(destPath)) {
         allFiles.push(comp.src);
       }
     } else if (fileExists(destPath)) {
+      // Scan the TEMPLATE source, not the destination, so user files aren't adopted.
       const paths = filterFn
-        ? getRelativePathsFiltered(destPath, filterFn)
-        : getRelativePaths(destPath);
+        ? getRelativePathsFiltered(scanBase, filterFn)
+        : getRelativePaths(scanBase);
       allFiles.push(...paths.map((rel) => path.join(comp.src, rel)));
     }
   }

package/src/utils.js

@@ -304,15 +304,17 @@ const COMPONENTS = {
   },
   commands: {
     src: '.claude/commands',
-    label: 'BTO Commands (5 commands)',
+    label: 'BTO Commands (verify-chain + bto*)',
     group: 'core',
     filter: 'bto',
+    extra: ['verify-chain.md'],
   },
   rules: {
     src: '.claude/rules',
-    label: 'BTO Quality Gate Rules',
+    label: 'BTO Quality Gate Rules (+ witness-chain)',
     group: 'core',
     filter: 'bto',
+    extra: ['witness-chain.md'],
   },
   agents: {
     src: '.claude/agents',
@@ -326,6 +328,13 @@ const COMPONENTS = {
     group: 'core',
     filter: 'bto',
   },
+  lib: {
+    // verify-chain.md reads lib/witness-chain.md + lib/judge-attestation.md at runtime;
+    // without this component those reads fail (finding #26).
+    src: 'lib',
+    label: 'BTO verification protocols (witness-chain, judge-attestation)',
+    group: 'core',
+  },
 };
 
 // ===========================================================================
@@ -348,9 +357,10 @@ function getComponentFilter(comp) {
   // rules:   bto-*.md  (e.g., bto-quality-gate.md)
   // agents:  bto-*.md  (e.g., bto-builder.md)
   const prefix = comp.filter; // 'bto'
+  const extra = comp.extra || []; // explicit allowlist of non-prefixed files to include
 
   return (filename) => {
-    return filename.startsWith(prefix);
+    return filename.startsWith(prefix) || extra.includes(filename);
   };
 }
 

package/templates/.claude/agents/bto-optimizer-worker.md

@@ -179,3 +179,10 @@ Each worker writes a log regardless of success:
 ## Reusability Note
 This template is artifact-type agnostic. Replace BASE_ARTIFACT_PATH and RUBRIC_PATH
 to optimize any text artifact: prompts, skills, presentations, research sections, code docstrings.
+
+## Hold-out validation (delegate to `dz bto-optimize`)
+When `dz` is available, do NOT accept a variant on its tuning-set score alone. Split scenarios
+(`dz bto-optimize --split`), score candidates on the tune set, validate the top candidate on the held-out set,
+and call `dz bto-optimize --select` — it accepts a winner only if the weakest dimension improves on the HOLDOUT
+with no regression elsewhere. This is the anti-Goodhart guard the heuristic loop lacks. See
+`skills/bto/modules/optimize.md` → "Rigorous validation".

package/templates/.claude/commands/bto-benchmark.md

@@ -60,13 +60,13 @@ For each required section in the golden sample, check if the artifact contains a
 
 **Axis 2 — Ordering Score:**
 Compare the ordering of matched sections against the golden sample ordering.
-- Use normalized Kendall tau distance: `ordering_score = 1 - (inversions / max_inversions)`
-- If fewer than 3 matched sections → ordering_score = 1.0 (too few to meaningfully compare)
+- Use normalized inversion distance: `ordering_score = 1 - (inversion_count / max_inversions)` where `max_inversions = n * (n - 1) / 2` (n = number of matched sections)
+- If only 0 or 1 sections are matched → ordering_score defaults to 0.0 (too few to compare)
 
 **Axis 3 — Proportion Score:**
-For each matched section, compute the ratio of artifact section length to golden sample section length.
-- `proportion_score = 1 - mean(|log(artifact_len / golden_len)|)` clamped to [0, 1]
-- Measures whether sections are proportionally sized (not too bloated, not too thin)
+For each matched section, compute its proportion of total content (by character count) and compare against the golden sample's expected proportion.
+- `proportion_score = 1 - mean(|actual_proportion[i] - golden_proportion[i]|)` clamped to [0, 1]
+- Measures whether sections are proportionally sized (not too bloated, not too thin). Unmapped sections are excluded.
 
 **Display per-section MATCH/MISS table:**
 
@@ -82,36 +82,36 @@ Golden Section              Status    Artifact Section             Size Ratio
 
 **Golden Similarity Score:**
 ```
-B0 = section_coverage * 0.50 + ordering_score * 0.25 + proportion_score * 0.25
+B0 = mean(section_coverage, ordering_score, proportion_score)
 ```
 
-**Gate:** If B0 < 0.30 → BLOCK immediately. The artifact's structure is too far from the expected form to benchmark meaningfully.
+B0 is not gated on its own — it feeds the aggregate BENCHMARK_SCORE (Step 8), where a low B0 (< 0.40) is surfaced in the escalation fix list. Do not BLOCK at B0 in isolation.
 
 ### Step 5: Layer B1 — Deterministic Test Suite
 
 **Purpose:** Run all applicable deterministic tests for the detected artifact type. Zero LLM cost.
 
-Execute all applicable checks from `benchmark.md` for the detected artifact type. These overlap with but extend the Layer 0 checks from `/bto-test`:
+Execute all applicable checks from `references/quality-checklist.md` for the detected artifact type. (The `benchmark.md` module documents these same checks as its TEST-* suite; `references/quality-checklist.md` is the authoritative ID index.) These overlap with but extend the Layer 0 checks from `/bto-test`:
 
-**Universal Checks (U1-U12):**
+**Universal Checks (U-01 through U-12):**
 - U-01: File exists and is non-empty
 - U-02: UTF-8 encoding valid
 - U-03: Starts with level-1 heading
-- U-04: No placeholder text (`[TODO]`, `[TBD]`, `<INSERT>`, `[PLACEHOLDER]`)
+- U-04: No placeholder text (`TODO`, `FIXME`, `[INSERT`, `<YOUR_`, `...`)
 - U-05: No empty sections (heading with no content before next heading)
-- U-06: Consistent heading hierarchy (no h1→h3 jumps)
+- U-06: Consistent heading hierarchy (no `##`→`####` jumps)
 - U-07: No broken internal cross-references
 - U-08: File size within bounds (200B – 100KB per file)
 - U-09: No trailing whitespace on lines
-- U-10: Standard Markdown only (no HTML unless in code blocks)
+- U-10: Standard Markdown only (no raw HTML)
 - U-11: All code blocks properly closed
 - U-12: No duplicate top-level sections
 
-**Type-Specific Checks:**
+**Type-Specific Checks (see `references/quality-checklist.md` for full definitions):**
 - Skill: SK-01 through SK-16 (SKILL.md exists, required sections, modules/, references/, examples/, etc.)
-- Command: CM-01 through CM-08 (Usage section, Parameters, Protocol steps, Checkpoint, Critical Rules, etc.)
-- Rule: RL-01 through RL-06 (Table format, detection signals, fix actions, no contradictions, etc.)
-- Agent: AG-01 through AG-04 (Model specified, isolation rules, naming convention, output format)
+- Command: CM-01 through CM-11 (location, `$ARGUMENTS`, checkpoint, skill loading, usage line, examples, empty-argument handling, etc.)
+- Rule: RL-01 through RL-09 (table format, detection signals, fix actions, severity designation, no vague patterns, etc.)
+- Agent: AT-01 through AT-10 (purpose, model specified + justified, isolation scope, output format, failure protocol, naming convention, etc.)
 
 **Display per-test PASS/FAIL table:**
 
@@ -132,7 +132,7 @@ SK-03   Has ## Anti-Patterns section       FAIL     Section missing
 B1 = passed_tests / total_applicable_tests
 ```
 
-**Gate:** If B1 < 0.60 → BLOCK. Too many structural failures to proceed to consistency probing.
+B1 is not gated on its own — it feeds the aggregate BENCHMARK_SCORE (Step 8), where a low B1 (< 0.40) is surfaced in the escalation fix list. Do not BLOCK at B1 in isolation.
 
 ### Step 6: Layer B2 — Consistency Probe
 
@@ -142,29 +142,44 @@ B1 = passed_tests / total_applicable_tests
 
 **Spawn 3 parallel haiku agents with the identical evaluation prompt:**
 
-Each agent receives the exact same prompt:
+Each agent independently answers the same 4 probe questions about the artifact:
 ```
-You are evaluating a Claude Code artifact for structural quality.
-Rate these 4 dimensions (0.0 to 1.0 each, two decimal places):
-1. STRUCTURE — Does the artifact follow a clear, logical organization?
-2. CLARITY — Are instructions unambiguous and actionable?
-3. COVERAGE — Does it address all aspects implied by its title/scope?
-4. CONSISTENCY — Is the internal terminology and style uniform?
-Output as JSON: {"structure": X.XX, "clarity": X.XX, "coverage": X.XX, "consistency": X.XX}
+You are analyzing a Claude Code {artifact_type} for consistency.
+Read the artifact carefully, then answer these 4 questions. Be specific and concise.
+
+Q1: What is the PRIMARY purpose of this artifact? (one sentence)
+Q2: List the TOP 3 most important sections/components. (ordered list)
+Q3: What is the MAIN anti-pattern or risk this artifact addresses? (one sentence)
+Q4: If an agent followed this artifact, what would the OUTPUT look like? (2-3 sentences)
+
+## Required Output Format
+PURPOSE: [answer to Q1]
+TOP_SECTIONS:
+1. [section]
+2. [section]
+3. [section]
+MAIN_RISK: [answer to Q3]
+EXPECTED_OUTPUT: [answer to Q4]
 ```
 
 **Isolation:** Each agent evaluates independently. No cross-communication.
 
-**After all 3 return, compute structural agreement:**
+**After all 3 return, the orchestrator (not haiku) measures agreement per question:**
 
-For each dimension:
-```
-agreement[dim] = 1 - (max(scores[dim]) - min(scores[dim]))
-```
+| Agreement Level | Score | Criteria |
+|----------------|-------|----------|
+| Full agreement (3/3 match) | 1.0 | All 3 responses convey the same meaning (semantic match) |
+| Majority agreement (2/3 match) | 0.67 | 2 of 3 agree, 1 diverges |
+| No agreement (all different) | 0.0 | All 3 responses are substantively different |
+
+- Q1 (PURPOSE): all 3 identify the same core purpose (semantic, not lexical)
+- Q2 (TOP_SECTIONS): overlap score = |intersection of all 3| / 3
+- Q3 (MAIN_RISK): all 3 identify the same risk category
+- Q4 (EXPECTED_OUTPUT): all 3 describe structurally similar output
 
 **Consistency Score:**
 ```
-B2 = mean(agreement across all 4 dimensions)
+B2 = mean(q1_agreement, q2_agreement, q3_agreement, q4_agreement)
 ```
 
 **Interpretation:**
@@ -172,16 +187,16 @@ B2 = mean(agreement across all 4 dimensions)
 - B2 0.60-0.85 → MODERATE consistency — some sections are open to interpretation
 - B2 < 0.60 → LOW consistency — the artifact is structurally ambiguous
 
-**Display per-probe comparison:**
+**Display per-question comparison:**
 
 ```
-Dimension      Probe 1   Probe 2   Probe 3   Range    Agreement
-──────────── ────────── ────────── ────────── ──────── ──────────
-STRUCTURE      0.80       0.85       0.82      0.05     0.95
-CLARITY        0.70       0.75       0.65      0.10     0.90
-COVERAGE       0.60       0.70       0.55      0.15     0.85
-CONSISTENCY    0.75       0.78       0.73      0.05     0.95
-                                                Mean:    0.91
+Question                     Agreement   Detail
+──────────────────────────── ─────────── ────────────────────────────────
+Q1 PURPOSE                    1.00        3/3 agree
+Q2 TOP_SECTIONS               0.67        2/3 sections overlap
+Q3 MAIN_RISK                  1.00        3/3 agree
+Q4 EXPECTED_OUTPUT            0.67        2/3 agree
+                                  Mean:    0.84
 ```
 
 No gate on B2 — consistency is informational and feeds into the aggregate score.
@@ -192,43 +207,46 @@ No gate on B2 — consistency is informational and feeds into the aggregate scor
 
 **Metric 1 — Token Efficiency:**
 ```
-token_efficiency = meaningful_content_tokens / total_tokens
+token_efficiency = content_chars / (content_chars + formatting_chars)
 ```
-Where `meaningful_content_tokens` excludes: blank lines, decorative separators, repeated boilerplate headers, and excessive whitespace. Target: > 0.70.
+Where `content_chars` = characters in prose and code blocks, and `formatting_chars` = characters in markdown formatting (headings, dividers, table pipes, bullets). Healthy range: 0.60–0.85.
 
 **Metric 2 — Information Density:**
 ```
-information_density = unique_concepts / total_sections
+density_score = min(unique_concepts / (total_sections * 3), 1.0)
 ```
-Where `unique_concepts` = count of distinct topics, terms, or instructions introduced. Measures whether each section contributes new information vs. repeating prior content. Target: > 2.0 concepts per section.
+Where `unique_concepts` = count of distinct key terms/concepts (unique multi-word phrases in headings + bold text + table headers) and `total_sections` = count of `##` headings. Each section should introduce ~3 unique concepts.
 
 **Metric 3 — Bloat Detection:**
-Identify sections where:
-- Section length > 3x the golden sample equivalent → BLOATED
-- Section length < 0.2x the golden sample equivalent → THIN
-- Section repeats > 30% of content from another section → REDUNDANT
+```
+avg_section_size = mean(section_sizes)
+bloated_sections = sections where size > 3 * avg_section_size
+bloat_ratio = count(bloated_sections) / total_sections
+```
+`bloat_ratio` of 0 is ideal; above 0.3 is problematic.
 
+**Metric 4 — Redundancy Detection:**
 ```
-bloat_score = 1 - (bloated_sections + thin_sections + redundant_sections) / total_sections
+repeated_phrases = phrases of 5+ words appearing 3+ times (excluding code blocks and table formatting)
+redundancy_score = min(count(repeated_phrases) / total_sections, 1.0)
 ```
-Clamped to [0, 1].
+A score of 0 means no redundancy detected.
 
 **Performance Score:**
 ```
-B3 = token_efficiency * 0.35 + normalize(information_density, 0, 4) * 0.35 + bloat_score * 0.30
+B3 = mean(token_efficiency, density_score, 1 - bloat_ratio, 1 - redundancy_score)
 ```
-Where `normalize(x, min, max)` clamps and scales x to [0, 1].
 
 **Display per-section metrics:**
 
 ```
-Section                    Tokens   Concepts   Golden Ratio   Flag
-─────────────────────────  ──────── ────────── ──────────────  ──────────
-## Overview                 320      4          1.2x           OK
-## Protocol                 1840     12         0.8x           OK
-## Anti-Patterns            580      6          2.8x           BLOATED
-## Quick Start              95       2          0.3x           THIN
-## Dependencies             210      3          1.0x           OK
+Section                    Tokens   Concepts   vs Avg Size   Flag
+─────────────────────────  ──────── ────────── ─────────────  ──────────
+## Overview                 320      4          0.4x          OK
+## Protocol                 1840     12         2.3x          OK
+## Anti-Patterns            580      6          0.7x          OK
+## Examples                 2900     4          3.6x          BLOATED
+## Dependencies             210      3          0.3x          OK
 ```
 
 ### Step 8: Aggregate & Gate
@@ -245,8 +263,8 @@ BENCHMARK_SCORE = B0 * 0.30 + B1 * 0.35 + B2 * 0.15 + B3 * 0.20
 - B3 (Performance) at 0.20 — efficiency matters for production artifacts
 
 **Gate:**
-- BENCHMARK_SCORE < 0.50 → **BLOCK** — artifact requires significant rework before evaluation
-- BENCHMARK_SCORE 0.50-0.70 → **WARN** — artifact has structural issues; proceed to TEST with caution
+- BENCHMARK_SCORE < 0.50 → **BLOCK** — do NOT proceed to TEST; return to BUILD with a prioritized fix list (maximum 2 automatic retries, then escalate to human). On retry, re-run only the deterministic layers (B0 + B1).
+- BENCHMARK_SCORE 0.50-0.70 → **WARN** — proceed to TEST with an advisory flag (per-layer scores + top failures passed to the judges)
 - BENCHMARK_SCORE > 0.70 → **PASS** — artifact meets benchmark standards; ready for TEST
 
 Record `BENCHMARK_SCORE` for downstream consumption by `/bto-test` and `/bto`.
@@ -295,8 +313,8 @@ The `BENCHMARK_SCORE` is available to downstream commands:
 - Layer B2 uses haiku only — NEVER use sonnet or opus for consistency probes
 - Agent tool is REQUIRED for Layer B2 — do not run probes sequentially
 - Golden samples from `references/golden-samples.md` are the authoritative structural reference
-- If golden sample does not exist for the detected type → skip B0, set B0 = 0.50 (neutral), and log a warning
+- If golden sample does not exist for the detected type → skip B0 entirely and renormalize the remaining weights (B1/B2/B3) per the module's Partial Evaluation rule; log a warning
 - Report `BENCHMARK_SCORE` explicitly so downstream commands can consume it
-- BLOCK verdict at B0 or B1 halts the benchmark — do NOT proceed to subsequent layers
-- Layer B3 bloat detection uses golden sample proportions — without a golden sample, use absolute thresholds only
+- The gate is on the aggregate `BENCHMARK_SCORE` only (Step 8) — there are no per-layer BLOCK gates at B0 or B1
+- Layer B3 bloat detection is based on each section's size relative to the artifact's own average section size (no golden sample required)
 - If "verbose" is in $ARGUMENTS, show expanded per-section diagnostics for all layers without prompting

package/templates/.claude/commands/bto-optimize.md

@@ -206,3 +206,11 @@ This command is also invoked internally by `/bto` as the final step (OPTIMIZE ph
 - Always save a `.pre-optimize.bak` backup before overwriting original
 - Round 3 evaluation must use Layer 2 (sonnet judges), not Layer 1 (haiku)
 - Preserve original artifact intent — optimization changes HOW, not WHAT
+
+## Rigorous validation (recommended when `dz` ≥ 0.3.119)
+
+For a reproducible, budget-capped run that resists judge-gaming, follow the **"Rigorous validation (hold-out +
+no-regress)"** protocol in `skills/bto/modules/optimize.md`: it splits scenarios into a tuning set and a held-out
+set, tunes candidates on the tune set, and accepts a winner ONLY if it lifts the weakest dimension on the UNSEEN
+holdout without regressing the others (`dz bto-optimize --split/--plan/--select`). Prose-only, diff-confirmed,
+never auto-written. Absent `dz`, the heuristic evolutionary loop runs unchanged.

package/templates/.claude/skills/bto/modules/optimize.md

@@ -199,3 +199,46 @@ Stop optimization immediately if:
 2. Critical structural checks (Layer 0) fail on any variant
 3. Artifact semantics change fundamentally
 4. User requests stop
+
+---
+
+## Rigorous validation (hold-out + no-regress) — `dz bto-optimize` (when `dz` ≥ 0.3.119)
+
+The evolutionary loop above SELECTS the highest score on the **same** eval it tuned on — which lets a variant
+that flatters the judge panel (verbosity, buzzwords) win even if it doesn't help real users (**Goodhart**). When
+the `dz` CLI is available, delegate the deterministic validation steps to it so acceptance is gated on **unseen**
+scenarios with a **no-regress** guard and a **hard budget cap**. This strengthens the loop; it does not replace it.
+
+**Scope (Phase-1):** only the **directive prose** of `SKILL.md` is mutated — the "when to activate" block + the
+core instruction. Frontmatter, section headings, and examples are OFF-LIMITS (`--scope-check` rejects a candidate
+that touches them). Never auto-write; the human confirms the diff.
+
+Protocol (folds into the steps above):
+
+1. **Split** the BTO scenario ids into a tuning set and a held-out set (deterministic):
+   ```bash
+   dz bto-optimize --split --scenarios @scenarios.json --holdout 0.34   # → { tune:[...], holdout:[...] }
+   ```
+2. **Plan the budget** and respect the hard cap (the engine trims candidates/rounds to fit and reports it):
+   ```bash
+   dz bto-optimize --plan --candidates 5 --rounds 1 --tune <#tune> --holdout <#holdout> --max 24
+   ```
+   Never run more judge passes than the printed plan.
+3. **Tune** — generate the K prose candidates (existing strategies) and score each via the judge panel on the
+   **tune** scenarios ONLY. Record per-candidate per-dimension scores.
+4. **Validate** — score the top tune candidate(s) on the **holdout** scenarios (unseen).
+5. **Select** — the engine accepts a candidate ONLY if, on the holdout, the weakest dimension improves AND no
+   other dimension / the aggregate regresses (beyond `--tolerance`, default 0):
+   ```bash
+   dz bto-optimize --select --baseline @baseline.json --candidates @candidates.json   # → { winner|null, reason }
+   ```
+   `baseline.json` = `{ "holdout": {<DimScores>} }`; each candidate = `{ id, prose, tune:{DimScores}, holdout:{DimScores} }`.
+   A tune-winner that regresses on the holdout is **rejected** — this is the anti-gaming guarantee.
+6. **Confirm gate** — show the prose diff + the tune/holdout deltas and let the human accept before any write:
+   ```bash
+   dz bto-optimize --scope-check --original SKILL.md --candidate candidate.md   # prose-only guard
+   dz bto-optimize --diff --original SKILL.md --candidate candidate.md          # the diff to confirm
+   ```
+
+Grounded in dspy.ts MIPROv2 (propose → minibatch-tune → **validate on held-out** → best). Absent `dz` ⇒ fall
+back to the heuristic loop above (unchanged).