@dzhechkov/harness-cli 0.3.224 → 0.3.225

package/README.md

@@ -431,7 +431,7 @@ Get the whole set with `dz init --target claude-code --preset meta`, or pick one
 
 > **A skill and its npx toolkit are not duplicates — they're a graduation.** Several skills (e.g. `feature-adr`, `design-thinking`) exist BOTH as a skill inside a `dz` preset AND as a standalone `npx` package. The preset's SKILL.md is **fully functional on its own** (the whole methodology — modules + references — travels with it, and it auto-activates by description), and it's the only way to compile that capability to the **non-Claude platforms** (Codex/OpenCode/Hermes/OpenClaude) via `dz`. The npx package adds **project-level runtime governance** around the same skill: a slash command, governance rules, a context shard, and (for feature-adr) reward-learning + `/harvest`. So: pick the **skill/preset** for a working capability across platforms; pick the **npx toolkit** when you want it as a governed, command-driven fixture of one project.
 
-## All Commands (50)
+## All Commands (51)
 
 ```
 dz setup             --target <name> [--preset <name>] [--select id,id,...] [--skills-dir <dir>] [--memory agentdb] [--no-memory] [--no-hooks] [--install-driver] [--force]
@@ -484,6 +484,7 @@ dz retro             [transcript] [--json] [--threshold N] [--no-teach] [--insta
 dz feature-adr-setup [--plan] [--from-spec <f>] [--apply]   # guided project onboarding engine (behind the `configure-feature-adr` skill): --plan shows which docs exist/missing; --from-spec scaffolds vision/map/testing/project-skills (propose; --apply writes; augment-never-clobber)
 dz challenge --plan <plan.md> [--json] [--context-only] [--author <model>]   # adversarial plan-gate (behind the `challenge-panel` skill): assemble a WIDE context pack (plan + vision + testing + map + degradations) + the fixed C1-C8 "break it" brief for a FRESH adversary (≠ plan author); advisory, never blocks
 dz routing [--stage <s>] [--json]   # inspect the learned cost-optimal routing store: what `args.models.<stage>='auto-cost'` believes per (stage, complexity-tier, model) — gated attempts/successes/rate (feeds feature-adr model selection)
+dz bto-optimize --split|--plan|--select|--scope-check|--diff [--json]   # deterministic engine behind /bto-optimize: hold-out split + hard-capped budget + no-regress-on-holdout winner selection (defeats judge-gaming); prose-only, diff-confirmed, never auto-writes
 dz publish           [--filter <name>] [--bump-only] [--claim-check <off|warn|error>]   (dry-run by default; pass --yes/--confirm to go live; claim-check gate defaults to warn — surfaces README claim findings, never blocks)
 dz auto-canonicalize --source <github-url> --pack <skills-pack>
 dz sync-upstream     [--package <dir>] [--list] [--all]
@@ -1490,6 +1491,30 @@ the coder** (a model never self-QEs), and the whole thing is **opt-in** — with
 pipeline is byte-identical and the store is never touched. Layered under the usage-adaptive override (a session
 near its limit still switches to Codex first). Storage is a plain JSON file — zero native deps.
 
+### `dz bto-optimize` — when you want to improve a skill WITHOUT gaming its own benchmark
+
+The BTO pack already grades a skill (`/bto`) and can evolve it (`/bto-optimize`) — but the evolutionary loop
+picks the highest score on the **same** eval it tuned on, so a variant that flatters the LLM judge panel
+(verbosity, buzzwords) can win even if it doesn't help real users (**Goodhart's law**). `dz bto-optimize` is the
+deterministic engine the `/bto-optimize` skill delegates to so acceptance is gated on **unseen** scenarios:
+```bash
+dz bto-optimize --split --scenarios @scenarios.json --holdout 0.34          # deterministic tune/holdout split
+dz bto-optimize --plan --candidates 5 --rounds 1 --tune 4 --holdout 2 --max 24  # budget plan (trims to the cap)
+dz bto-optimize --select --baseline @baseline.json --candidates @cands.json  # accept only on holdout no-regress
+```
+```
+budget plan: 5 candidates × 1 round(s) → 20 tune + 4 holdout = 24 judge run(s) (cap 24, within cap)
+✓ winner: real — accepted on holdout: CORRECTNESS +2.00, aggregate +0.40, no regression
+✗ no winner — gamer: lifted CORRECTNESS but regressed DEPTH on holdout
+```
+A candidate is accepted **only** if it lifts the weakest dimension (METHODOLOGY/DEPTH/CORRECTNESS/USABILITY/
+ROBUSTNESS) on the **held-out** scenarios **without regressing** the others — a variant that wins on the tuning
+set but regresses on the holdout is rejected. Three guarantees: only the **directive prose** is mutated
+(frontmatter + headings are off-limits, `--scope-check`), the budget has a **hard cap** (the plan trims to fit
+and never lies about the run count), and the engine **never auto-writes** — it renders a diff you confirm.
+Grounded in dspy.ts's MIPROv2 (propose → tune → validate-on-held-out). In the skill: **"прогони bto-optimize с
+hold-out"** / **"optimize this skill with hold-out validation"**.
+
 ## Recipes & FAQ
 
 Scenario → exact command → how to verify → gotcha. **Three gotchas worth calling out up front:**

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkOH,2EAA2E;AAC3E,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAk8ID,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAE,KAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAoI5E"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAyOH,2EAA2E;AAC3E,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAmhJD,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAE,KAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAsI5E"}

package/dist/cli.js

@@ -9,7 +9,7 @@ import { fileURLToPath } from 'node:url';
 import { execSync } from 'node:child_process';
 import { homedir } from 'node:os';
 import { createRequire } from 'node:module';
-import { createSkill, getSkillInfo, getWorkflow, isTargetName, listSkills, runDoctor, runInit, benchmarkSkill, benchmarkSkills, scanMcp, reconcileCapabilities, RECONCILE_BANNER, buildRegistry, discoverSkillPackDirs, checkUpstream, compareSkills, checkAllUpstream, sweepSkillDrift, syncCanonicalSkill, checkUpgrades, discoverPackages, discoverSourcePackages, fetchAllDownloads, filterByCategory, pretrain, recommend, generatePlugin, publishPackages, runSetup, runMigrate, searchRegistry, runSync, runVerify, runInitAgentsMd, runInitGeminiMd, TARGET_NAMES, WORKFLOW_NAMES, importEcc, recordPattern, resolveLearningBackend, storeStats, consolidateSessions, pruneNoisePatterns, lessonDeltaReport, removePatternsByIds, snapshotStore, recallHybrid, teachGuard, mirrorPatternsToVector, vectorMirrorEnabled, vectorTierStatus, resolveVectorEngine, reindexVectorStore, harmonizeVectorStore, importRvfCheckpoint, statuslineData, writeFeatureAdrState, computeUsage, deriveUsageCalibration, normalizeClaudeUsageModelKey, readUsageLimits, claimCheck, summarize, queryBookKnowledge, loadStorePatternsSync, patternRecordId, loadStoreRecords, recordToPattern, bundleSkills, brainHome, listBrain, promoteProjectToBrain, updateBrainSource, queryBrain, groundPrompt, expandKu, reindexBrainVectors, buildPrimer, exportBrainSlice, importBrainSlice, registerKusToBrain, RECALL_USAGE_LOG_RELATIVE, RECALL_USAGE_LOG_MAX_BYTES, parseRecallUsageLog, buildRecallUsageReport, buildManifest, buildSbom, resolveTrustRoot, decideVerifyPolicy, decideProvenance, isInsideTree, signManifest, verifyManifest, assertKeyOutsideTree, decidePublishGate, MANIFEST_NAME, SBOM_NAME, buildArchitectureMap, renderMapHuman, findArchitectureDrift, renderDriftReport, scanWorkspacePackages, loadSubsystemManifest, loadProductVision, checkFeatureAgainstArchitecture, renderArchCheck, planProjectSkills, guidanceForStage, renderInjectionReport, analyzeCorpus, renderRakeReport, renderCriticSection, rakeAsLesson, rakeReward, DEFAULT_RAKE_THRESHOLDS, streamSessionEvents, findLatestTranscript, detectProcessRakes, buildRetro, renderRetro, retroLessonText, PROCESS_SIGNATURES, RETRO_DOMAIN, scanForSetup, buildSetupPlan, scaffoldFromSpec, renderScaffoldPreview, readExistingForScaffold, assembleChallengeContext, buildChallengeBrief, pickAdversaryModel, CHALLENGE_QUESTIONS, loadOutcomes, renderOutcomes, statsForKey, selectAutoCost, recordProvisional, finalizeOutcome, COST_LADDER, } from '@dzhechkov/harness-core';
+import { createSkill, getSkillInfo, getWorkflow, isTargetName, listSkills, runDoctor, runInit, benchmarkSkill, benchmarkSkills, scanMcp, reconcileCapabilities, RECONCILE_BANNER, buildRegistry, discoverSkillPackDirs, checkUpstream, compareSkills, checkAllUpstream, sweepSkillDrift, syncCanonicalSkill, checkUpgrades, discoverPackages, discoverSourcePackages, fetchAllDownloads, filterByCategory, pretrain, recommend, generatePlugin, publishPackages, runSetup, runMigrate, searchRegistry, runSync, runVerify, runInitAgentsMd, runInitGeminiMd, TARGET_NAMES, WORKFLOW_NAMES, importEcc, recordPattern, resolveLearningBackend, storeStats, consolidateSessions, pruneNoisePatterns, lessonDeltaReport, removePatternsByIds, snapshotStore, recallHybrid, teachGuard, mirrorPatternsToVector, vectorMirrorEnabled, vectorTierStatus, resolveVectorEngine, reindexVectorStore, harmonizeVectorStore, importRvfCheckpoint, statuslineData, writeFeatureAdrState, computeUsage, deriveUsageCalibration, normalizeClaudeUsageModelKey, readUsageLimits, claimCheck, summarize, queryBookKnowledge, loadStorePatternsSync, patternRecordId, loadStoreRecords, recordToPattern, bundleSkills, brainHome, listBrain, promoteProjectToBrain, updateBrainSource, queryBrain, groundPrompt, expandKu, reindexBrainVectors, buildPrimer, exportBrainSlice, importBrainSlice, registerKusToBrain, RECALL_USAGE_LOG_RELATIVE, RECALL_USAGE_LOG_MAX_BYTES, parseRecallUsageLog, buildRecallUsageReport, buildManifest, buildSbom, resolveTrustRoot, decideVerifyPolicy, decideProvenance, isInsideTree, signManifest, verifyManifest, assertKeyOutsideTree, decidePublishGate, MANIFEST_NAME, SBOM_NAME, buildArchitectureMap, renderMapHuman, findArchitectureDrift, renderDriftReport, scanWorkspacePackages, loadSubsystemManifest, loadProductVision, checkFeatureAgainstArchitecture, renderArchCheck, planProjectSkills, guidanceForStage, renderInjectionReport, analyzeCorpus, renderRakeReport, renderCriticSection, rakeAsLesson, rakeReward, DEFAULT_RAKE_THRESHOLDS, streamSessionEvents, findLatestTranscript, detectProcessRakes, buildRetro, renderRetro, retroLessonText, PROCESS_SIGNATURES, RETRO_DOMAIN, scanForSetup, buildSetupPlan, scaffoldFromSpec, renderScaffoldPreview, readExistingForScaffold, assembleChallengeContext, buildChallengeBrief, pickAdversaryModel, CHALLENGE_QUESTIONS, loadOutcomes, renderOutcomes, statsForKey, selectAutoCost, recordProvisional, finalizeOutcome, COST_LADDER, splitScenarios, budgetPlan, selectWinner, proseScopeOk, renderProseDiff, readScenarioIds, DEFAULT_MAX_JUDGE_RUNS, } from '@dzhechkov/harness-core';
 import { getPreset, PRESET_NAMES } from '@dzhechkov/harness-presets';
 import { scanGitHub, analyzeRepo, generateReport, deepAnalyze, scanAllSources, ScoutMemory } from '@dzhechkov/scout';
 const USAGE = `dz - DZ cross-platform harness CLI
@@ -4316,6 +4316,107 @@ function cmdRouting(options, flags, cwd, write) {
     write(renderOutcomes(store, stage));
     return 0;
 }
+/**
+ * `dz bto-optimize` — the deterministic engine BEHIND the `/bto-optimize` skill (feature bto-optimize-holdout).
+ * Adds dspy-MIPROv2 rigor the current evolutionary loop lacks: a hold-out split, a hard budget cap, and a
+ * no-regress-on-holdout winner selector. NOT a rival command — the skill delegates these steps; candidate-prose
+ * generation + judge scoring stay skill-side. `--json` on every subcommand.
+ *   --split --scenarios <csv|@file> [--holdout <r>]                 deterministic tune/holdout split
+ *   --plan --candidates K --rounds R --tune N --holdout M [--max C]  budget plan (trims to the cap)
+ *   --select --baseline <@json> --candidates <@json> [--tolerance t] accept only on holdout no-regress
+ *   --scope-check --original <f> --candidate <f>                     prose-only guard
+ *   --diff --original <f> --candidate <f>                            the prose diff to confirm
+ */
+function cmdBtoOptimize(options, flags, cwd, write) {
+    const json = flags.has('json');
+    const emit = (o) => { write(JSON.stringify(o, null, json ? 2 : 0)); return 0; };
+    const readContained = (rel) => {
+        // Containment (QE #traversal): an @file must stay under cwd — `@../../etc/passwd` must not read outside.
+        const rootAbs = resolve(cwd);
+        const abs = resolve(rootAbs, rel);
+        if (abs !== rootAbs && !abs.startsWith(rootAbs + sep))
+            throw new Error(`path escapes the working directory: ${rel}`);
+        return readFileSync(abs, 'utf-8');
+    };
+    const readJson = (spec) => {
+        if (spec === undefined)
+            return undefined;
+        const raw = spec.startsWith('@') ? readContained(spec.slice(1)) : spec;
+        return JSON.parse(raw);
+    };
+    try {
+        if (flags.has('split')) {
+            const sc = options.get('scenarios');
+            if (sc === undefined) {
+                write('dz bto-optimize --split needs --scenarios <csv|@file>');
+                return 1;
+            }
+            const ids = sc.startsWith('@') ? readScenarioIds(resolve(cwd, sc.slice(1))) : sc.split(',').map((s) => s.trim()).filter(Boolean);
+            const ratio = Number.parseFloat(options.get('holdout') ?? '');
+            const split = splitScenarios(ids, Number.isFinite(ratio) ? ratio : undefined);
+            if (json)
+                return emit(split);
+            write(`tune (${split.tune.length}): ${split.tune.join(', ')}`);
+            write(`holdout (${split.holdout.length}): ${split.holdout.join(', ')}`);
+            return 0;
+        }
+        if (flags.has('plan')) {
+            const n = (k, d) => { const v = Number.parseInt(options.get(k) ?? '', 10); return Number.isFinite(v) ? v : d; };
+            const max = Number.parseInt(options.get('max') ?? '', 10);
+            const plan = budgetPlan({ candidates: n('candidates', 5), rounds: n('rounds', 1), tuneCount: n('tune', 3), holdoutCount: n('holdout', 2) }, Number.isFinite(max) ? max : DEFAULT_MAX_JUDGE_RUNS);
+            if (json)
+                return emit(plan);
+            write(`budget plan: ${plan.candidates} candidates × ${plan.rounds} round(s) → ${plan.tuneRuns} tune + ${plan.holdoutRuns} holdout = ${plan.totalRuns} judge run(s) (cap ${plan.cap}, ${plan.withinCap ? 'within cap' : 'OVER CAP'})`);
+            if (plan.trimmed)
+                write(`  trimmed to fit: ${plan.trimmed}`);
+            return 0;
+        }
+        if (flags.has('select')) {
+            const baseline = readJson(options.get('baseline'));
+            const candidates = readJson(options.get('candidates'));
+            if (!baseline || !Array.isArray(candidates)) {
+                write('dz bto-optimize --select needs --baseline <@json {holdout}> and --candidates <@json [..]>');
+                return 1;
+            }
+            const tol = Number.parseFloat(options.get('tolerance') ?? '');
+            const result = selectWinner(baseline, candidates, Number.isFinite(tol) ? { tolerance: tol } : {});
+            if (json)
+                return emit(result);
+            write(result.winner ? `✓ winner: ${result.winner} — ${result.reason}` : `✗ no winner — ${result.reason}`);
+            return 0;
+        }
+        if (flags.has('scope-check') || flags.has('diff')) {
+            const o = options.get('original');
+            const c = options.get('candidate');
+            if (o === undefined || c === undefined) {
+                write('needs --original <file> and --candidate <file>');
+                return 1;
+            }
+            const origText = readContained(o);
+            const candText = readContained(c);
+            if (flags.has('scope-check')) {
+                const r = proseScopeOk(origText, candText);
+                if (json)
+                    return emit(r);
+                write(r.ok ? `✓ prose-only: ${r.reason}` : `✗ out of scope: ${r.reason}`);
+                return r.ok ? 0 : 1;
+            }
+            write(renderProseDiff(origText, candText));
+            return 0;
+        }
+        write('dz bto-optimize: pass --split | --plan | --select | --scope-check | --diff (see docs)');
+        return 1;
+    }
+    catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        if (json) {
+            write(JSON.stringify({ error: msg }));
+            return 1;
+        }
+        write(`dz bto-optimize: ${msg}`);
+        return 1;
+    }
+}
 function cmdStats(cwd, write) {
     const baseDir = join(cwd, 'packages', '@dzhechkov');
     if (!existsSync(baseDir)) {
@@ -4577,6 +4678,8 @@ export async function runCli(argv, io = {}) {
                 return cmdChallenge(options, flags, cwd, write);
             case 'routing':
                 return cmdRouting(options, flags, cwd, write);
+            case 'bto-optimize':
+                return cmdBtoOptimize(options, flags, cwd, write);
             case 'dashboard':
                 return cmdDashboard(cwd, write);
             case 'roam':