@dzhechkov/keysarium 1.5.1 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +0 -0
- package/package.json +2 -2
- package/src/cli.js +10 -1
- package/src/commands/init.js +13 -5
- package/src/commands/update.js +5 -1
- package/src/utils.js +34 -0
- package/templates/.claude/commands/bto-benchmark.md +0 -302
package/bin/cli.js
CHANGED
|
File without changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dzhechkov/keysarium",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.2",
|
|
4
4
|
"description": "AI Case Research Toolkit for Claude Code — full 7-phase pipeline with feature-adr, dream cycles, background workers, multi-platform support, reward learning, and Cloud.ru AI Factory mapping (--ai-factory flag)",
|
|
5
5
|
"main": "src/cli.js",
|
|
6
6
|
"bin": {
|
|
@@ -50,4 +50,4 @@
|
|
|
50
50
|
"publishConfig": {
|
|
51
51
|
"access": "public"
|
|
52
52
|
}
|
|
53
|
-
}
|
|
53
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -91,8 +91,17 @@ function parseArgs(argv) {
|
|
|
91
91
|
command = 'version';
|
|
92
92
|
break;
|
|
93
93
|
default:
|
|
94
|
-
if (
|
|
94
|
+
if (arg.startsWith('-')) {
|
|
95
|
+
// Reject unknown options loudly — a typo like --dry-rum or --froce must
|
|
96
|
+
// never silently downgrade behavior (real install instead of dry-run, or
|
|
97
|
+
// an unguarded destructive command).
|
|
98
|
+
error(`Unknown option: ${arg}`);
|
|
99
|
+
process.exit(1);
|
|
100
|
+
} else if (command === null) {
|
|
95
101
|
command = arg;
|
|
102
|
+
} else {
|
|
103
|
+
error(`Unexpected argument: ${arg}`);
|
|
104
|
+
process.exit(1);
|
|
96
105
|
}
|
|
97
106
|
break;
|
|
98
107
|
}
|
package/src/commands/init.js
CHANGED
|
@@ -5,7 +5,7 @@ const path = require('path');
|
|
|
5
5
|
const {
|
|
6
6
|
green, yellow, cyan, bold, dim,
|
|
7
7
|
info, success, warn, error: logError, step,
|
|
8
|
-
copyDirRecursive, fileExists, readJSON,
|
|
8
|
+
copyDirRecursive, copyDirTracked, fileExists, readJSON,
|
|
9
9
|
ensureDir, getRelativePaths,
|
|
10
10
|
createManifest, writeManifest, getTemplatesDir,
|
|
11
11
|
COMPONENTS, MANIFEST_FILE,
|
|
@@ -41,7 +41,7 @@ function resolveComponents(options) {
|
|
|
41
41
|
* Copy a single component from templates to the target directory.
|
|
42
42
|
* Returns array of relative file paths that were installed.
|
|
43
43
|
*/
|
|
44
|
-
function installComponent(comp, templatesDir, targetDir) {
|
|
44
|
+
function installComponent(comp, templatesDir, targetDir, force) {
|
|
45
45
|
const src = path.join(templatesDir, comp.src);
|
|
46
46
|
const dest = path.join(targetDir, comp.src);
|
|
47
47
|
|
|
@@ -51,13 +51,21 @@ function installComponent(comp, templatesDir, targetDir) {
|
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
if (comp.isFile) {
|
|
54
|
+
if (fileExists(dest) && !force) {
|
|
55
|
+
// Preserve a user-modified file (e.g. their own CLAUDE.md / settings.json)
|
|
56
|
+
// when re-installing without --force. It is still manifest-tracked because
|
|
57
|
+
// it originates from the template.
|
|
58
|
+
return [comp.src];
|
|
59
|
+
}
|
|
54
60
|
ensureDir(path.dirname(dest));
|
|
55
61
|
fs.copyFileSync(src, dest);
|
|
56
62
|
return [comp.src];
|
|
57
63
|
}
|
|
58
64
|
|
|
59
|
-
|
|
60
|
-
|
|
65
|
+
// Track files via the TEMPLATE source — never a scan of the destination —
|
|
66
|
+
// so user-created files under dest are neither recorded nor (later) removed.
|
|
67
|
+
const { copied, skipped } = copyDirTracked(src, dest, force);
|
|
68
|
+
return [...copied, ...skipped].map((rel) => path.join(comp.src, rel));
|
|
61
69
|
}
|
|
62
70
|
|
|
63
71
|
// ---------------------------------------------------------------------------
|
|
@@ -118,7 +126,7 @@ async function run(options) {
|
|
|
118
126
|
const comp = COMPONENTS[key];
|
|
119
127
|
step(i + 1, totalComponents, `Installing ${comp.label}...`);
|
|
120
128
|
|
|
121
|
-
const files = installComponent(comp, templatesDir, targetDir);
|
|
129
|
+
const files = installComponent(comp, templatesDir, targetDir, force);
|
|
122
130
|
installedFiles.push(...files);
|
|
123
131
|
}
|
|
124
132
|
|
package/src/commands/update.js
CHANGED
|
@@ -159,12 +159,16 @@ async function run(options) {
|
|
|
159
159
|
const comp = COMPONENTS[key];
|
|
160
160
|
if (!comp) continue;
|
|
161
161
|
const destPath = path.join(targetDir, comp.src);
|
|
162
|
+
const srcPath = path.join(templatesDir, comp.src);
|
|
162
163
|
if (comp.isFile) {
|
|
163
164
|
if (fileExists(destPath)) {
|
|
164
165
|
allFiles.push(comp.src);
|
|
165
166
|
}
|
|
166
167
|
} else if (fileExists(destPath)) {
|
|
167
|
-
|
|
168
|
+
// Record files from the TEMPLATE source, not a scan of the destination —
|
|
169
|
+
// otherwise user-created files in dest get adopted into the manifest and
|
|
170
|
+
// a later `remove` would delete them.
|
|
171
|
+
const paths = getRelativePaths(fileExists(srcPath) ? srcPath : destPath);
|
|
168
172
|
allFiles.push(...paths.map((rel) => path.join(comp.src, rel)));
|
|
169
173
|
}
|
|
170
174
|
}
|
package/src/utils.js
CHANGED
|
@@ -64,6 +64,39 @@ function copyDirRecursive(src, dest) {
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Copy from a TEMPLATE source into dest, tracking which files were written.
|
|
69
|
+
* Walks the SOURCE tree (never the destination), so only template-originated
|
|
70
|
+
* files are ever recorded — a user-created file in dest with no template
|
|
71
|
+
* counterpart is never touched and never returned (prevents `remove` from
|
|
72
|
+
* deleting user data). When `force` is false, a pre-existing destination file
|
|
73
|
+
* is PRESERVED (skipped) rather than clobbered (protects user-modified files
|
|
74
|
+
* on a re-install where the manifest was lost).
|
|
75
|
+
*
|
|
76
|
+
* @returns {{copied: string[], skipped: string[]}} relative paths (to dest root)
|
|
77
|
+
*/
|
|
78
|
+
function copyDirTracked(src, dest, force, rel = '', acc = { copied: [], skipped: [] }) {
|
|
79
|
+
const stat = fs.statSync(src);
|
|
80
|
+
if (stat.isFile()) {
|
|
81
|
+
if (fileExists(dest) && !force) {
|
|
82
|
+
acc.skipped.push(rel);
|
|
83
|
+
} else {
|
|
84
|
+
ensureDir(path.dirname(dest));
|
|
85
|
+
fs.copyFileSync(src, dest);
|
|
86
|
+
acc.copied.push(rel);
|
|
87
|
+
}
|
|
88
|
+
return acc;
|
|
89
|
+
}
|
|
90
|
+
if (stat.isDirectory()) {
|
|
91
|
+
ensureDir(dest);
|
|
92
|
+
for (const entry of fs.readdirSync(src)) {
|
|
93
|
+
copyDirTracked(path.join(src, entry), path.join(dest, entry), force,
|
|
94
|
+
rel ? path.join(rel, entry) : entry, acc);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return acc;
|
|
98
|
+
}
|
|
99
|
+
|
|
67
100
|
/**
|
|
68
101
|
* Returns true if the path exists (file or directory).
|
|
69
102
|
*/
|
|
@@ -254,6 +287,7 @@ module.exports = {
|
|
|
254
287
|
|
|
255
288
|
// File operations
|
|
256
289
|
copyDirRecursive,
|
|
290
|
+
copyDirTracked,
|
|
257
291
|
fileExists,
|
|
258
292
|
readJSON,
|
|
259
293
|
writeJSON,
|
|
@@ -1,302 +0,0 @@
|
|
|
1
|
-
# /bto-benchmark — Deterministic Benchmarking of a Skill or Command
|
|
2
|
-
|
|
3
|
-
## Usage
|
|
4
|
-
```
|
|
5
|
-
/bto-benchmark [path to skill directory or file]
|
|
6
|
-
```
|
|
7
|
-
|
|
8
|
-
## Parameters
|
|
9
|
-
- $ARGUMENTS — Path to a skill directory (`.claude/skills/<name>/`), a command file (`.claude/commands/<name>.md`), a rule file (`.claude/rules/<name>.md`), or any structured artifact. Optionally include "verbose" to show expanded per-section diagnostics for all layers.
|
|
10
|
-
|
|
11
|
-
## Protocol
|
|
12
|
-
|
|
13
|
-
### Step 1: Load Skill and Module
|
|
14
|
-
|
|
15
|
-
Read `.claude/skills/bto/SKILL.md`
|
|
16
|
-
Read `.claude/skills/bto/modules/benchmark.md`
|
|
17
|
-
Read `.claude/skills/bto/references/golden-samples.md`
|
|
18
|
-
|
|
19
|
-
### Step 2: Validate Input
|
|
20
|
-
|
|
21
|
-
If $ARGUMENTS is empty:
|
|
22
|
-
- Ask: "Provide a path to the skill directory or artifact file you want to benchmark."
|
|
23
|
-
- Stop and wait.
|
|
24
|
-
|
|
25
|
-
Resolve the artifact path from $ARGUMENTS:
|
|
26
|
-
- Strip any trailing slash
|
|
27
|
-
- If the path points to a directory → look for `SKILL.md` inside it as the primary file, but include all files in the directory for context
|
|
28
|
-
- If the path points to a file → use that file directly
|
|
29
|
-
- If the path does not exist → report "Path not found: [path]" and stop
|
|
30
|
-
|
|
31
|
-
### Step 3: Detect Artifact Type
|
|
32
|
-
|
|
33
|
-
Auto-detect from path pattern:
|
|
34
|
-
|
|
35
|
-
| Path Pattern | Detected Type |
|
|
36
|
-
|-------------|--------------|
|
|
37
|
-
| `.claude/skills/*/SKILL.md` or `.claude/skills/*/` | skill |
|
|
38
|
-
| `.claude/commands/*.md` | command |
|
|
39
|
-
| `.claude/rules/*.md` | rule |
|
|
40
|
-
| `.claude/agents/*.md` | agent |
|
|
41
|
-
| `researches/**/*.md` | research artifact |
|
|
42
|
-
|
|
43
|
-
If type cannot be determined from path → infer from file content structure.
|
|
44
|
-
|
|
45
|
-
### Step 4: Layer B0 — Golden Sample Comparison (Deterministic)
|
|
46
|
-
|
|
47
|
-
**Purpose:** Compare artifact structure against the canonical golden sample for its type. Zero LLM cost.
|
|
48
|
-
|
|
49
|
-
Load golden samples for the detected type from `references/golden-samples.md`.
|
|
50
|
-
|
|
51
|
-
**Procedure:**
|
|
52
|
-
|
|
53
|
-
1. Parse the artifact into a section tree (heading hierarchy, section lengths, ordering)
|
|
54
|
-
2. Parse the matching golden sample into the same section tree format
|
|
55
|
-
3. Compare across three structural axes:
|
|
56
|
-
|
|
57
|
-
**Axis 1 — Section Coverage:**
|
|
58
|
-
For each required section in the golden sample, check if the artifact contains a matching section (by heading text or semantic equivalent).
|
|
59
|
-
- `section_coverage = matched_sections / total_golden_sections`
|
|
60
|
-
|
|
61
|
-
**Axis 2 — Ordering Score:**
|
|
62
|
-
Compare the ordering of matched sections against the golden sample ordering.
|
|
63
|
-
- Use normalized Kendall tau distance: `ordering_score = 1 - (inversions / max_inversions)`
|
|
64
|
-
- If fewer than 3 matched sections → ordering_score = 1.0 (too few to meaningfully compare)
|
|
65
|
-
|
|
66
|
-
**Axis 3 — Proportion Score:**
|
|
67
|
-
For each matched section, compute the ratio of artifact section length to golden sample section length.
|
|
68
|
-
- `proportion_score = 1 - mean(|log(artifact_len / golden_len)|)` clamped to [0, 1]
|
|
69
|
-
- Measures whether sections are proportionally sized (not too bloated, not too thin)
|
|
70
|
-
|
|
71
|
-
**Display per-section MATCH/MISS table:**
|
|
72
|
-
|
|
73
|
-
```
|
|
74
|
-
Golden Section Status Artifact Section Size Ratio
|
|
75
|
-
─────────────────────────── ──────── ──────────────────────────── ──────────
|
|
76
|
-
## Overview MATCH ## Overview 1.2x
|
|
77
|
-
## Protocol MATCH ## Protocol 0.8x
|
|
78
|
-
## Anti-Patterns MATCH ## Anti-Patterns 1.0x
|
|
79
|
-
## Dependencies MISS — —
|
|
80
|
-
## Quick Start MATCH ## Quick Start 1.5x
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
**Golden Similarity Score:**
|
|
84
|
-
```
|
|
85
|
-
B0 = section_coverage * 0.50 + ordering_score * 0.25 + proportion_score * 0.25
|
|
86
|
-
```
|
|
87
|
-
|
|
88
|
-
**Gate:** If B0 < 0.30 → BLOCK immediately. The artifact's structure is too far from the expected form to benchmark meaningfully.
|
|
89
|
-
|
|
90
|
-
### Step 5: Layer B1 — Deterministic Test Suite
|
|
91
|
-
|
|
92
|
-
**Purpose:** Run all applicable deterministic tests for the detected artifact type. Zero LLM cost.
|
|
93
|
-
|
|
94
|
-
Execute all applicable checks from `benchmark.md` for the detected artifact type. These overlap with but extend the Layer 0 checks from `/bto-test`:
|
|
95
|
-
|
|
96
|
-
**Universal Checks (U1-U12):**
|
|
97
|
-
- U-01: File exists and is non-empty
|
|
98
|
-
- U-02: UTF-8 encoding valid
|
|
99
|
-
- U-03: Starts with level-1 heading
|
|
100
|
-
- U-04: No placeholder text (`[TODO]`, `[TBD]`, `<INSERT>`, `[PLACEHOLDER]`)
|
|
101
|
-
- U-05: No empty sections (heading with no content before next heading)
|
|
102
|
-
- U-06: Consistent heading hierarchy (no h1→h3 jumps)
|
|
103
|
-
- U-07: No broken internal cross-references
|
|
104
|
-
- U-08: File size within bounds (200B – 100KB per file)
|
|
105
|
-
- U-09: No trailing whitespace on lines
|
|
106
|
-
- U-10: Standard Markdown only (no HTML unless in code blocks)
|
|
107
|
-
- U-11: All code blocks properly closed
|
|
108
|
-
- U-12: No duplicate top-level sections
|
|
109
|
-
|
|
110
|
-
**Type-Specific Checks:**
|
|
111
|
-
- Skill: SK-01 through SK-16 (SKILL.md exists, required sections, modules/, references/, examples/, etc.)
|
|
112
|
-
- Command: CM-01 through CM-08 (Usage section, Parameters, Protocol steps, Checkpoint, Critical Rules, etc.)
|
|
113
|
-
- Rule: RL-01 through RL-06 (Table format, detection signals, fix actions, no contradictions, etc.)
|
|
114
|
-
- Agent: AG-01 through AG-04 (Model specified, isolation rules, naming convention, output format)
|
|
115
|
-
|
|
116
|
-
**Display per-test PASS/FAIL table:**
|
|
117
|
-
|
|
118
|
-
```
|
|
119
|
-
ID Check Result Note
|
|
120
|
-
─────── ────────────────────────────────── ──────── ────────────────────────────
|
|
121
|
-
U-01 File exists and is non-empty PASS 8.4 KB
|
|
122
|
-
U-02 UTF-8 encoding PASS
|
|
123
|
-
...
|
|
124
|
-
SK-01 SKILL.md exists at skill root PASS
|
|
125
|
-
SK-02 Has ## Overview section PASS
|
|
126
|
-
SK-03 Has ## Anti-Patterns section FAIL Section missing
|
|
127
|
-
...
|
|
128
|
-
```
|
|
129
|
-
|
|
130
|
-
**Test Pass Rate:**
|
|
131
|
-
```
|
|
132
|
-
B1 = passed_tests / total_applicable_tests
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
**Gate:** If B1 < 0.60 → BLOCK. Too many structural failures to proceed to consistency probing.
|
|
136
|
-
|
|
137
|
-
### Step 6: Layer B2 — Consistency Probe
|
|
138
|
-
|
|
139
|
-
**Purpose:** Measure evaluation stability by running identical prompts across multiple agents. Detects artifacts that produce inconsistent interpretations.
|
|
140
|
-
|
|
141
|
-
**Model:** haiku (cost-optimized)
|
|
142
|
-
|
|
143
|
-
**Spawn 3 parallel haiku agents with the identical evaluation prompt:**
|
|
144
|
-
|
|
145
|
-
Each agent receives the exact same prompt:
|
|
146
|
-
```
|
|
147
|
-
You are evaluating a Claude Code artifact for structural quality.
|
|
148
|
-
Rate these 4 dimensions (0.0 to 1.0 each, two decimal places):
|
|
149
|
-
1. STRUCTURE — Does the artifact follow a clear, logical organization?
|
|
150
|
-
2. CLARITY — Are instructions unambiguous and actionable?
|
|
151
|
-
3. COVERAGE — Does it address all aspects implied by its title/scope?
|
|
152
|
-
4. CONSISTENCY — Is the internal terminology and style uniform?
|
|
153
|
-
Output as JSON: {"structure": X.XX, "clarity": X.XX, "coverage": X.XX, "consistency": X.XX}
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
**Isolation:** Each agent evaluates independently. No cross-communication.
|
|
157
|
-
|
|
158
|
-
**After all 3 return, compute structural agreement:**
|
|
159
|
-
|
|
160
|
-
For each dimension:
|
|
161
|
-
```
|
|
162
|
-
agreement[dim] = 1 - (max(scores[dim]) - min(scores[dim]))
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
**Consistency Score:**
|
|
166
|
-
```
|
|
167
|
-
B2 = mean(agreement across all 4 dimensions)
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
**Interpretation:**
|
|
171
|
-
- B2 > 0.85 → HIGH consistency — the artifact is unambiguously structured
|
|
172
|
-
- B2 0.60-0.85 → MODERATE consistency — some sections are open to interpretation
|
|
173
|
-
- B2 < 0.60 → LOW consistency — the artifact is structurally ambiguous
|
|
174
|
-
|
|
175
|
-
**Display per-probe comparison:**
|
|
176
|
-
|
|
177
|
-
```
|
|
178
|
-
Dimension Probe 1 Probe 2 Probe 3 Range Agreement
|
|
179
|
-
──────────── ────────── ────────── ────────── ──────── ──────────
|
|
180
|
-
STRUCTURE 0.80 0.85 0.82 0.05 0.95
|
|
181
|
-
CLARITY 0.70 0.75 0.65 0.10 0.90
|
|
182
|
-
COVERAGE 0.60 0.70 0.55 0.15 0.85
|
|
183
|
-
CONSISTENCY 0.75 0.78 0.73 0.05 0.95
|
|
184
|
-
Mean: 0.91
|
|
185
|
-
```
|
|
186
|
-
|
|
187
|
-
No gate on B2 — consistency is informational and feeds into the aggregate score.
|
|
188
|
-
|
|
189
|
-
### Step 7: Layer B3 — Performance Metrics
|
|
190
|
-
|
|
191
|
-
**Purpose:** Quantify token efficiency and information density. Zero LLM cost.
|
|
192
|
-
|
|
193
|
-
**Metric 1 — Token Efficiency:**
|
|
194
|
-
```
|
|
195
|
-
token_efficiency = meaningful_content_tokens / total_tokens
|
|
196
|
-
```
|
|
197
|
-
Where `meaningful_content_tokens` excludes: blank lines, decorative separators, repeated boilerplate headers, and excessive whitespace. Target: > 0.70.
|
|
198
|
-
|
|
199
|
-
**Metric 2 — Information Density:**
|
|
200
|
-
```
|
|
201
|
-
information_density = unique_concepts / total_sections
|
|
202
|
-
```
|
|
203
|
-
Where `unique_concepts` = count of distinct topics, terms, or instructions introduced. Measures whether each section contributes new information vs. repeating prior content. Target: > 2.0 concepts per section.
|
|
204
|
-
|
|
205
|
-
**Metric 3 — Bloat Detection:**
|
|
206
|
-
Identify sections where:
|
|
207
|
-
- Section length > 3x the golden sample equivalent → BLOATED
|
|
208
|
-
- Section length < 0.2x the golden sample equivalent → THIN
|
|
209
|
-
- Section repeats > 30% of content from another section → REDUNDANT
|
|
210
|
-
|
|
211
|
-
```
|
|
212
|
-
bloat_score = 1 - (bloated_sections + thin_sections + redundant_sections) / total_sections
|
|
213
|
-
```
|
|
214
|
-
Clamped to [0, 1].
|
|
215
|
-
|
|
216
|
-
**Performance Score:**
|
|
217
|
-
```
|
|
218
|
-
B3 = token_efficiency * 0.35 + normalize(information_density, 0, 4) * 0.35 + bloat_score * 0.30
|
|
219
|
-
```
|
|
220
|
-
Where `normalize(x, min, max)` clamps and scales x to [0, 1].
|
|
221
|
-
|
|
222
|
-
**Display per-section metrics:**
|
|
223
|
-
|
|
224
|
-
```
|
|
225
|
-
Section Tokens Concepts Golden Ratio Flag
|
|
226
|
-
───────────────────────── ──────── ────────── ────────────── ──────────
|
|
227
|
-
## Overview 320 4 1.2x OK
|
|
228
|
-
## Protocol 1840 12 0.8x OK
|
|
229
|
-
## Anti-Patterns 580 6 2.8x BLOATED
|
|
230
|
-
## Quick Start 95 2 0.3x THIN
|
|
231
|
-
## Dependencies 210 3 1.0x OK
|
|
232
|
-
```
|
|
233
|
-
|
|
234
|
-
### Step 8: Aggregate & Gate
|
|
235
|
-
|
|
236
|
-
**BENCHMARK_SCORE formula:**
|
|
237
|
-
```
|
|
238
|
-
BENCHMARK_SCORE = B0 * 0.30 + B1 * 0.35 + B2 * 0.15 + B3 * 0.20
|
|
239
|
-
```
|
|
240
|
-
|
|
241
|
-
**Weight rationale:**
|
|
242
|
-
- B0 (Golden Comparison) at 0.30 — structural conformance is the primary benchmark signal
|
|
243
|
-
- B1 (Test Suite) at 0.35 — deterministic correctness is the most reliable signal
|
|
244
|
-
- B2 (Consistency) at 0.15 — stability matters but is a secondary signal
|
|
245
|
-
- B3 (Performance) at 0.20 — efficiency matters for production artifacts
|
|
246
|
-
|
|
247
|
-
**Gate:**
|
|
248
|
-
- BENCHMARK_SCORE < 0.50 → **BLOCK** — artifact requires significant rework before evaluation
|
|
249
|
-
- BENCHMARK_SCORE 0.50-0.70 → **WARN** — artifact has structural issues; proceed to TEST with caution
|
|
250
|
-
- BENCHMARK_SCORE > 0.70 → **PASS** — artifact meets benchmark standards; ready for TEST
|
|
251
|
-
|
|
252
|
-
Record `BENCHMARK_SCORE` for downstream consumption by `/bto-test` and `/bto`.
|
|
253
|
-
|
|
254
|
-
---
|
|
255
|
-
|
|
256
|
-
## Checkpoint
|
|
257
|
-
|
|
258
|
-
```
|
|
259
|
-
═══════════════════════════════════════════════════════
|
|
260
|
-
CHECKPOINT: BENCHMARK Complete
|
|
261
|
-
<promise>BTO_BENCHMARKED</promise>
|
|
262
|
-
Artifact: [path]
|
|
263
|
-
Type: [detected type]
|
|
264
|
-
|
|
265
|
-
Layer B0 (Golden): X.XX — section coverage, ordering, proportions
|
|
266
|
-
Layer B1 (Test Suite): X.XX — Y/Z tests passed
|
|
267
|
-
Layer B2 (Consistency): X.XX — agreement across 3 probes
|
|
268
|
-
Layer B3 (Performance): X.XX — token efficiency, density, bloat
|
|
269
|
-
|
|
270
|
-
BENCHMARK SCORE: X.XX [PASS / WARN / BLOCK]
|
|
271
|
-
|
|
272
|
-
• "ок" — proceed to TEST
|
|
273
|
-
• "покажи детали [B0/B1/B2/B3]" — expand layer details
|
|
274
|
-
• "пропусти тест" — skip TEST (if PASS)
|
|
275
|
-
═══════════════════════════════════════════════════════
|
|
276
|
-
```
|
|
277
|
-
|
|
278
|
-
Wait for user confirmation.
|
|
279
|
-
|
|
280
|
-
---
|
|
281
|
-
|
|
282
|
-
## Modular Usage
|
|
283
|
-
|
|
284
|
-
This command is also invoked internally by:
|
|
285
|
-
- `/bto` — as a pre-flight step before TEST phase (if benchmark module is loaded)
|
|
286
|
-
- `/bto-test` — optionally as a prerequisite gate (if user requests `benchmark` flag)
|
|
287
|
-
|
|
288
|
-
The `BENCHMARK_SCORE` is available to downstream commands:
|
|
289
|
-
- `/bto-test` may use it to calibrate Layer 1 expectations
|
|
290
|
-
- `/bto-optimize` may use it to set a structural baseline floor
|
|
291
|
-
|
|
292
|
-
## Critical Rules
|
|
293
|
-
|
|
294
|
-
- Layer B0 and B1 are fully deterministic — no LLM calls, zero cost
|
|
295
|
-
- Layer B2 uses haiku only — NEVER use sonnet or opus for consistency probes
|
|
296
|
-
- Agent tool is REQUIRED for Layer B2 — do not run probes sequentially
|
|
297
|
-
- Golden samples from `references/golden-samples.md` are the authoritative structural reference
|
|
298
|
-
- If golden sample does not exist for the detected type → skip B0, set B0 = 0.50 (neutral), and log a warning
|
|
299
|
-
- Report `BENCHMARK_SCORE` explicitly so downstream commands can consume it
|
|
300
|
-
- BLOCK verdict at B0 or B1 halts the benchmark — do NOT proceed to subsequent layers
|
|
301
|
-
- Layer B3 bloat detection uses golden sample proportions — without a golden sample, use absolute thresholds only
|
|
302
|
-
- If "verbose" is in $ARGUMENTS, show expanded per-section diagnostics for all layers without prompting
|