@dzhechkov/keysarium 1.5.1 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/cli.js CHANGED
File without changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dzhechkov/keysarium",
3
- "version": "1.5.1",
3
+ "version": "1.5.2",
4
4
  "description": "AI Case Research Toolkit for Claude Code — full 7-phase pipeline with feature-adr, dream cycles, background workers, multi-platform support, reward learning, and Cloud.ru AI Factory mapping (--ai-factory flag)",
5
5
  "main": "src/cli.js",
6
6
  "bin": {
@@ -50,4 +50,4 @@
50
50
  "publishConfig": {
51
51
  "access": "public"
52
52
  }
53
- }
53
+ }
package/src/cli.js CHANGED
@@ -91,8 +91,17 @@ function parseArgs(argv) {
91
91
  command = 'version';
92
92
  break;
93
93
  default:
94
- if (!arg.startsWith('-') && command === null) {
94
+ if (arg.startsWith('-')) {
95
+ // Reject unknown options loudly — a typo like --dry-rum or --froce must
96
+ // never silently downgrade behavior (real install instead of dry-run, or
97
+ // an unguarded destructive command).
98
+ error(`Unknown option: ${arg}`);
99
+ process.exit(1);
100
+ } else if (command === null) {
95
101
  command = arg;
102
+ } else {
103
+ error(`Unexpected argument: ${arg}`);
104
+ process.exit(1);
96
105
  }
97
106
  break;
98
107
  }
@@ -5,7 +5,7 @@ const path = require('path');
5
5
  const {
6
6
  green, yellow, cyan, bold, dim,
7
7
  info, success, warn, error: logError, step,
8
- copyDirRecursive, fileExists, readJSON,
8
+ copyDirRecursive, copyDirTracked, fileExists, readJSON,
9
9
  ensureDir, getRelativePaths,
10
10
  createManifest, writeManifest, getTemplatesDir,
11
11
  COMPONENTS, MANIFEST_FILE,
@@ -41,7 +41,7 @@ function resolveComponents(options) {
41
41
  * Copy a single component from templates to the target directory.
42
42
  * Returns array of relative file paths that were installed.
43
43
  */
44
- function installComponent(comp, templatesDir, targetDir) {
44
+ function installComponent(comp, templatesDir, targetDir, force) {
45
45
  const src = path.join(templatesDir, comp.src);
46
46
  const dest = path.join(targetDir, comp.src);
47
47
 
@@ -51,13 +51,21 @@ function installComponent(comp, templatesDir, targetDir) {
51
51
  }
52
52
 
53
53
  if (comp.isFile) {
54
+ if (fileExists(dest) && !force) {
55
+ // Preserve a user-modified file (e.g. their own CLAUDE.md / settings.json)
56
+ // when re-installing without --force. It is still manifest-tracked because
57
+ // it originates from the template.
58
+ return [comp.src];
59
+ }
54
60
  ensureDir(path.dirname(dest));
55
61
  fs.copyFileSync(src, dest);
56
62
  return [comp.src];
57
63
  }
58
64
 
59
- copyDirRecursive(src, dest);
60
- return getRelativePaths(dest).map((rel) => path.join(comp.src, rel));
65
+ // Track files via the TEMPLATE source — never a scan of the destination —
66
+ // so user-created files under dest are neither recorded nor (later) removed.
67
+ const { copied, skipped } = copyDirTracked(src, dest, force);
68
+ return [...copied, ...skipped].map((rel) => path.join(comp.src, rel));
61
69
  }
62
70
 
63
71
  // ---------------------------------------------------------------------------
@@ -118,7 +126,7 @@ async function run(options) {
118
126
  const comp = COMPONENTS[key];
119
127
  step(i + 1, totalComponents, `Installing ${comp.label}...`);
120
128
 
121
- const files = installComponent(comp, templatesDir, targetDir);
129
+ const files = installComponent(comp, templatesDir, targetDir, force);
122
130
  installedFiles.push(...files);
123
131
  }
124
132
 
@@ -159,12 +159,16 @@ async function run(options) {
159
159
  const comp = COMPONENTS[key];
160
160
  if (!comp) continue;
161
161
  const destPath = path.join(targetDir, comp.src);
162
+ const srcPath = path.join(templatesDir, comp.src);
162
163
  if (comp.isFile) {
163
164
  if (fileExists(destPath)) {
164
165
  allFiles.push(comp.src);
165
166
  }
166
167
  } else if (fileExists(destPath)) {
167
- const paths = getRelativePaths(destPath);
168
+ // Record files from the TEMPLATE source, not a scan of the destination —
169
+ // otherwise user-created files in dest get adopted into the manifest and
170
+ // a later `remove` would delete them.
171
+ const paths = getRelativePaths(fileExists(srcPath) ? srcPath : destPath);
168
172
  allFiles.push(...paths.map((rel) => path.join(comp.src, rel)));
169
173
  }
170
174
  }
package/src/utils.js CHANGED
@@ -64,6 +64,39 @@ function copyDirRecursive(src, dest) {
64
64
  }
65
65
  }
66
66
 
67
+ /**
68
+ * Copy from a TEMPLATE source into dest, tracking which files were written.
69
+ * Walks the SOURCE tree (never the destination), so only template-originated
70
+ * files are ever recorded — a user-created file in dest with no template
71
+ * counterpart is never touched and never returned (prevents `remove` from
72
+ * deleting user data). When `force` is false, a pre-existing destination file
73
+ * is PRESERVED (skipped) rather than clobbered (protects user-modified files
74
+ * on a re-install where the manifest was lost).
75
+ *
76
+ * @returns {{copied: string[], skipped: string[]}} relative paths (to dest root)
77
+ */
78
+ function copyDirTracked(src, dest, force, rel = '', acc = { copied: [], skipped: [] }) {
79
+ const stat = fs.statSync(src);
80
+ if (stat.isFile()) {
81
+ if (fileExists(dest) && !force) {
82
+ acc.skipped.push(rel);
83
+ } else {
84
+ ensureDir(path.dirname(dest));
85
+ fs.copyFileSync(src, dest);
86
+ acc.copied.push(rel);
87
+ }
88
+ return acc;
89
+ }
90
+ if (stat.isDirectory()) {
91
+ ensureDir(dest);
92
+ for (const entry of fs.readdirSync(src)) {
93
+ copyDirTracked(path.join(src, entry), path.join(dest, entry), force,
94
+ rel ? path.join(rel, entry) : entry, acc);
95
+ }
96
+ }
97
+ return acc;
98
+ }
99
+
67
100
  /**
68
101
  * Returns true if the path exists (file or directory).
69
102
  */
@@ -254,6 +287,7 @@ module.exports = {
254
287
 
255
288
  // File operations
256
289
  copyDirRecursive,
290
+ copyDirTracked,
257
291
  fileExists,
258
292
  readJSON,
259
293
  writeJSON,
@@ -1,302 +0,0 @@
1
- # /bto-benchmark — Deterministic Benchmarking of a Skill or Command
2
-
3
- ## Usage
4
- ```
5
- /bto-benchmark [path to skill directory or file]
6
- ```
7
-
8
- ## Parameters
9
- - $ARGUMENTS — Path to a skill directory (`.claude/skills/<name>/`), a command file (`.claude/commands/<name>.md`), a rule file (`.claude/rules/<name>.md`), or any structured artifact. Optionally include "verbose" to show expanded per-section diagnostics for all layers.
10
-
11
- ## Protocol
12
-
13
- ### Step 1: Load Skill and Module
14
-
15
- Read `.claude/skills/bto/SKILL.md`
16
- Read `.claude/skills/bto/modules/benchmark.md`
17
- Read `.claude/skills/bto/references/golden-samples.md`
18
-
19
- ### Step 2: Validate Input
20
-
21
- If $ARGUMENTS is empty:
22
- - Ask: "Provide a path to the skill directory or artifact file you want to benchmark."
23
- - Stop and wait.
24
-
25
- Resolve the artifact path from $ARGUMENTS:
26
- - Strip any trailing slash
27
- - If the path points to a directory → look for `SKILL.md` inside it as the primary file, but include all files in the directory for context
28
- - If the path points to a file → use that file directly
29
- - If the path does not exist → report "Path not found: [path]" and stop
30
-
31
- ### Step 3: Detect Artifact Type
32
-
33
- Auto-detect from path pattern:
34
-
35
- | Path Pattern | Detected Type |
36
- |-------------|--------------|
37
- | `.claude/skills/*/SKILL.md` or `.claude/skills/*/` | skill |
38
- | `.claude/commands/*.md` | command |
39
- | `.claude/rules/*.md` | rule |
40
- | `.claude/agents/*.md` | agent |
41
- | `researches/**/*.md` | research artifact |
42
-
43
- If type cannot be determined from path → infer from file content structure.
44
-
45
- ### Step 4: Layer B0 — Golden Sample Comparison (Deterministic)
46
-
47
- **Purpose:** Compare artifact structure against the canonical golden sample for its type. Zero LLM cost.
48
-
49
- Load golden samples for the detected type from `references/golden-samples.md`.
50
-
51
- **Procedure:**
52
-
53
- 1. Parse the artifact into a section tree (heading hierarchy, section lengths, ordering)
54
- 2. Parse the matching golden sample into the same section tree format
55
- 3. Compare across three structural axes:
56
-
57
- **Axis 1 — Section Coverage:**
58
- For each required section in the golden sample, check if the artifact contains a matching section (by heading text or semantic equivalent).
59
- - `section_coverage = matched_sections / total_golden_sections`
60
-
61
- **Axis 2 — Ordering Score:**
62
- Compare the ordering of matched sections against the golden sample ordering.
63
- - Use normalized Kendall tau distance: `ordering_score = 1 - (inversions / max_inversions)`
64
- - If fewer than 3 matched sections → ordering_score = 1.0 (too few to meaningfully compare)
65
-
66
- **Axis 3 — Proportion Score:**
67
- For each matched section, compute the ratio of artifact section length to golden sample section length.
68
- - `proportion_score = 1 - mean(|log(artifact_len / golden_len)|)` clamped to [0, 1]
69
- - Measures whether sections are proportionally sized (not too bloated, not too thin)
70
-
71
- **Display per-section MATCH/MISS table:**
72
-
73
- ```
74
- Golden Section Status Artifact Section Size Ratio
75
- ─────────────────────────── ──────── ──────────────────────────── ──────────
76
- ## Overview MATCH ## Overview 1.2x
77
- ## Protocol MATCH ## Protocol 0.8x
78
- ## Anti-Patterns MATCH ## Anti-Patterns 1.0x
79
- ## Dependencies MISS — —
80
- ## Quick Start MATCH ## Quick Start 1.5x
81
- ```
82
-
83
- **Golden Similarity Score:**
84
- ```
85
- B0 = section_coverage * 0.50 + ordering_score * 0.25 + proportion_score * 0.25
86
- ```
87
-
88
- **Gate:** If B0 < 0.30 → BLOCK immediately. The artifact's structure is too far from the expected form to benchmark meaningfully.
89
-
90
- ### Step 5: Layer B1 — Deterministic Test Suite
91
-
92
- **Purpose:** Run all applicable deterministic tests for the detected artifact type. Zero LLM cost.
93
-
94
- Execute all applicable checks from `benchmark.md` for the detected artifact type. These overlap with but extend the Layer 0 checks from `/bto-test`:
95
-
96
- **Universal Checks (U1-U12):**
97
- - U-01: File exists and is non-empty
98
- - U-02: UTF-8 encoding valid
99
- - U-03: Starts with level-1 heading
100
- - U-04: No placeholder text (`[TODO]`, `[TBD]`, `<INSERT>`, `[PLACEHOLDER]`)
101
- - U-05: No empty sections (heading with no content before next heading)
102
- - U-06: Consistent heading hierarchy (no h1→h3 jumps)
103
- - U-07: No broken internal cross-references
104
- - U-08: File size within bounds (200B – 100KB per file)
105
- - U-09: No trailing whitespace on lines
106
- - U-10: Standard Markdown only (no HTML unless in code blocks)
107
- - U-11: All code blocks properly closed
108
- - U-12: No duplicate top-level sections
109
-
110
- **Type-Specific Checks:**
111
- - Skill: SK-01 through SK-16 (SKILL.md exists, required sections, modules/, references/, examples/, etc.)
112
- - Command: CM-01 through CM-08 (Usage section, Parameters, Protocol steps, Checkpoint, Critical Rules, etc.)
113
- - Rule: RL-01 through RL-06 (Table format, detection signals, fix actions, no contradictions, etc.)
114
- - Agent: AG-01 through AG-04 (Model specified, isolation rules, naming convention, output format)
115
-
116
- **Display per-test PASS/FAIL table:**
117
-
118
- ```
119
- ID Check Result Note
120
- ─────── ────────────────────────────────── ──────── ────────────────────────────
121
- U-01 File exists and is non-empty PASS 8.4 KB
122
- U-02 UTF-8 encoding PASS
123
- ...
124
- SK-01 SKILL.md exists at skill root PASS
125
- SK-02 Has ## Overview section PASS
126
- SK-03 Has ## Anti-Patterns section FAIL Section missing
127
- ...
128
- ```
129
-
130
- **Test Pass Rate:**
131
- ```
132
- B1 = passed_tests / total_applicable_tests
133
- ```
134
-
135
- **Gate:** If B1 < 0.60 → BLOCK. Too many structural failures to proceed to consistency probing.
136
-
137
- ### Step 6: Layer B2 — Consistency Probe
138
-
139
- **Purpose:** Measure evaluation stability by running identical prompts across multiple agents. Detects artifacts that produce inconsistent interpretations.
140
-
141
- **Model:** haiku (cost-optimized)
142
-
143
- **Spawn 3 parallel haiku agents with the identical evaluation prompt:**
144
-
145
- Each agent receives the exact same prompt:
146
- ```
147
- You are evaluating a Claude Code artifact for structural quality.
148
- Rate these 4 dimensions (0.0 to 1.0 each, two decimal places):
149
- 1. STRUCTURE — Does the artifact follow a clear, logical organization?
150
- 2. CLARITY — Are instructions unambiguous and actionable?
151
- 3. COVERAGE — Does it address all aspects implied by its title/scope?
152
- 4. CONSISTENCY — Is the internal terminology and style uniform?
153
- Output as JSON: {"structure": X.XX, "clarity": X.XX, "coverage": X.XX, "consistency": X.XX}
154
- ```
155
-
156
- **Isolation:** Each agent evaluates independently. No cross-communication.
157
-
158
- **After all 3 return, compute structural agreement:**
159
-
160
- For each dimension:
161
- ```
162
- agreement[dim] = 1 - (max(scores[dim]) - min(scores[dim]))
163
- ```
164
-
165
- **Consistency Score:**
166
- ```
167
- B2 = mean(agreement across all 4 dimensions)
168
- ```
169
-
170
- **Interpretation:**
171
- - B2 > 0.85 → HIGH consistency — the artifact is unambiguously structured
172
- - B2 0.60-0.85 → MODERATE consistency — some sections are open to interpretation
173
- - B2 < 0.60 → LOW consistency — the artifact is structurally ambiguous
174
-
175
- **Display per-probe comparison:**
176
-
177
- ```
178
- Dimension Probe 1 Probe 2 Probe 3 Range Agreement
179
- ──────────── ────────── ────────── ────────── ──────── ──────────
180
- STRUCTURE 0.80 0.85 0.82 0.05 0.95
181
- CLARITY 0.70 0.75 0.65 0.10 0.90
182
- COVERAGE 0.60 0.70 0.55 0.15 0.85
183
- CONSISTENCY 0.75 0.78 0.73 0.05 0.95
184
- Mean: 0.91
185
- ```
186
-
187
- No gate on B2 — consistency is informational and feeds into the aggregate score.
188
-
189
- ### Step 7: Layer B3 — Performance Metrics
190
-
191
- **Purpose:** Quantify token efficiency and information density. Zero LLM cost.
192
-
193
- **Metric 1 — Token Efficiency:**
194
- ```
195
- token_efficiency = meaningful_content_tokens / total_tokens
196
- ```
197
- Where `meaningful_content_tokens` excludes: blank lines, decorative separators, repeated boilerplate headers, and excessive whitespace. Target: > 0.70.
198
-
199
- **Metric 2 — Information Density:**
200
- ```
201
- information_density = unique_concepts / total_sections
202
- ```
203
- Where `unique_concepts` = count of distinct topics, terms, or instructions introduced. Measures whether each section contributes new information vs. repeating prior content. Target: > 2.0 concepts per section.
204
-
205
- **Metric 3 — Bloat Detection:**
206
- Identify sections where:
207
- - Section length > 3x the golden sample equivalent → BLOATED
208
- - Section length < 0.2x the golden sample equivalent → THIN
209
- - Section repeats > 30% of content from another section → REDUNDANT
210
-
211
- ```
212
- bloat_score = 1 - (bloated_sections + thin_sections + redundant_sections) / total_sections
213
- ```
214
- Clamped to [0, 1].
215
-
216
- **Performance Score:**
217
- ```
218
- B3 = token_efficiency * 0.35 + normalize(information_density, 0, 4) * 0.35 + bloat_score * 0.30
219
- ```
220
- Where `normalize(x, min, max)` clamps and scales x to [0, 1].
221
-
222
- **Display per-section metrics:**
223
-
224
- ```
225
- Section Tokens Concepts Golden Ratio Flag
226
- ───────────────────────── ──────── ────────── ────────────── ──────────
227
- ## Overview 320 4 1.2x OK
228
- ## Protocol 1840 12 0.8x OK
229
- ## Anti-Patterns 580 6 2.8x BLOATED
230
- ## Quick Start 95 2 0.3x THIN
231
- ## Dependencies 210 3 1.0x OK
232
- ```
233
-
234
- ### Step 8: Aggregate & Gate
235
-
236
- **BENCHMARK_SCORE formula:**
237
- ```
238
- BENCHMARK_SCORE = B0 * 0.30 + B1 * 0.35 + B2 * 0.15 + B3 * 0.20
239
- ```
240
-
241
- **Weight rationale:**
242
- - B0 (Golden Comparison) at 0.30 — structural conformance is the primary benchmark signal
243
- - B1 (Test Suite) at 0.35 — deterministic correctness is the most reliable signal
244
- - B2 (Consistency) at 0.15 — stability matters but is a secondary signal
245
- - B3 (Performance) at 0.20 — efficiency matters for production artifacts
246
-
247
- **Gate:**
248
- - BENCHMARK_SCORE < 0.50 → **BLOCK** — artifact requires significant rework before evaluation
249
- - BENCHMARK_SCORE 0.50-0.70 → **WARN** — artifact has structural issues; proceed to TEST with caution
250
- - BENCHMARK_SCORE > 0.70 → **PASS** — artifact meets benchmark standards; ready for TEST
251
-
252
- Record `BENCHMARK_SCORE` for downstream consumption by `/bto-test` and `/bto`.
253
-
254
- ---
255
-
256
- ## Checkpoint
257
-
258
- ```
259
- ═══════════════════════════════════════════════════════
260
- CHECKPOINT: BENCHMARK Complete
261
- <promise>BTO_BENCHMARKED</promise>
262
- Artifact: [path]
263
- Type: [detected type]
264
-
265
- Layer B0 (Golden): X.XX — section coverage, ordering, proportions
266
- Layer B1 (Test Suite): X.XX — Y/Z tests passed
267
- Layer B2 (Consistency): X.XX — agreement across 3 probes
268
- Layer B3 (Performance): X.XX — token efficiency, density, bloat
269
-
270
- BENCHMARK SCORE: X.XX [PASS / WARN / BLOCK]
271
-
272
- • "ок" — proceed to TEST
273
- • "покажи детали [B0/B1/B2/B3]" — expand layer details
274
- • "пропусти тест" — skip TEST (if PASS)
275
- ═══════════════════════════════════════════════════════
276
- ```
277
-
278
- Wait for user confirmation.
279
-
280
- ---
281
-
282
- ## Modular Usage
283
-
284
- This command is also invoked internally by:
285
- - `/bto` — as a pre-flight step before TEST phase (if benchmark module is loaded)
286
- - `/bto-test` — optionally as a prerequisite gate (if user requests `benchmark` flag)
287
-
288
- The `BENCHMARK_SCORE` is available to downstream commands:
289
- - `/bto-test` may use it to calibrate Layer 1 expectations
290
- - `/bto-optimize` may use it to set a structural baseline floor
291
-
292
- ## Critical Rules
293
-
294
- - Layer B0 and B1 are fully deterministic — no LLM calls, zero cost
295
- - Layer B2 uses haiku only — NEVER use sonnet or opus for consistency probes
296
- - Agent tool is REQUIRED for Layer B2 — do not run probes sequentially
297
- - Golden samples from `references/golden-samples.md` are the authoritative structural reference
298
- - If golden sample does not exist for the detected type → skip B0, set B0 = 0.50 (neutral), and log a warning
299
- - Report `BENCHMARK_SCORE` explicitly so downstream commands can consume it
300
- - BLOCK verdict at B0 or B1 halts the benchmark — do NOT proceed to subsequent layers
301
- - Layer B3 bloat detection uses golden sample proportions — without a golden sample, use absolute thresholds only
302
- - If "verbose" is in $ARGUMENTS, show expanded per-section diagnostics for all layers without prompting