@erclx/aitk 3.16.0 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/claude/.claude-plugin/plugin.json +1 -1
- package/claude/skills/bash-script/SKILL.md +1 -1
- package/claude/skills/ci-workflow/SKILL.md +1 -1
- package/claude/skills/cli-script/SKILL.md +1 -1
- package/claude/skills/create-skill/SKILL.md +1 -1
- package/claude/skills/create-snippet/SKILL.md +1 -1
- package/claude/skills/create-standard/SKILL.md +1 -1
- package/claude/skills/decision-escalate/SKILL.md +1 -1
- package/claude/skills/git-branch/SKILL.md +1 -1
- package/claude/skills/git-pr/SKILL.md +1 -1
- package/claude/skills/git-stage/SKILL.md +1 -1
- package/claude/skills/youtube-transcripts/SKILL.md +1 -1
- package/docs/agents/audits.md +3 -3
- package/docs/agents/commands.md +22 -21
- package/package.json +1 -1
- package/src/audits/catalog.ts +35 -0
- package/src/claude/cases/all.ts +24 -0
- package/src/claude/cases/authoring.ts +53 -0
- package/src/claude/cases/claude-workflow.ts +153 -0
- package/src/claude/cases/git.ts +44 -0
- package/src/claude/cases/misc.ts +27 -0
- package/src/claude/cases/setup.ts +78 -0
- package/src/claude/skills-rank.ts +240 -0
- package/src/commands/claude.ts +148 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: bash-script
|
|
3
|
-
description: Generates production Bash scripts with a visual timeline UI, state-based interactive prompts, and strict error handling. Use when asked
|
|
3
|
+
description: Generates production Bash scripts with a visual timeline UI, state-based interactive prompts, and strict error handling. Use when asked for "a human-facing shell tool", an interactive bash script, or a DevOps tool with framed terminal output. Do NOT use for a non-interactive automation, CI, or pipeline script, that is `cli-script`.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Bash script
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: ci-workflow
|
|
3
|
-
description: Generates GitHub Actions CI workflow files with parallel jobs, emoji job names, and gated deploy stages. Use when asked to write a CI workflow, add GitHub Actions, set up a CI pipeline, or create a `.github/workflows` file.
|
|
3
|
+
description: Generates GitHub Actions CI workflow files with parallel jobs, emoji job names, and gated deploy stages. Use when asked to "write a CI workflow", add GitHub Actions, set up a CI pipeline, or create a `.github/workflows` file.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# CI workflow
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: cli-script
|
|
3
|
-
description: Generates non-interactive Bash scripts for automation, CI, and agent-run tasks. Lean functional style with structured logging, strict error handling, and a clean stdout, no interactive UI. Use when asked for an automation script, a CI or cron script, a pipeline helper
|
|
3
|
+
description: Generates non-interactive Bash scripts for automation, CI, and agent-run tasks. Lean functional style with structured logging, strict error handling, and a clean stdout, no interactive UI. Use when asked for "a non-interactive shell script", an automation script, a CI or cron script, or a pipeline helper. Do NOT use for a human-facing interactive tool with prompts or a visual timeline UI, that is `bash-script`.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# CLI script
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: create-skill
|
|
3
|
-
description: Creates a new `SKILL.md` in `.claude/skills/`. Use when asked to create a skill, add a skill, or make a new skill.
|
|
3
|
+
description: Creates a new `SKILL.md` in `.claude/skills/`. Use when asked to "create a skill", add a skill, or make a new skill.
|
|
4
4
|
disable-model-invocation: true
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: create-snippet
|
|
3
|
-
description: Creates a new snippet file in `snippets/` or `.claude/snippets/`. Use when asked to create a snippet, add a snippet, write a reusable prompt, or make a new snippet. Do NOT use to edit an existing snippet.
|
|
3
|
+
description: Creates a new snippet file in `snippets/` or `.claude/snippets/`. Use when asked to "create a snippet", add a snippet, write a reusable prompt, or make a new snippet. Do NOT use to edit an existing snippet.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Create snippet
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: create-standard
|
|
3
|
-
description: Creates a new standard file in `standards/`. Use when asked to create a standard, add a standard, or write a new authoring convention. Do NOT use to edit an existing standard.
|
|
3
|
+
description: Creates a new standard file in `standards/`. Use when asked to "create a standard", add a standard, or write a new authoring convention. Do NOT use to edit an existing standard.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Create standard
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: decision-escalate
|
|
3
|
-
description: Collects every open decision whose answer turns on the operator's preference, puts them as one batch of questions each carrying options and a recommended default, then waits. Use when asked to "escalate this", "ask me the open questions", "batch the open decisions", "what do you need from me", or "stop and ask before you pick". Do NOT use for a judgment call the session can settle, which is a pick with the tradeoff stated in one sentence.
|
|
3
|
+
description: Collects every open decision whose answer turns on the operator's preference, since only they know what they need, puts them as one batch of questions each carrying options and a recommended default, then waits. Use when asked to "escalate this", "ask me the open questions", "batch the open decisions", "what do you need from me", or "stop and ask before you pick". Do NOT use for a judgment call the session can settle, which is a pick with the tradeoff stated in one sentence.
|
|
4
4
|
disable-model-invocation: true
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: git-branch
|
|
3
|
-
description: Generates and validates conventional branch names. Use for any branch naming or renaming.
|
|
3
|
+
description: Generates and validates conventional branch names. Use for any branch naming or renaming, such as when asked to "name this branch".
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Git branch
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: git-pr
|
|
3
|
-
description: Generates pull request titles and descriptions from git diffs. Use for any PR creation or update.
|
|
3
|
+
description: Generates pull request titles and descriptions from git diffs. Use for any PR creation or update, such as when asked to "open a pull request".
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Git PR
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: git-stage
|
|
3
|
-
description: Groups staged files by concern and generates one conventional commit per group. Use when staged changes span multiple concerns and need to ship as separate commits. Do NOT use on a single-concern staged set. That is `git-commit`, which also preserves a hunk-level selection this skill's unstage and restage over whole files would widen.
|
|
3
|
+
description: Groups staged files by concern and generates one conventional commit per group. Use when asked to "group these changes into commits", or when staged changes span multiple concerns and need to ship as separate commits. Do NOT use on a single-concern staged set. That is `git-commit`, which also preserves a hunk-level selection this skill's unstage and restage over whole files would widen.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Git stage
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: youtube-transcripts
|
|
3
|
-
description: Fetches a YouTube video transcript with metadata frontmatter into the current repo via `aitk transcripts`. Use when the user pastes a YouTube URL and asks to grab the transcript, pull captions, or save a transcript for context. Do NOT use for downloading the video file, summarizing a transcript that already exists, or non-YouTube links.
|
|
3
|
+
description: Fetches a YouTube video transcript with metadata frontmatter into the current repo via `aitk transcripts`. Use when the user pastes a YouTube URL and asks to "grab the transcript", pull captions, or save a transcript for context. Do NOT use for downloading the video file, summarizing a transcript that already exists, or non-YouTube links.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# YouTube transcripts
|
package/docs/agents/audits.md
CHANGED
|
@@ -22,11 +22,11 @@ aitk audits list --json
|
|
|
22
22
|
|
|
23
23
|
## What it runs
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
Nineteen verbs, listed by `aitk audits list`. Each runs once in its fullest form, and the aggregate reads that verb's own record rather than imposing a shared envelope on it. Every one of those records already has consumers naming its keys, so a common shape would be a breaking change bought for tidiness.
|
|
26
26
|
|
|
27
27
|
The verbs walk separate trees and share no state, so they run together. Measured on the authoring machine at twelve verbs, a run finished in 0.8 seconds of wall clock against 4.4 seconds of processor, which is under every other stage in `bun run check`. `aitk deps audit` is the one that changes that reading, since it reaches a network rather than a tree and its latency is the index's rather than this machine's.
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
Sixteen of the nineteen read a tree on this disk, the one added by `restated.md` and the one added by `census.md` among them, since the first reads three such trees against each other and the second counts every file in one. The two added by `state-scoped-risk.md` read committed state rather than an arriving change, which is the gap every review surface here leaves by construction, and the one added by `label-coverage.md` reads a branch range against a map the project declares.
|
|
30
30
|
|
|
31
31
|
Each is invoked as the CLI the caller is running rather than as a global `aitk`. A globally installed binary resolves to the main checkout no matter which worktree is executing, so the aggregate would measure a tree the branch never touched and report a pass over it.
|
|
32
32
|
|
|
@@ -57,7 +57,7 @@ The verdict is published as a field as well as an exit code. A reader branching
|
|
|
57
57
|
|
|
58
58
|
A per-machine folder that is not on this disk reports as `absent` and moves neither the verdict nor the exit code. Every one of those folders is gitignored, so a fresh clone and every CI run carry none of them, and counting six expected absences as failures pins the verdict at `incomplete` forever. A signal that never changes is one nobody reads after the second time they see it.
|
|
59
59
|
|
|
60
|
-
Which refusals count as an absence is declared per audit rather than inferred from the corpus alone. A per-machine corpus counts a missing folder. An upstream corpus counts every reason the advisory verb publishes, since each of the three means there is nothing resolved to measure. Several tracked audits override that default too, wherever the corpus not being there is the ordinary state of a target rather than a broken checkout: the secret scan for a project that publishes nothing, the skill corpora
|
|
60
|
+
Which refusals count as an absence is declared per audit rather than inferred from the corpus alone. A per-machine corpus counts a missing folder. An upstream corpus counts every reason the advisory verb publishes, since each of the three means there is nothing resolved to measure. Several tracked audits override that default too, wherever the corpus not being there is the ordinary state of a target rather than a broken checkout: the secret scan for a project that publishes nothing, the skill corpora, the citation reach check, and the routing measure for a project holding no skills tree, and the context audit for a project that adopted none of the folders it measures.
|
|
61
61
|
|
|
62
62
|
Everything else stays `unmeasured`. That covers a tree git cannot list, which is a broken checkout, and a package declaring no `files` field, where a publish would pack the whole tree and the scan read none of it. Both are a corpus that exists and went unread, so softening either would report a pass over a shipped tree nobody measured.
|
|
63
63
|
|
package/docs/agents/commands.md
CHANGED
|
@@ -48,6 +48,7 @@ Full help: `aitk <command> --help`. Behavior notes for the install and sync verb
|
|
|
48
48
|
| `aitk claude skills audit` | Report both skill corpora against the mechanical rules in `standards/skill.md` |
|
|
49
49
|
| `aitk claude skills drift` | Name the shipped skill bodies rewritten between a given ref and `HEAD`, and the installed version against the newest published (`--json`) |
|
|
50
50
|
| `aitk claude skills reach` | Report the shipped bodies citing a toolkit path no target project receives, exiting 2 on an unqualified one |
|
|
51
|
+
| `aitk claude skills rank` | Score the shipped catalog's descriptions against a hand-authored case corpus by TF-IDF cosine similarity, reporting rank-one and top-three |
|
|
51
52
|
| `aitk claude routing` | Report per `CLAUDE.md` section how many bullets name a path and how many of those a path-scoped rule already covers (`--json`) |
|
|
52
53
|
| `aitk gov test-order` | Report where an implementation reached history ahead of the test covering it (`--json`) |
|
|
53
54
|
| `aitk gov superseded` | Report where the tree still asserts a value a changed convention no longer produces, keyed on the value (`--json`) |
|
|
@@ -67,27 +68,27 @@ Full help: `aitk <command> --help`. Behavior notes for the install and sync verb
|
|
|
67
68
|
|
|
68
69
|
Each domain exposes a consistent shape where applicable: `list`, `install`, `sync`, `create`.
|
|
69
70
|
|
|
70
|
-
| Domain | Subcommands
|
|
71
|
-
| ----------- |
|
|
72
|
-
| `tooling` | `list`, `sync`, `ref`, `create`, `verify`, `inject`, `prune-gitignore`
|
|
73
|
-
| `snippets` | `list`, `install`, `sync`, `create`
|
|
74
|
-
| `standards` | `list`, `<name>`
|
|
75
|
-
| `gov` | `list`, `install`, `sync`, `build`, `regen`, `test-order`, `superseded`
|
|
76
|
-
| `claude` | `init`, `sync`, `routing`, `seeds list`, `skills list`, `skills audit`, `skills drift`, `skills reach`, `setup [dest]` |
|
|
77
|
-
| `demo` | `compile`, `run`
|
|
78
|
-
| `wiki` | `init`
|
|
79
|
-
| `design` | `render`
|
|
80
|
-
| `slides` | `render`, `list`
|
|
81
|
-
| `tasks` | `archive`, `validate`
|
|
82
|
-
| `intake` | `list`, `answer`
|
|
83
|
-
| `teach` | `list`, `open`, `resource`, `glossary`
|
|
84
|
-
| `comments` | `scan`
|
|
85
|
-
| `context` | `audit`
|
|
86
|
-
| `markdown` | `audit`
|
|
87
|
-
| `secrets` | `scan`
|
|
88
|
-
| `deps` | `audit`
|
|
89
|
-
| `labels` | `audit`
|
|
90
|
-
| `audits` | `run`, `list`
|
|
71
|
+
| Domain | Subcommands |
|
|
72
|
+
| ----------- | ------------------------------------------------------------------------------------------------------------------------------------- |
|
|
73
|
+
| `tooling` | `list`, `sync`, `ref`, `create`, `verify`, `inject`, `prune-gitignore` |
|
|
74
|
+
| `snippets` | `list`, `install`, `sync`, `create` |
|
|
75
|
+
| `standards` | `list`, `<name>` |
|
|
76
|
+
| `gov` | `list`, `install`, `sync`, `build`, `regen`, `test-order`, `superseded` |
|
|
77
|
+
| `claude` | `init`, `sync`, `routing`, `seeds list`, `skills list`, `skills audit`, `skills drift`, `skills reach`, `skills rank`, `setup [dest]` |
|
|
78
|
+
| `demo` | `compile`, `run` |
|
|
79
|
+
| `wiki` | `init` |
|
|
80
|
+
| `design` | `render` |
|
|
81
|
+
| `slides` | `render`, `list` |
|
|
82
|
+
| `tasks` | `archive`, `validate` |
|
|
83
|
+
| `intake` | `list`, `answer` |
|
|
84
|
+
| `teach` | `list`, `open`, `resource`, `glossary` |
|
|
85
|
+
| `comments` | `scan` |
|
|
86
|
+
| `context` | `audit` |
|
|
87
|
+
| `markdown` | `audit` |
|
|
88
|
+
| `secrets` | `scan` |
|
|
89
|
+
| `deps` | `audit` |
|
|
90
|
+
| `labels` | `audit` |
|
|
91
|
+
| `audits` | `run`, `list` |
|
|
91
92
|
|
|
92
93
|
Common patterns:
|
|
93
94
|
|
package/package.json
CHANGED
package/src/audits/catalog.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { SkillsAuditRefusal } from '@/claude/skills-audit'
|
|
2
|
+
import type { RankRefusal } from '@/claude/skills-rank'
|
|
2
3
|
import type { ReachRefusal } from '@/claude/skills-reach'
|
|
3
4
|
import type { ContextAuditRefusal } from '@/context/audit'
|
|
4
5
|
import type { AuditRefusal } from '@/deps/audit'
|
|
@@ -306,6 +307,24 @@ function reachCounts(record: unknown): Record<string, number> | undefined {
|
|
|
306
307
|
return allOf({ unqualifiedCitations: lengthOf(root.unqualified) })
|
|
307
308
|
}
|
|
308
309
|
|
|
310
|
+
/**
|
|
311
|
+
* Reads the routing measure's miss and unmeasurable counts, leaving `rank1`
|
|
312
|
+
* and `top3` out. Both move in lockstep with `misses` over a fixed-size
|
|
313
|
+
* corpus, so carrying them would report one movement three times, the same
|
|
314
|
+
* reasoning `restatedCounts` already takes over its own pair of derived
|
|
315
|
+
* totals. `unmeasurable` is retained separately, since it counts a defect in
|
|
316
|
+
* the instrument rather than a collision the catalog itself carries.
|
|
317
|
+
*/
|
|
318
|
+
function rankCounts(record: unknown): Record<string, number> | undefined {
|
|
319
|
+
const root = asObject(record)
|
|
320
|
+
if (root === undefined) return undefined
|
|
321
|
+
|
|
322
|
+
return allOf({
|
|
323
|
+
misses: lengthOf(root.misses),
|
|
324
|
+
unmeasurable: lengthOf(root.unmeasurable),
|
|
325
|
+
})
|
|
326
|
+
}
|
|
327
|
+
|
|
309
328
|
function boardCounts(record: unknown): Record<string, number> | undefined {
|
|
310
329
|
const root = asObject(record)
|
|
311
330
|
if (root === undefined) return undefined
|
|
@@ -518,6 +537,22 @@ export const AUDITS: readonly AuditSpec[] = [
|
|
|
518
537
|
absentReasons: ['no-skills'] satisfies ReachRefusal[],
|
|
519
538
|
counts: reachCounts,
|
|
520
539
|
},
|
|
540
|
+
{
|
|
541
|
+
id: 'skills-rank',
|
|
542
|
+
label: 'Skill routing measure',
|
|
543
|
+
argv: ['claude', 'skills', 'rank', '--json'],
|
|
544
|
+
// Reports rather than gates. The corpus is a first run with no baseline to
|
|
545
|
+
// fail a push against, and a lexical ranker bounds a necessary condition
|
|
546
|
+
// rather than reporting real routing behavior, so a push failing on a
|
|
547
|
+
// moved rank would teach a contributor to route around the stage.
|
|
548
|
+
gatingExits: [],
|
|
549
|
+
corpus: 'tracked',
|
|
550
|
+
// The one reason this verb refuses for, and it is an absence for the same
|
|
551
|
+
// reason the reach check's is: a project carrying no `claude/skills/` has
|
|
552
|
+
// adopted no skill convention this audit reads.
|
|
553
|
+
absentReasons: ['no-skills'] satisfies RankRefusal[],
|
|
554
|
+
counts: rankCounts,
|
|
555
|
+
},
|
|
521
556
|
{
|
|
522
557
|
id: 'tasks',
|
|
523
558
|
label: 'Task board',
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
import { AUTHORING_CASES } from '@/claude/cases/authoring'
|
|
3
|
+
import { CLAUDE_WORKFLOW_CASES } from '@/claude/cases/claude-workflow'
|
|
4
|
+
import { GIT_CASES } from '@/claude/cases/git'
|
|
5
|
+
import { MISC_CASES } from '@/claude/cases/misc'
|
|
6
|
+
import { SETUP_CASES } from '@/claude/cases/setup'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The full routing case corpus, one file per domain so a description change
|
|
10
|
+
* in one family touches only the file beside it.
|
|
11
|
+
*
|
|
12
|
+
* Bootstrapped from each skill's own quoted trigger phrases and then
|
|
13
|
+
* rephrased and supplemented rather than shipped verbatim, since a positive
|
|
14
|
+
* lifted unchanged from a trigger passes by construction and never tests a
|
|
15
|
+
* person's own words. See `.claude/groundwork/42-ai-blueprint/08-spikes.md`
|
|
16
|
+
* for the extraction arm this corpus replaces as the shipped measure.
|
|
17
|
+
*/
|
|
18
|
+
export const SKILL_CASES: readonly SkillCase[] = [
|
|
19
|
+
...CLAUDE_WORKFLOW_CASES,
|
|
20
|
+
...GIT_CASES,
|
|
21
|
+
...SETUP_CASES,
|
|
22
|
+
...AUTHORING_CASES,
|
|
23
|
+
...MISC_CASES,
|
|
24
|
+
]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Script, skill, snippet, standard, and doc authoring, plus the two prose
|
|
5
|
+
* skills that revise rather than generate.
|
|
6
|
+
*/
|
|
7
|
+
export const AUTHORING_CASES: readonly SkillCase[] = [
|
|
8
|
+
{
|
|
9
|
+
prompt:
|
|
10
|
+
'I need a shell tool with prompts and a nice terminal UI for people to run by hand.',
|
|
11
|
+
expect: 'bash-script',
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
prompt:
|
|
15
|
+
'Wire up a GitHub Actions pipeline with parallel jobs for this repo.',
|
|
16
|
+
expect: 'ci-workflow',
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
prompt:
|
|
20
|
+
'Write me a headless automation script with no interactive prompts.',
|
|
21
|
+
expect: 'cli-script',
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
prompt: 'Scaffold a brand-new SKILL.md for this capability.',
|
|
25
|
+
expect: 'create-skill',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
prompt: 'I need a new reusable prompt snippet added to the corpus.',
|
|
29
|
+
expect: 'create-snippet',
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
prompt: 'Write a fresh authoring convention as a new standard file.',
|
|
33
|
+
expect: 'create-standard',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
prompt:
|
|
37
|
+
"The docs folder and README are stale against what's on main, refresh them.",
|
|
38
|
+
expect: 'docs-sync',
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
prompt: 'Say what that dense answer actually means in plain terms.',
|
|
42
|
+
expect: 'restate',
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
prompt: 'This passage reads flat and robotic, give it some real cadence.',
|
|
46
|
+
expect: 'write-human',
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
prompt:
|
|
50
|
+
'Pull the captions off this YouTube link and save them with metadata.',
|
|
51
|
+
expect: 'youtube-transcripts',
|
|
52
|
+
},
|
|
53
|
+
]
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The `claude-*` skill family: feature planning, review, and the session
|
|
5
|
+
* artifacts that coordinate work across sessions.
|
|
6
|
+
*
|
|
7
|
+
* Every positive is phrased away from its skill's own quoted trigger, since a
|
|
8
|
+
* verbatim trigger passes by construction and proves nothing about a prompt in
|
|
9
|
+
* someone's own words. Negatives target the pairs whose bodies already state
|
|
10
|
+
* an explicit `Do NOT` boundary against each other, since a boundary in a
|
|
11
|
+
* body does not reach the field a router reads.
|
|
12
|
+
*/
|
|
13
|
+
export const CLAUDE_WORKFLOW_CASES: readonly SkillCase[] = [
|
|
14
|
+
{
|
|
15
|
+
prompt: 'Go fix everything the reviewer flagged on my open PR.',
|
|
16
|
+
expect: 'claude-address-review',
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
prompt:
|
|
20
|
+
'Run the whole implement, verify, review, and ship pipeline for the approved plan.',
|
|
21
|
+
expect: 'claude-autoship',
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
prompt: "Pull together a design system from what's already in the app.",
|
|
25
|
+
expect: 'claude-design-extract',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
prompt: 'Draw me a diagram of how the pieces of this system connect.',
|
|
29
|
+
expect: 'claude-diagram',
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
prompt:
|
|
33
|
+
'Bring the internal planning docs under .claude up to date with what we decided this session.',
|
|
34
|
+
expect: 'claude-docs',
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
prompt:
|
|
38
|
+
'Sketch out a plan for adding this new capability before we touch any code.',
|
|
39
|
+
expect: 'claude-feature',
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
prompt:
|
|
43
|
+
'We need to measure this properly before committing to an approach.',
|
|
44
|
+
expect: 'claude-groundwork',
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
prompt: 'Take this pile of raw notes and turn it into filed items.',
|
|
48
|
+
expect: 'claude-intake',
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
prompt: 'Walk me through the open intake items so I can decide on each.',
|
|
52
|
+
expect: 'claude-intake-answer',
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
prompt:
|
|
56
|
+
"Draft a diff-based proposal for tightening this passage in the standard, don't just edit it.",
|
|
57
|
+
expect: 'claude-markdown-propose',
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
prompt: 'Pull the durable lessons out of this session before it ends.',
|
|
61
|
+
expect: 'claude-memory-capture',
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
prompt:
|
|
65
|
+
'Go through the memory folder and propose what to do with each entry.',
|
|
66
|
+
expect: 'claude-memory-review',
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
prompt:
|
|
70
|
+
'Take on the orchestrator role and coordinate the parallel feature builds.',
|
|
71
|
+
expect: 'claude-orchestrate',
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
prompt: 'Post a formal review with findings on that open pull request.',
|
|
75
|
+
expect: 'claude-pr-review',
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
prompt:
|
|
79
|
+
'Look over everything that changed on this branch for bugs and edge cases.',
|
|
80
|
+
expect: 'claude-review',
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
prompt: 'Draft me a script with beats for a screencast recording.',
|
|
84
|
+
expect: 'claude-screencast',
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
prompt:
|
|
88
|
+
"Check whether my installed Claude seed docs have drifted from the toolkit's.",
|
|
89
|
+
expect: 'claude-seed-sync',
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
prompt: 'Turn this topic into a slide deck I can render.',
|
|
93
|
+
expect: 'claude-slides-draft',
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
prompt:
|
|
97
|
+
'Check whether the markdown I changed violates any authoring standards.',
|
|
98
|
+
expect: 'claude-standards-audit',
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
prompt: 'Open a new entry on the task board for this piece of work.',
|
|
102
|
+
expect: 'claude-tasks',
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
prompt: 'Open a learning workspace so I can study this topic properly.',
|
|
106
|
+
expect: 'claude-teach',
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
prompt: 'Generate the Playwright tests I need after this UI change.',
|
|
110
|
+
expect: 'claude-ui-test',
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
prompt:
|
|
114
|
+
'Look over the interface and tell me what feels unfinished or confusing.',
|
|
115
|
+
expect: 'claude-ux-audit',
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
prompt: 'Tell me the paint and layout cost of this page right now.',
|
|
119
|
+
expect: 'claude-ux-measure',
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
prompt: 'Get me set up in a fresh Claude Code worktree for this branch.',
|
|
123
|
+
expect: 'claude-worktree',
|
|
124
|
+
},
|
|
125
|
+
|
|
126
|
+
// Negatives: pairs whose bodies already carry an explicit Do NOT boundary.
|
|
127
|
+
{
|
|
128
|
+
prompt:
|
|
129
|
+
'Refresh the stale sections of README and the docs folder based on what changed.',
|
|
130
|
+
expect: 'docs-sync',
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
prompt: 'Sort through this brain dump and write it up as findings.',
|
|
134
|
+
expect: 'claude-intake',
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
prompt:
|
|
138
|
+
"Go through what's still unanswered in the intake folder and decide.",
|
|
139
|
+
expect: 'claude-intake-answer',
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
prompt: 'Check this markdown against the house style rules.',
|
|
143
|
+
expect: 'claude-standards-audit',
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
prompt: 'Find the rough, unfinished-feeling spots in this interface.',
|
|
147
|
+
expect: 'claude-ux-audit',
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
prompt: 'Tell me the render cost of this page in the browser.',
|
|
151
|
+
expect: 'claude-ux-measure',
|
|
152
|
+
},
|
|
153
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
|
|
3
|
+
/** The `git-*` skill family: branch, commit, and pull-request mechanics. */
|
|
4
|
+
export const GIT_CASES: readonly SkillCase[] = [
|
|
5
|
+
{
|
|
6
|
+
prompt: 'Come up with a conventional name for this branch.',
|
|
7
|
+
expect: 'git-branch',
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
prompt: "Turn what's staged into one commit message.",
|
|
11
|
+
expect: 'git-commit',
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
prompt: "Push a quick follow-up fix onto the PR that's already open.",
|
|
15
|
+
expect: 'git-followup',
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
prompt: 'Turn this bug into a GitHub issue and file it.',
|
|
19
|
+
expect: 'git-issue',
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
prompt: 'Write the title and body for this pull request.',
|
|
23
|
+
expect: 'git-pr',
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
prompt: 'Take this finished feature all the way through to an opened PR.',
|
|
27
|
+
expect: 'git-ship',
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
prompt:
|
|
31
|
+
'This branch has unrelated commits mixed together, break it into separate branches.',
|
|
32
|
+
expect: 'git-split',
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
prompt:
|
|
36
|
+
'These staged changes cover more than one concern, commit them separately.',
|
|
37
|
+
expect: 'git-stage',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
prompt:
|
|
41
|
+
'Show me the linked worktrees and clear out the ones already merged.',
|
|
42
|
+
expect: 'git-worktree',
|
|
43
|
+
},
|
|
44
|
+
]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
|
|
3
|
+
/** Skills that share no prefix with any other domain file here. */
|
|
4
|
+
export const MISC_CASES: readonly SkillCase[] = [
|
|
5
|
+
{
|
|
6
|
+
prompt:
|
|
7
|
+
'Bundle up every open call that only I can make and ask me all at once.',
|
|
8
|
+
expect: 'decision-escalate',
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
prompt: 'Fire up the dev server the way this project documents it.',
|
|
12
|
+
expect: 'project-commands',
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
prompt:
|
|
16
|
+
"We're about to hit the context limit, write the handoff before we lose state.",
|
|
17
|
+
expect: 'session-map',
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
prompt: 'Catch me up on what was in progress before this session started.',
|
|
21
|
+
expect: 'session-resume',
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
prompt: "This test just started failing and I don't know why yet.",
|
|
25
|
+
expect: 'systematic-debugging',
|
|
26
|
+
},
|
|
27
|
+
]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { SkillCase } from '@/claude/skills-rank'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `setup-*`, `migration-*`, `toolkit-*`, and `create-rule`: scaffolding,
|
|
5
|
+
* proposal-only migrations, and the toolkit's own reference and feedback
|
|
6
|
+
* surfaces.
|
|
7
|
+
*/
|
|
8
|
+
export const SETUP_CASES: readonly SkillCase[] = [
|
|
9
|
+
{
|
|
10
|
+
prompt:
|
|
11
|
+
'This project has no rules installed yet, get the right governance in place.',
|
|
12
|
+
expect: 'setup-gov',
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
prompt:
|
|
16
|
+
"Get the index.md system bootstrapped across this project's folders.",
|
|
17
|
+
expect: 'setup-indexes',
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
prompt:
|
|
21
|
+
'This is a brand-new project, get the toolkit bootstrapped in one shot.',
|
|
22
|
+
expect: 'setup-init',
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
prompt: 'Get the usual Claude Code plugins provisioned on this machine.',
|
|
26
|
+
expect: 'setup-plugins',
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
prompt:
|
|
30
|
+
"Run through the generated scaffold's scripts and confirm each one passes.",
|
|
31
|
+
expect: 'setup-verify',
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
prompt:
|
|
35
|
+
'This CLAUDE.md file has grown huge, break it apart into the tiered context model.',
|
|
36
|
+
expect: 'migration-claude-md',
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
prompt:
|
|
40
|
+
'Move the agent-flavored docs out of the docs folder and into context.',
|
|
41
|
+
expect: 'migration-context',
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
prompt:
|
|
45
|
+
'The snippets folder needs to move under .claude to match the current layout.',
|
|
46
|
+
expect: 'migration-standards',
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
prompt:
|
|
50
|
+
'This file was replaced by a folder, help me split its content into it.',
|
|
51
|
+
expect: 'migration-superseded',
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
prompt:
|
|
55
|
+
"Before I run this sync, tell me exactly what it's going to overwrite.",
|
|
56
|
+
expect: 'toolkit-cli',
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
prompt:
|
|
60
|
+
'Something about the toolkit itself is broken, write it up and send it back to the maintainers.',
|
|
61
|
+
expect: 'toolkit-feedback',
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
prompt:
|
|
65
|
+
"I don't know which specific toolkit skill I need, just handle it for me.",
|
|
66
|
+
expect: 'toolkit-operator',
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
prompt:
|
|
70
|
+
'Work through the open feedback issues on the toolkit repo one by one.',
|
|
71
|
+
expect: 'toolkit-triage',
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
prompt:
|
|
75
|
+
"This project needs its own coding rule that the toolkit doesn't ship.",
|
|
76
|
+
expect: 'create-rule',
|
|
77
|
+
},
|
|
78
|
+
]
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
import { listSkills } from '@/claude/skills-list'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Whether a prompt reaches the right skill, measured by TF-IDF cosine
|
|
7
|
+
* similarity over the shipped catalog's own frontmatter descriptions. This is
|
|
8
|
+
* a necessary condition rather than a report of real routing behavior: it
|
|
9
|
+
* asks whether the descriptions are separable by the words they use, and
|
|
10
|
+
* Claude Code does not route this way.
|
|
11
|
+
*
|
|
12
|
+
* Ported from `.claude/groundwork/42-ai-blueprint/evidence/rank.ts`, which
|
|
13
|
+
* ran once against this catalog and named the collisions this measure now
|
|
14
|
+
* tracks on a cadence.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const SKILLS_DIR = join('claude', 'skills')
|
|
18
|
+
|
|
19
|
+
const STOP = new Set(
|
|
20
|
+
'a about after again all also and any are as at be before but by can do does for from has have help how in into is it its just make not of on or our so that the then there these this to use used uses using want was what when where which who why with you your run'.split(
|
|
21
|
+
' ',
|
|
22
|
+
),
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
export interface RankedSkill {
|
|
26
|
+
readonly name: string
|
|
27
|
+
readonly description: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** One prompt and the skill it should reach. */
|
|
31
|
+
export interface SkillCase {
|
|
32
|
+
readonly prompt: string
|
|
33
|
+
readonly expect: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A case whose prompt did not rank its expected skill first. */
|
|
37
|
+
export interface Miss {
|
|
38
|
+
readonly prompt: string
|
|
39
|
+
readonly expect: string
|
|
40
|
+
/** The skill the ranker placed first instead. */
|
|
41
|
+
readonly won: string
|
|
42
|
+
/** Where `expect` placed, or 0 when it never appears in the catalog. */
|
|
43
|
+
readonly rank: number
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Why a measure produced no reading, which is never the same as a clean one. */
|
|
47
|
+
export type RankRefusal = 'no-skills'
|
|
48
|
+
|
|
49
|
+
export type RankReport =
|
|
50
|
+
| {
|
|
51
|
+
readonly kind: 'measured'
|
|
52
|
+
readonly skills: number
|
|
53
|
+
readonly cases: number
|
|
54
|
+
readonly rank1: number
|
|
55
|
+
readonly top3: number
|
|
56
|
+
readonly misses: readonly Miss[]
|
|
57
|
+
/** Cases whose prompt carried no vocabulary to score. */
|
|
58
|
+
readonly unmeasurable: readonly SkillCase[]
|
|
59
|
+
}
|
|
60
|
+
| { readonly kind: 'refused'; readonly reason: RankRefusal }
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Every shipped skill's frontmatter description, read the way a prompt is
|
|
64
|
+
* matched against it: whole, including the quoted trigger phrases it states.
|
|
65
|
+
* A skill whose frontmatter carries no description contributes no vocabulary
|
|
66
|
+
* and never wins a rank, so it is dropped rather than scored on nothing.
|
|
67
|
+
*/
|
|
68
|
+
export function loadCatalog(root: string): RankedSkill[] {
|
|
69
|
+
return listSkills(root)
|
|
70
|
+
.filter((skill) => skill.description !== '')
|
|
71
|
+
.map((skill) => ({ name: skill.name, description: skill.description }))
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function tokenize(text: string): string[] {
|
|
75
|
+
return text
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(/[^a-z0-9\s-]/g, ' ')
|
|
78
|
+
.split(/\s+/)
|
|
79
|
+
.filter((token) => token.length > 2 && !STOP.has(token))
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function termCounts(tokens: readonly string[]): Map<string, number> {
|
|
83
|
+
const counts = new Map<string, number>()
|
|
84
|
+
for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1)
|
|
85
|
+
return counts
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function buildIdf(
|
|
89
|
+
docs: readonly Map<string, number>[],
|
|
90
|
+
): (term: string) => number {
|
|
91
|
+
const total = docs.length
|
|
92
|
+
const documentFrequency = new Map<string, number>()
|
|
93
|
+
for (const doc of docs) {
|
|
94
|
+
for (const term of doc.keys()) {
|
|
95
|
+
documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return (term) =>
|
|
99
|
+
Math.log((total + 1) / ((documentFrequency.get(term) ?? 0) + 1)) + 1
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function tfIdfVector(
|
|
103
|
+
terms: Map<string, number>,
|
|
104
|
+
idf: (term: string) => number,
|
|
105
|
+
): Map<string, number> {
|
|
106
|
+
const vec = new Map<string, number>()
|
|
107
|
+
for (const [term, frequency] of terms) vec.set(term, frequency * idf(term))
|
|
108
|
+
return vec
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function cosineSimilarity(
|
|
112
|
+
a: Map<string, number>,
|
|
113
|
+
b: Map<string, number>,
|
|
114
|
+
): number {
|
|
115
|
+
let dot = 0
|
|
116
|
+
for (const [term, weight] of a) dot += weight * (b.get(term) ?? 0)
|
|
117
|
+
if (dot === 0) return 0
|
|
118
|
+
|
|
119
|
+
const norm = (vec: Map<string, number>) =>
|
|
120
|
+
Math.sqrt(
|
|
121
|
+
[...vec.values()].reduce((sum, weight) => sum + weight * weight, 0),
|
|
122
|
+
)
|
|
123
|
+
const denominator = norm(a) * norm(b)
|
|
124
|
+
return denominator === 0 ? 0 : dot / denominator
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export interface RankedResult {
|
|
128
|
+
readonly name: string
|
|
129
|
+
readonly score: number
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Every skill's TF-IDF vector over the catalog it was loaded with, and the
|
|
134
|
+
* IDF weighting that both the catalog and a scored prompt read from. A model
|
|
135
|
+
* is built once per catalog and reused across every case, since the IDF
|
|
136
|
+
* weights are the same question asked of the same corpus each time.
|
|
137
|
+
*/
|
|
138
|
+
export interface RankModel {
|
|
139
|
+
readonly rank: (prompt: string) => readonly RankedResult[]
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function buildModel(catalog: readonly RankedSkill[]): RankModel {
|
|
143
|
+
const docs = catalog.map((skill) => termCounts(tokenize(skill.description)))
|
|
144
|
+
const idf = buildIdf(docs)
|
|
145
|
+
const vectors = new Map(
|
|
146
|
+
catalog.map((skill, index) => [skill.name, tfIdfVector(docs[index], idf)]),
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
rank: (prompt: string): readonly RankedResult[] => {
|
|
151
|
+
const promptVector = tfIdfVector(termCounts(tokenize(prompt)), idf)
|
|
152
|
+
// A prompt built entirely from stopwords and short words tokenizes to
|
|
153
|
+
// nothing, so every skill would score 0 and the sort would fall through
|
|
154
|
+
// to `localeCompare`, handing the alphabetically first skill a win no
|
|
155
|
+
// description earned. Reporting no ranking at all is what keeps that
|
|
156
|
+
// tie-break from reading as a measurement.
|
|
157
|
+
if (promptVector.size === 0) return []
|
|
158
|
+
|
|
159
|
+
return [...vectors]
|
|
160
|
+
.map(([name, vector]) => ({
|
|
161
|
+
name,
|
|
162
|
+
score: cosineSimilarity(promptVector, vector),
|
|
163
|
+
}))
|
|
164
|
+
.sort(
|
|
165
|
+
(left, right) =>
|
|
166
|
+
right.score - left.score || left.name.localeCompare(right.name),
|
|
167
|
+
)
|
|
168
|
+
},
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Scores every case against the catalog's own descriptions, which is the
|
|
174
|
+
* production mechanism: Claude Code matches a prompt against the whole
|
|
175
|
+
* description a skill ships, triggers included.
|
|
176
|
+
*/
|
|
177
|
+
export function measureCases(
|
|
178
|
+
catalog: readonly RankedSkill[],
|
|
179
|
+
cases: readonly SkillCase[],
|
|
180
|
+
): {
|
|
181
|
+
readonly rank1: number
|
|
182
|
+
readonly top3: number
|
|
183
|
+
readonly misses: readonly Miss[]
|
|
184
|
+
readonly unmeasurable: readonly SkillCase[]
|
|
185
|
+
} {
|
|
186
|
+
const model = buildModel(catalog)
|
|
187
|
+
let rank1 = 0
|
|
188
|
+
let top3 = 0
|
|
189
|
+
const misses: Miss[] = []
|
|
190
|
+
const unmeasurable: SkillCase[] = []
|
|
191
|
+
|
|
192
|
+
for (const skillCase of cases) {
|
|
193
|
+
const ranked = model.rank(skillCase.prompt)
|
|
194
|
+
if (ranked.length === 0) {
|
|
195
|
+
unmeasurable.push(skillCase)
|
|
196
|
+
continue
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const at = ranked.findIndex((entry) => entry.name === skillCase.expect) + 1
|
|
200
|
+
|
|
201
|
+
if (at === 1) rank1 += 1
|
|
202
|
+
if (at > 0 && at <= 3) top3 += 1
|
|
203
|
+
if (at !== 1) {
|
|
204
|
+
misses.push({
|
|
205
|
+
prompt: skillCase.prompt,
|
|
206
|
+
expect: skillCase.expect,
|
|
207
|
+
won: ranked[0]?.name ?? '',
|
|
208
|
+
rank: at,
|
|
209
|
+
})
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return { rank1, top3, misses, unmeasurable }
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Reads the shipped catalog off disk and scores it against the given case
|
|
218
|
+
* corpus. Measures the cwd's catalog rather than the toolkit root, matching
|
|
219
|
+
* the reach and audit verbs, so a linked worktree reads its own branch.
|
|
220
|
+
*/
|
|
221
|
+
export function scanRank(
|
|
222
|
+
root: string,
|
|
223
|
+
cases: readonly SkillCase[],
|
|
224
|
+
): RankReport {
|
|
225
|
+
const skillsRoot = join(root, SKILLS_DIR)
|
|
226
|
+
if (!existsSync(skillsRoot)) return { kind: 'refused', reason: 'no-skills' }
|
|
227
|
+
|
|
228
|
+
const catalog = loadCatalog(root)
|
|
229
|
+
const { rank1, top3, misses, unmeasurable } = measureCases(catalog, cases)
|
|
230
|
+
|
|
231
|
+
return {
|
|
232
|
+
kind: 'measured',
|
|
233
|
+
skills: catalog.length,
|
|
234
|
+
cases: cases.length,
|
|
235
|
+
rank1,
|
|
236
|
+
top3,
|
|
237
|
+
misses,
|
|
238
|
+
unmeasurable,
|
|
239
|
+
}
|
|
240
|
+
}
|
package/src/commands/claude.ts
CHANGED
|
@@ -34,6 +34,12 @@ import {
|
|
|
34
34
|
type ReachReport,
|
|
35
35
|
scanReach,
|
|
36
36
|
} from '@/claude/skills-reach'
|
|
37
|
+
import { SKILL_CASES } from '@/claude/cases/all'
|
|
38
|
+
import {
|
|
39
|
+
type RankRefusal,
|
|
40
|
+
type RankReport,
|
|
41
|
+
scanRank,
|
|
42
|
+
} from '@/claude/skills-rank'
|
|
37
43
|
import {
|
|
38
44
|
planSettings,
|
|
39
45
|
readSettings,
|
|
@@ -85,6 +91,10 @@ interface SkillsReachOptions {
|
|
|
85
91
|
readonly json?: boolean
|
|
86
92
|
}
|
|
87
93
|
|
|
94
|
+
interface SkillsRankOptions {
|
|
95
|
+
readonly json?: boolean
|
|
96
|
+
}
|
|
97
|
+
|
|
88
98
|
interface RoutingOptions {
|
|
89
99
|
readonly json?: boolean
|
|
90
100
|
}
|
|
@@ -215,15 +225,18 @@ export function register(program: Command): void {
|
|
|
215
225
|
|
|
216
226
|
const skills = claude
|
|
217
227
|
.command('skills')
|
|
218
|
-
.description('Plugin skill catalog (list, audit, drift, reach)')
|
|
219
|
-
.argument(
|
|
228
|
+
.description('Plugin skill catalog (list, audit, drift, reach, rank)')
|
|
229
|
+
.argument(
|
|
230
|
+
'[subcommand]',
|
|
231
|
+
"One of 'list', 'audit', 'drift', 'reach', or 'rank'",
|
|
232
|
+
)
|
|
220
233
|
.helpOption('-h, --help', 'Show this help message')
|
|
221
234
|
.action((subcommand: string | undefined) => {
|
|
222
235
|
intro('aitk claude')
|
|
223
236
|
logError(
|
|
224
237
|
subcommand === undefined
|
|
225
|
-
? "Missing subcommand. Use 'list', 'audit', 'drift', or '
|
|
226
|
-
: `Unknown subcommand: ${subcommand}. Use 'list', 'audit', 'drift', or '
|
|
238
|
+
? "Missing subcommand. Use 'list', 'audit', 'drift', 'reach', or 'rank'."
|
|
239
|
+
: `Unknown subcommand: ${subcommand}. Use 'list', 'audit', 'drift', 'reach', or 'rank'.`,
|
|
227
240
|
)
|
|
228
241
|
outro()
|
|
229
242
|
process.exitCode = 1
|
|
@@ -351,6 +364,42 @@ export function register(program: Command): void {
|
|
|
351
364
|
.action((path: string | undefined, opts: SkillsReachOptions) => {
|
|
352
365
|
process.exitCode = runSkillsReach(path, opts)
|
|
353
366
|
})
|
|
367
|
+
|
|
368
|
+
skills
|
|
369
|
+
.command('rank')
|
|
370
|
+
.description('Score the shipped catalog against the routing case corpus')
|
|
371
|
+
.argument('[path]', 'Repository root, defaulting to the current directory')
|
|
372
|
+
.helpOption('-h, --help', 'Show this help message')
|
|
373
|
+
.option('--json', 'Add a machine-readable record on stdout')
|
|
374
|
+
.addHelpText(
|
|
375
|
+
'after',
|
|
376
|
+
[
|
|
377
|
+
'',
|
|
378
|
+
'Scope:',
|
|
379
|
+
' TF-IDF cosine similarity over every claude/skills/*/SKILL.md',
|
|
380
|
+
' frontmatter description, scored against the hand-authored corpus',
|
|
381
|
+
' at src/claude/cases/. A necessary condition rather than a report of',
|
|
382
|
+
' real routing behavior: it asks whether the descriptions are',
|
|
383
|
+
' separable by the words they use, and Claude Code does not route',
|
|
384
|
+
' this way.',
|
|
385
|
+
'',
|
|
386
|
+
'Exit codes:',
|
|
387
|
+
' 0 the catalog was read, whether or not a case missed rank one',
|
|
388
|
+
' 1 refused, with the reason on stderr',
|
|
389
|
+
'',
|
|
390
|
+
'Reports rather than gates. The corpus is a first run with no',
|
|
391
|
+
'baseline to fail a push against, so `aitk audits run` registers',
|
|
392
|
+
'this with no gating exit and joins the ratchet instead.',
|
|
393
|
+
'',
|
|
394
|
+
'Examples:',
|
|
395
|
+
' aitk claude skills rank',
|
|
396
|
+
' aitk claude skills rank --json',
|
|
397
|
+
'',
|
|
398
|
+
].join('\n'),
|
|
399
|
+
)
|
|
400
|
+
.action((path: string | undefined, opts: SkillsRankOptions) => {
|
|
401
|
+
process.exitCode = runSkillsRank(path, opts)
|
|
402
|
+
})
|
|
354
403
|
}
|
|
355
404
|
|
|
356
405
|
function succeed(message: string): number {
|
|
@@ -816,6 +865,101 @@ function reportReach(report: Extract<ReachReport, { kind: 'measured' }>): void {
|
|
|
816
865
|
)
|
|
817
866
|
}
|
|
818
867
|
|
|
868
|
+
/** What a reader does about the one way the measure fails to build. */
|
|
869
|
+
const RANK_REFUSALS: Record<RankRefusal, string> = {
|
|
870
|
+
'no-skills':
|
|
871
|
+
'No claude/skills/ here, so this tree ships no plugin body to measure.',
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Measures the cwd rather than the toolkit root, matching the reach and audit
|
|
876
|
+
* verbs, so a linked worktree reads its own branch instead of `main`. The
|
|
877
|
+
* case corpus is the toolkit's own, since a target project ships no cases of
|
|
878
|
+
* its own for a catalog it did not author.
|
|
879
|
+
*/
|
|
880
|
+
function runSkillsRank(
|
|
881
|
+
path: string | undefined,
|
|
882
|
+
opts: SkillsRankOptions,
|
|
883
|
+
): number {
|
|
884
|
+
const root = resolve(path ?? process.cwd())
|
|
885
|
+
const report = scanRank(root, SKILL_CASES)
|
|
886
|
+
|
|
887
|
+
if (report.kind === 'refused') {
|
|
888
|
+
frameError(RANK_REFUSALS[report.reason])
|
|
889
|
+
if (opts.json) {
|
|
890
|
+
process.stdout.write(
|
|
891
|
+
`${JSON.stringify({
|
|
892
|
+
root,
|
|
893
|
+
reason: report.reason,
|
|
894
|
+
message: RANK_REFUSALS[report.reason],
|
|
895
|
+
})}\n`,
|
|
896
|
+
)
|
|
897
|
+
}
|
|
898
|
+
return 1
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
intro('aitk claude skills rank')
|
|
902
|
+
reportRank(report)
|
|
903
|
+
outro()
|
|
904
|
+
|
|
905
|
+
if (opts.json) {
|
|
906
|
+
process.stdout.write(
|
|
907
|
+
`${JSON.stringify({
|
|
908
|
+
root,
|
|
909
|
+
skills: report.skills,
|
|
910
|
+
cases: report.cases,
|
|
911
|
+
rank1: report.rank1,
|
|
912
|
+
top3: report.top3,
|
|
913
|
+
misses: report.misses,
|
|
914
|
+
unmeasurable: report.unmeasurable,
|
|
915
|
+
})}\n`,
|
|
916
|
+
)
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
return 0
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
/**
|
|
923
|
+
* States the corpus and both counts on every run, including a clean one. A
|
|
924
|
+
* miss list alone reads as a verdict on the catalog unless the run also says
|
|
925
|
+
* how many skills and cases it measured against.
|
|
926
|
+
*/
|
|
927
|
+
function reportRank(report: Extract<RankReport, { kind: 'measured' }>): void {
|
|
928
|
+
logStep('Corpus')
|
|
929
|
+
logInfo(
|
|
930
|
+
`${plural(report.skills, 'skill')} scored against ${plural(report.cases, 'case')}`,
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
logStep('Score')
|
|
934
|
+
logInfo(
|
|
935
|
+
`rank one: ${report.rank1}/${report.cases}, top three: ${report.top3}/${report.cases}`,
|
|
936
|
+
)
|
|
937
|
+
|
|
938
|
+
if (report.unmeasurable.length > 0) {
|
|
939
|
+
logWarn(plural(report.unmeasurable.length, 'unmeasurable case'))
|
|
940
|
+
pipeOutput(
|
|
941
|
+
report.unmeasurable
|
|
942
|
+
.map((skillCase) => `${skillCase.expect} ${skillCase.prompt}`)
|
|
943
|
+
.join('\n'),
|
|
944
|
+
)
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
if (report.misses.length === 0) {
|
|
948
|
+
logInfo('Every measurable case ranked its expected skill first.')
|
|
949
|
+
return
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
logWarn(plural(report.misses.length, 'collision'))
|
|
953
|
+
pipeOutput(
|
|
954
|
+
report.misses
|
|
955
|
+
.map(
|
|
956
|
+
(miss) =>
|
|
957
|
+
`${miss.expect} lost to ${miss.won} (rank ${miss.rank}) ${miss.prompt}`,
|
|
958
|
+
)
|
|
959
|
+
.join('\n'),
|
|
960
|
+
)
|
|
961
|
+
}
|
|
962
|
+
|
|
819
963
|
/**
|
|
820
964
|
* States the bound on every run, including the run that names nothing. A report
|
|
821
965
|
* listing only what moved reads as a verdict on what a session holds, and the
|