@erclx/aitk 3.15.0 → 3.17.0

package/src/claude/cases/setup.ts

@@ -0,0 +1,78 @@
+import type { SkillCase } from '@/claude/skills-rank'
+
+/**
+ * `setup-*`, `migration-*`, `toolkit-*`, and `create-rule`: scaffolding,
+ * proposal-only migrations, and the toolkit's own reference and feedback
+ * surfaces.
+ */
+export const SETUP_CASES: readonly SkillCase[] = [
+  {
+    prompt:
+      'This project has no rules installed yet, get the right governance in place.',
+    expect: 'setup-gov',
+  },
+  {
+    prompt:
+      "Get the index.md system bootstrapped across this project's folders.",
+    expect: 'setup-indexes',
+  },
+  {
+    prompt:
+      'This is a brand-new project, get the toolkit bootstrapped in one shot.',
+    expect: 'setup-init',
+  },
+  {
+    prompt: 'Get the usual Claude Code plugins provisioned on this machine.',
+    expect: 'setup-plugins',
+  },
+  {
+    prompt:
+      "Run through the generated scaffold's scripts and confirm each one passes.",
+    expect: 'setup-verify',
+  },
+  {
+    prompt:
+      'This CLAUDE.md file has grown huge, break it apart into the tiered context model.',
+    expect: 'migration-claude-md',
+  },
+  {
+    prompt:
+      'Move the agent-flavored docs out of the docs folder and into context.',
+    expect: 'migration-context',
+  },
+  {
+    prompt:
+      'The snippets folder needs to move under .claude to match the current layout.',
+    expect: 'migration-standards',
+  },
+  {
+    prompt:
+      'This file was replaced by a folder, help me split its content into it.',
+    expect: 'migration-superseded',
+  },
+  {
+    prompt:
+      "Before I run this sync, tell me exactly what it's going to overwrite.",
+    expect: 'toolkit-cli',
+  },
+  {
+    prompt:
+      'Something about the toolkit itself is broken, write it up and send it back to the maintainers.',
+    expect: 'toolkit-feedback',
+  },
+  {
+    prompt:
+      "I don't know which specific toolkit skill I need, just handle it for me.",
+    expect: 'toolkit-operator',
+  },
+  {
+    prompt:
+      'Work through the open feedback issues on the toolkit repo one by one.',
+    expect: 'toolkit-triage',
+  },
+  {
+    prompt:
+      "This project needs its own coding rule that the toolkit doesn't ship.",
+    expect: 'create-rule',
+  },
+]

package/src/claude/skills-rank.ts

@@ -0,0 +1,240 @@
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { listSkills } from '@/claude/skills-list'
+
+/**
+ * Whether a prompt reaches the right skill, measured by TF-IDF cosine
+ * similarity over the shipped catalog's own frontmatter descriptions. This is
+ * a necessary condition rather than a report of real routing behavior: it
+ * asks whether the descriptions are separable by the words they use, and
+ * Claude Code does not route this way.
+ *
+ * Ported from `.claude/groundwork/42-ai-blueprint/evidence/rank.ts`, which
+ * ran once against this catalog and named the collisions this measure now
+ * tracks on a cadence.
+ */
+
+const SKILLS_DIR = join('claude', 'skills')
+
+const STOP = new Set(
+  'a about after again all also and any are as at be before but by can do does for from has have help how in into is it its just make not of on or our so that the then there these this to use used uses using want was what when where which who why with you your run'.split(
+    ' ',
+  ),
+)
+
+export interface RankedSkill {
+  readonly name: string
+  readonly description: string
+}
+
+/** One prompt and the skill it should reach. */
+export interface SkillCase {
+  readonly prompt: string
+  readonly expect: string
+}
+
+/** A case whose prompt did not rank its expected skill first. */
+export interface Miss {
+  readonly prompt: string
+  readonly expect: string
+  /** The skill the ranker placed first instead. */
+  readonly won: string
+  /** Where `expect` placed, or 0 when it never appears in the catalog. */
+  readonly rank: number
+}
+
+/** Why a measure produced no reading, which is never the same as a clean one. */
+export type RankRefusal = 'no-skills'
+
+export type RankReport =
+  | {
+      readonly kind: 'measured'
+      readonly skills: number
+      readonly cases: number
+      readonly rank1: number
+      readonly top3: number
+      readonly misses: readonly Miss[]
+      /** Cases whose prompt carried no vocabulary to score. */
+      readonly unmeasurable: readonly SkillCase[]
+    }
+  | { readonly kind: 'refused'; readonly reason: RankRefusal }
+
+/**
+ * Every shipped skill's frontmatter description, read the way a prompt is
+ * matched against it: whole, including the quoted trigger phrases it states.
+ * A skill whose frontmatter carries no description contributes no vocabulary
+ * and never wins a rank, so it is dropped rather than scored on nothing.
+ */
+export function loadCatalog(root: string): RankedSkill[] {
+  return listSkills(root)
+    .filter((skill) => skill.description !== '')
+    .map((skill) => ({ name: skill.name, description: skill.description }))
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length > 2 && !STOP.has(token))
+}
+
+function termCounts(tokens: readonly string[]): Map<string, number> {
+  const counts = new Map<string, number>()
+  for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1)
+  return counts
+}
+
+function buildIdf(
+  docs: readonly Map<string, number>[],
+): (term: string) => number {
+  const total = docs.length
+  const documentFrequency = new Map<string, number>()
+  for (const doc of docs) {
+    for (const term of doc.keys()) {
+      documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1)
+    }
+  }
+  return (term) =>
+    Math.log((total + 1) / ((documentFrequency.get(term) ?? 0) + 1)) + 1
+}
+
+function tfIdfVector(
+  terms: Map<string, number>,
+  idf: (term: string) => number,
+): Map<string, number> {
+  const vec = new Map<string, number>()
+  for (const [term, frequency] of terms) vec.set(term, frequency * idf(term))
+  return vec
+}
+
+function cosineSimilarity(
+  a: Map<string, number>,
+  b: Map<string, number>,
+): number {
+  let dot = 0
+  for (const [term, weight] of a) dot += weight * (b.get(term) ?? 0)
+  if (dot === 0) return 0
+
+  const norm = (vec: Map<string, number>) =>
+    Math.sqrt(
+      [...vec.values()].reduce((sum, weight) => sum + weight * weight, 0),
+    )
+  const denominator = norm(a) * norm(b)
+  return denominator === 0 ? 0 : dot / denominator
+}
+
+export interface RankedResult {
+  readonly name: string
+  readonly score: number
+}
+
+/**
+ * Every skill's TF-IDF vector over the catalog it was loaded with, and the
+ * IDF weighting that both the catalog and a scored prompt read from. A model
+ * is built once per catalog and reused across every case, since the IDF
+ * weights are the same question asked of the same corpus each time.
+ */
+export interface RankModel {
+  readonly rank: (prompt: string) => readonly RankedResult[]
+}
+
+export function buildModel(catalog: readonly RankedSkill[]): RankModel {
+  const docs = catalog.map((skill) => termCounts(tokenize(skill.description)))
+  const idf = buildIdf(docs)
+  const vectors = new Map(
+    catalog.map((skill, index) => [skill.name, tfIdfVector(docs[index], idf)]),
+  )
+
+  return {
+    rank: (prompt: string): readonly RankedResult[] => {
+      const promptVector = tfIdfVector(termCounts(tokenize(prompt)), idf)
+      // A prompt built entirely from stopwords and short words tokenizes to
+      // nothing, so every skill would score 0 and the sort would fall through
+      // to `localeCompare`, handing the alphabetically first skill a win no
+      // description earned. Reporting no ranking at all is what keeps that
+      // tie-break from reading as a measurement.
+      if (promptVector.size === 0) return []
+
+      return [...vectors]
+        .map(([name, vector]) => ({
+          name,
+          score: cosineSimilarity(promptVector, vector),
+        }))
+        .sort(
+          (left, right) =>
+            right.score - left.score || left.name.localeCompare(right.name),
+        )
+    },
+  }
+}
+
+/**
+ * Scores every case against the catalog's own descriptions, which is the
+ * production mechanism: Claude Code matches a prompt against the whole
+ * description a skill ships, triggers included.
+ */
+export function measureCases(
+  catalog: readonly RankedSkill[],
+  cases: readonly SkillCase[],
+): {
+  readonly rank1: number
+  readonly top3: number
+  readonly misses: readonly Miss[]
+  readonly unmeasurable: readonly SkillCase[]
+} {
+  const model = buildModel(catalog)
+  let rank1 = 0
+  let top3 = 0
+  const misses: Miss[] = []
+  const unmeasurable: SkillCase[] = []
+
+  for (const skillCase of cases) {
+    const ranked = model.rank(skillCase.prompt)
+    if (ranked.length === 0) {
+      unmeasurable.push(skillCase)
+      continue
+    }
+
+    const at = ranked.findIndex((entry) => entry.name === skillCase.expect) + 1
+
+    if (at === 1) rank1 += 1
+    if (at > 0 && at <= 3) top3 += 1
+    if (at !== 1) {
+      misses.push({
+        prompt: skillCase.prompt,
+        expect: skillCase.expect,
+        won: ranked[0]?.name ?? '',
+        rank: at,
+      })
+    }
+  }
+
+  return { rank1, top3, misses, unmeasurable }
+}
+
+/**
+ * Reads the shipped catalog off disk and scores it against the given case
+ * corpus. Measures the cwd's catalog rather than the toolkit root, matching
+ * the reach and audit verbs, so a linked worktree reads its own branch.
+ */
+export function scanRank(
+  root: string,
+  cases: readonly SkillCase[],
+): RankReport {
+  const skillsRoot = join(root, SKILLS_DIR)
+  if (!existsSync(skillsRoot)) return { kind: 'refused', reason: 'no-skills' }
+
+  const catalog = loadCatalog(root)
+  const { rank1, top3, misses, unmeasurable } = measureCases(catalog, cases)
+
+  return {
+    kind: 'measured',
+    skills: catalog.length,
+    cases: cases.length,
+    rank1,
+    top3,
+    misses,
+    unmeasurable,
+  }
+}

package/src/commands/claude.ts

@@ -34,6 +34,12 @@ import {
   type ReachReport,
   scanReach,
 } from '@/claude/skills-reach'
+import { SKILL_CASES } from '@/claude/cases/all'
+import {
+  type RankRefusal,
+  type RankReport,
+  scanRank,
+} from '@/claude/skills-rank'
 import {
   planSettings,
   readSettings,
@@ -85,6 +91,10 @@ interface SkillsReachOptions {
   readonly json?: boolean
 }
 
+interface SkillsRankOptions {
+  readonly json?: boolean
+}
+
 interface RoutingOptions {
   readonly json?: boolean
 }
@@ -215,15 +225,18 @@ export function register(program: Command): void {
 
   const skills = claude
     .command('skills')
-    .description('Plugin skill catalog (list, audit, drift, reach)')
-    .argument('[subcommand]', "One of 'list', 'audit', 'drift', or 'reach'")
+    .description('Plugin skill catalog (list, audit, drift, reach, rank)')
+    .argument(
+      '[subcommand]',
+      "One of 'list', 'audit', 'drift', 'reach', or 'rank'",
+    )
     .helpOption('-h, --help', 'Show this help message')
     .action((subcommand: string | undefined) => {
       intro('aitk claude')
       logError(
         subcommand === undefined
-          ? "Missing subcommand. Use 'list', 'audit', 'drift', or 'reach'."
-          : `Unknown subcommand: ${subcommand}. Use 'list', 'audit', 'drift', or 'reach'.`,
+          ? "Missing subcommand. Use 'list', 'audit', 'drift', 'reach', or 'rank'."
+          : `Unknown subcommand: ${subcommand}. Use 'list', 'audit', 'drift', 'reach', or 'rank'.`,
       )
       outro()
       process.exitCode = 1
@@ -351,6 +364,42 @@ export function register(program: Command): void {
     .action((path: string | undefined, opts: SkillsReachOptions) => {
       process.exitCode = runSkillsReach(path, opts)
     })
+
+  skills
+    .command('rank')
+    .description('Score the shipped catalog against the routing case corpus')
+    .argument('[path]', 'Repository root, defaulting to the current directory')
+    .helpOption('-h, --help', 'Show this help message')
+    .option('--json', 'Add a machine-readable record on stdout')
+    .addHelpText(
+      'after',
+      [
+        '',
+        'Scope:',
+        '  TF-IDF cosine similarity over every claude/skills/*/SKILL.md',
+        '  frontmatter description, scored against the hand-authored corpus',
+        '  at src/claude/cases/. A necessary condition rather than a report of',
+        '  real routing behavior: it asks whether the descriptions are',
+        '  separable by the words they use, and Claude Code does not route',
+        '  this way.',
+        '',
+        'Exit codes:',
+        '  0  the catalog was read, whether or not a case missed rank one',
+        '  1  refused, with the reason on stderr',
+        '',
+        'Reports rather than gates. The corpus is a first run with no',
+        'baseline to fail a push against, so `aitk audits run` registers',
+        'this with no gating exit and joins the ratchet instead.',
+        '',
+        'Examples:',
+        '  aitk claude skills rank',
+        '  aitk claude skills rank --json',
+        '',
+      ].join('\n'),
+    )
+    .action((path: string | undefined, opts: SkillsRankOptions) => {
+      process.exitCode = runSkillsRank(path, opts)
+    })
 }
 
 function succeed(message: string): number {
@@ -816,6 +865,101 @@ function reportReach(report: Extract<ReachReport, { kind: 'measured' }>): void {
   )
 }
 
+/** What a reader does about the one way the measure fails to build. */
+const RANK_REFUSALS: Record<RankRefusal, string> = {
+  'no-skills':
+    'No claude/skills/ here, so this tree ships no plugin body to measure.',
+}
+
+/**
+ * Measures the cwd rather than the toolkit root, matching the reach and audit
+ * verbs, so a linked worktree reads its own branch instead of `main`. The
+ * case corpus is the toolkit's own, since a target project ships no cases of
+ * its own for a catalog it did not author.
+ */
+function runSkillsRank(
+  path: string | undefined,
+  opts: SkillsRankOptions,
+): number {
+  const root = resolve(path ?? process.cwd())
+  const report = scanRank(root, SKILL_CASES)
+
+  if (report.kind === 'refused') {
+    frameError(RANK_REFUSALS[report.reason])
+    if (opts.json) {
+      process.stdout.write(
+        `${JSON.stringify({
+          root,
+          reason: report.reason,
+          message: RANK_REFUSALS[report.reason],
+        })}\n`,
+      )
+    }
+    return 1
+  }
+
+  intro('aitk claude skills rank')
+  reportRank(report)
+  outro()
+
+  if (opts.json) {
+    process.stdout.write(
+      `${JSON.stringify({
+        root,
+        skills: report.skills,
+        cases: report.cases,
+        rank1: report.rank1,
+        top3: report.top3,
+        misses: report.misses,
+        unmeasurable: report.unmeasurable,
+      })}\n`,
+    )
+  }
+
+  return 0
+}
+
+/**
+ * States the corpus and both counts on every run, including a clean one. A
+ * miss list alone reads as a verdict on the catalog unless the run also says
+ * how many skills and cases it measured against.
+ */
+function reportRank(report: Extract<RankReport, { kind: 'measured' }>): void {
+  logStep('Corpus')
+  logInfo(
+    `${plural(report.skills, 'skill')} scored against ${plural(report.cases, 'case')}`,
+  )
+
+  logStep('Score')
+  logInfo(
+    `rank one: ${report.rank1}/${report.cases}, top three: ${report.top3}/${report.cases}`,
+  )
+
+  if (report.unmeasurable.length > 0) {
+    logWarn(plural(report.unmeasurable.length, 'unmeasurable case'))
+    pipeOutput(
+      report.unmeasurable
+        .map((skillCase) => `${skillCase.expect}  ${skillCase.prompt}`)
+        .join('\n'),
+    )
+  }
+
+  if (report.misses.length === 0) {
+    logInfo('Every measurable case ranked its expected skill first.')
+    return
+  }
+
+  logWarn(plural(report.misses.length, 'collision'))
+  pipeOutput(
+    report.misses
+      .map(
+        (miss) =>
+          `${miss.expect} lost to ${miss.won} (rank ${miss.rank})  ${miss.prompt}`,
+      )
+      .join('\n'),
+  )
+}
+
 /**
  * States the bound on every run, including the run that names nothing. A report
  * listing only what moved reads as a verdict on what a session holds, and the