@erclx/canon 4.0.0

package/src/sandbox/census.ts

@@ -0,0 +1,228 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { collectCoverage, type CoverageReport } from '@/sandbox/coverage'
+
+/**
+ * Holds the exemption declarations, beside the scenario categories rather than
+ * inside one. `listScenarios` in `coverage.ts` walks directories only, so a file
+ * at that level joins no category and never reads as a scenario.
+ */
+const EXEMPT_FILE = 'exempt.toml'
+
+export type SkillVerdict = 'asserted' | 'should-be-asserted' | 'exempt'
+
+export interface SkillCensusEntry {
+  readonly skill: string
+  readonly verdict: SkillVerdict
+  /**
+   * Every `<category>:<command>` pairing to this skill, empty when none does.
+   * Plural because the mapping is many-to-one: two scenarios can drive one
+   * skill, and naming only the first would credit its arms to the wrong file.
+   */
+  readonly scenarios: readonly string[]
+  /**
+   * Each armed arm as `<category>:<command>/<arm>`. Qualified rather than bare,
+   * because a skill two scenarios drive can hold two arms of the same name and
+   * a bare list renders them as one label twice. Deduplicating instead would
+   * read as one arm where two assert, which understates in the one direction
+   * this report exists to keep honest.
+   */
+  readonly armed: readonly string[]
+  /** Why no arm asserts this skill. Present only on `exempt`. */
+  readonly reason?: string
+}
+
+export interface CensusReport {
+  readonly skills: readonly SkillCensusEntry[]
+  readonly totalSkills: number
+  readonly asserted: number
+  readonly shouldBeAsserted: number
+  readonly exempt: number
+  /** Exemptions naming a skill the tree does not carry. */
+  readonly staleExemptions: readonly string[]
+  /**
+   * Exemptions on a skill an arm now asserts. The claim is wrong in the one
+   * direction a verdict decays, and the entry outranks nothing, so dropping it
+   * silently leaves committed data nobody is told to delete.
+   */
+  readonly supersededExemptions: readonly string[]
+}
+
+function directories(path: string): string[] {
+  if (!existsSync(path)) return []
+
+  return readdirSync(path, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort()
+}
+
+/**
+ * Every skill the plugin ships, which is the census denominator. Reads the
+ * authoring root rather than `.claude/skills/`, since the latter holds
+ * toolkit-internal skills that reach no target and would inflate the count.
+ */
+export function listSkills(root: string): string[] {
+  return directories(join(root, 'claude', 'skills'))
+}
+
+/**
+ * Maps a scenario to the skill it drives, trying two spellings in order.
+ *
+ * `<category>-<command>` is the rule `.claude/context/sandbox/coverage.md` states, and it
+ * alone pairs 29 of 54 skills. The bare `<command>` fallback is what reaches the
+ * rest: `claude/setup-init.sh` drives the `setup-init` skill, not a
+ * `claude-setup-init` that does not exist. Stating one spelling and shipping two
+ * is what let the audit report a paired skill as unpaired.
+ *
+ * Returns undefined for a scenario driving no skill at all, which is every
+ * `infra/` and `tooling/` scenario. Those exercise a CLI domain rather than a
+ * skill and belong to the scenario count, not to this one.
+ */
+export function skillForScenario(
+  category: string,
+  command: string,
+  skills: ReadonlySet<string>,
+): string | undefined {
+  const prefixed = `${category}-${command}`
+  if (skills.has(prefixed)) return prefixed
+  if (skills.has(command)) return command
+
+  return undefined
+}
+
+/**
+ * Reads the exemption declarations, keyed by skill with a `reason` each.
+ *
+ * The reason cannot live in the arm's `manual` array, which is where `#723` put
+ * prose a checker could not assert. `resolveVerdict` fails any declaration
+ * carrying zero mechanical assertions, so an `expect.toml` holding only an
+ * exempt reason goes red at the moment it is written. An exempt skill has no
+ * assertion to pair the prose with, which is what makes it exempt, so the two
+ * cases cannot share a home.
+ *
+ * Throws on a malformed file and on a table carrying no usable `reason`, rather
+ * than reporting a smaller set. Both losses are invisible downstream: the skill
+ * reclassifies to should-be-asserted and rejoins a work queue someone already
+ * ruled it out of, with nothing naming the entry that stopped counting. This is
+ * the stray-key rule `contentArray` applies in `expect.ts` for the same reason,
+ * where the declaration is well-formed and silently lost a key it appears to
+ * carry. `runCoverage` catches both and frames them.
+ */
+export function parseExemptions(source: string): Map<string, string> {
+  const parsed = Bun.TOML.parse(source) as Record<string, unknown>
+  const exemptions = new Map<string, string>()
+
+  for (const [skill, value] of Object.entries(parsed)) {
+    if (typeof value !== 'object' || value === null) {
+      throw new Error(
+        `exemption ${skill} is not a table. Write [${skill}] with a reason below it.`,
+      )
+    }
+
+    const reason = (value as Record<string, unknown>).reason
+    if (typeof reason !== 'string' || reason === '') {
+      throw new Error(
+        `exemption ${skill} declares no reason. An exemption without one cannot be checked or overturned.`,
+      )
+    }
+
+    exemptions.set(skill, reason)
+  }
+
+  return exemptions
+}
+
+function readExemptions(root: string): Map<string, string> {
+  const path = join(root, 'scripts', 'sandbox', EXEMPT_FILE)
+  if (!existsSync(path)) return new Map()
+
+  return parseExemptions(readFileSync(path, 'utf8'))
+}
+
+/**
+ * Gives every shipped skill an asserted, should-be-asserted, or exempt verdict
+ * against declared expectations rather than against a scenario existing.
+ *
+ * A paired scenario is not an asserted skill. A scenario with no `expect.toml`
+ * provisions a state and prints a human-readable `Expect:` line, so a run over
+ * it exits zero having proved nothing. Counting pairs would report coverage the
+ * suite does not have, which is the measure this report exists to replace.
+ *
+ * An armed arm outranks an exemption. A skill listed exempt that acquired an arm
+ * is asserted in fact, and reporting the stale claim instead would hide the one
+ * direction the verdict decays in.
+ */
+export function collectCensus(
+  root: string,
+  coverage: CoverageReport = collectCoverage(root),
+): CensusReport {
+  const skills = listSkills(root)
+  const known = new Set(skills)
+  const exemptions = readExemptions(root)
+
+  const pairings = new Map<string, { scenarios: string[]; armed: string[] }>()
+  for (const scenario of coverage.scenarios) {
+    const skill = skillForScenario(scenario.category, scenario.command, known)
+    if (skill === undefined) continue
+
+    // Accumulate rather than assign. Dropping a second scenario would report a
+    // skill unarmed while an arm under the other spelling asserts it.
+    const pair = `${scenario.category}:${scenario.command}`
+    const existing = pairings.get(skill) ?? { scenarios: [], armed: [] }
+    existing.scenarios.push(pair)
+    existing.armed.push(...scenario.armed.map((arm) => `${pair}/${arm}`))
+    pairings.set(skill, existing)
+  }
+
+  const entries = skills.map((skill): SkillCensusEntry => {
+    const pairing = pairings.get(skill)
+    const scenarios = pairing?.scenarios ?? []
+    const armed = pairing?.armed ?? []
+    const reason = exemptions.get(skill)
+
+    if (armed.length > 0) {
+      return { skill, verdict: 'asserted', scenarios, armed }
+    }
+
+    if (reason !== undefined) {
+      return { skill, verdict: 'exempt', scenarios, armed, reason }
+    }
+
+    return { skill, verdict: 'should-be-asserted', scenarios, armed }
+  })
+
+  // An exemption outranked by an arm is as wrong as one naming no skill, and it
+  // is the case the armed-wins rule creates rather than one the tree arrives
+  // with. Reading `entries` rather than recomputing keeps the two in step, so a
+  // change to the precedence cannot leave the report contradicting the verdict.
+  const assertedSkills = new Set(
+    entries.filter((e) => e.verdict === 'asserted').map((e) => e.skill),
+  )
+
+  return {
+    skills: entries,
+    totalSkills: entries.length,
+    asserted: assertedSkills.size,
+    shouldBeAsserted: entries.filter((e) => e.verdict === 'should-be-asserted')
+      .length,
+    exempt: entries.filter((e) => e.verdict === 'exempt').length,
+    staleExemptions: [...exemptions.keys()]
+      .filter((skill) => !known.has(skill))
+      .sort(),
+    supersededExemptions: [...exemptions.keys()]
+      .filter((skill) => assertedSkills.has(skill))
+      .sort(),
+  }
+}
+
+/**
+ * Whole percent, floored, over skills rather than scenarios. Floored for the
+ * reason `coveragePercent` is: the number exists to look worse than a wall of
+ * green does.
+ */
+export function assertedPercent(report: CensusReport): number {
+  if (report.totalSkills === 0) return 0
+
+  return Math.floor((report.asserted / report.totalSkills) * 100)
+}

package/src/sandbox/coverage.ts

@@ -0,0 +1,115 @@
+import { existsSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+
+/**
+ * The arm a scenario declares when it carries no named arm. `expectFilePath`
+ * joins an empty arm away, so the declaration lands at the command root and has
+ * no name to print.
+ */
+export const DEFAULT_ARM = '(default)'
+
+/** Holds fixture content rather than scenarios. Twin of the filter in `sandbox.ts`. */
+const FIXTURES_DIR = 'fixtures'
+
+export interface ScenarioCoverage {
+  readonly category: string
+  readonly command: string
+  readonly armed: readonly string[]
+}
+
+export interface CoverageReport {
+  readonly scenarios: readonly ScenarioCoverage[]
+  readonly totalScenarios: number
+  readonly armedScenarios: number
+  readonly armedArms: number
+}
+
+function directories(path: string): string[] {
+  if (!existsSync(path)) return []
+
+  return readdirSync(path, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort()
+}
+
+/**
+ * Enumerates from the scenario scripts rather than from the fixture tree. A
+ * scenario with no fixtures has no directory to find, and counting only what
+ * fixtures exist would hide exactly the arms this report exists to surface.
+ */
+function listScenarios(sandboxDir: string): ScenarioCoverage[] {
+  const scenarios: ScenarioCoverage[] = []
+
+  for (const category of directories(sandboxDir)) {
+    if (category === FIXTURES_DIR) continue
+
+    const commands = readdirSync(join(sandboxDir, category), {
+      withFileTypes: true,
+    })
+      .filter((entry) => entry.isFile() && entry.name.endsWith('.sh'))
+      .map((entry) => entry.name.replace(/\.sh$/, ''))
+      .sort()
+
+    for (const command of commands) {
+      scenarios.push({ category, command, armed: [] })
+    }
+  }
+
+  return scenarios
+}
+
+/**
+ * Names every arm of a scenario carrying an `expect.toml`. A declaration at the
+ * command root belongs to the unnamed arm and reports under `DEFAULT_ARM`, since
+ * an empty string in a report reads as a bug rather than as the default.
+ */
+function armsFor(
+  fixturesDir: string,
+  category: string,
+  command: string,
+): string[] {
+  const commandDir = join(fixturesDir, category, command)
+  if (!existsSync(commandDir)) return []
+
+  const armed: string[] = []
+  if (existsSync(join(commandDir, 'expect.toml'))) armed.push(DEFAULT_ARM)
+
+  for (const arm of directories(commandDir)) {
+    if (existsSync(join(commandDir, arm, 'expect.toml'))) armed.push(arm)
+  }
+
+  return armed
+}
+
+/**
+ * Reports which scenarios declare expectations and which only provision a state.
+ * The undeclared ones exit zero today, so the count is the only thing separating
+ * a suite that proves something from one that runs.
+ */
+export function collectCoverage(root: string): CoverageReport {
+  const sandboxDir = join(root, 'scripts', 'sandbox')
+  const fixturesDir = join(sandboxDir, FIXTURES_DIR)
+
+  const scenarios = listScenarios(sandboxDir).map((scenario) => ({
+    ...scenario,
+    armed: armsFor(fixturesDir, scenario.category, scenario.command),
+  }))
+
+  return {
+    scenarios,
+    totalScenarios: scenarios.length,
+    armedScenarios: scenarios.filter((s) => s.armed.length > 0).length,
+    armedArms: scenarios.reduce((total, s) => total + s.armed.length, 0),
+  }
+}
+
+/**
+ * Whole percent, floored. A rollout at 13 percent should not round to 13 from
+ * above, since the number exists to look worse than a wall of green does.
+ */
+export function coveragePercent(report: CoverageReport): number {
+  if (report.totalScenarios === 0) return 0
+
+  return Math.floor((report.armedScenarios / report.totalScenarios) * 100)
+}