@erclx/aitk 0.109.0 → 0.110.0

package/src/audits/catalog.ts

@@ -0,0 +1,464 @@
+import type { ValidateRefusal as RecordRefusal } from '@/records/validate'
+import type { ValidateRefusal as BoardRefusal } from '@/tasks/validate'
+
+/**
+ * The audits this repository already owns, and how to read each one's record.
+ *
+ * Every verb here ships its own `--json` shape with its own keys, and this
+ * module reads each shape rather than forcing a common envelope on them. Each
+ * record already has consumers, so an envelope would be a breaking change to
+ * every one of them bought for tidiness.
+ */
+
+/** How far a finding from this audit reaches. */
+export type Corpus =
+  /** Committed, so every clone measures the same files and a delta is shared. */
+  | 'tracked'
+  /** Gitignored session scratch, so the numbers are one machine's alone. */
+  | 'per-machine'
+
+export type AuditStatus =
+  /** The audit reported and every count it produced is zero. */
+  | 'clean'
+  /** The audit reported findings it does not gate on. */
+  | 'reported'
+  /** The audit reported a finding that is a fact, which fails the aggregate. */
+  | 'finding'
+  /**
+   * The per-machine corpus this audit reads is not on this disk, which is the
+   * ordinary state of a gitignored folder on a fresh clone and in CI.
+   */
+  | 'absent'
+  /** The audit did not report, so the aggregate measured less than the set. */
+  | 'unmeasured'
+
+export interface AuditSpec {
+  readonly id: string
+  readonly label: string
+  /** Arguments after `aitk`, always ending in `--json`. */
+  readonly argv: readonly string[]
+  /**
+   * Exit codes this verb sets on a finding that is a fact.
+   *
+   * Empty for every verb whose findings are judgments. The catalog decides
+   * this rather than the exit code, because a reporting verb sets 2 on
+   * findings it deliberately does not gate on, so the code alone cannot say
+   * whether a finding is a fact.
+   */
+  readonly gatingExits: readonly number[]
+  readonly corpus: Corpus
+  /**
+   * Pulls the counts worth retaining out of this verb's record.
+   *
+   * Returns `undefined` when the record does not carry the keys it reads,
+   * which is a measure that did not run rather than a corpus with nothing in
+   * it. Zero there would read as clean, which is the two-states-look-alike
+   * defect this repository has already fixed elsewhere.
+   */
+  readonly counts: (record: unknown) => Record<string, number> | undefined
+}
+
+export interface AuditResult {
+  readonly id: string
+  readonly label: string
+  readonly status: AuditStatus
+  readonly tracked: boolean
+  readonly exitCode: number
+  readonly counts?: Record<string, number>
+  /** Why the audit did not report, present only on `unmeasured`. */
+  readonly reason?: string
+}
+
+/** The exit code every command here sets when it refuses. */
+const EXIT_REFUSED = 1
+
+/** The exit code every command here sets when it carries findings. */
+const EXIT_FINDINGS = 2
+
+function asObject(value: unknown): Record<string, unknown> | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined
+  }
+  return value as Record<string, unknown>
+}
+
+function lengthOf(value: unknown): number | undefined {
+  return Array.isArray(value) ? value.length : undefined
+}
+
+/**
+ * Folds a set of per-key readings into one record, or `undefined` when any of
+ * them could not be read. One unreadable key means the record is not the shape
+ * the extractor was written against, and reporting the rest would publish a
+ * partial count under a name that claims to be whole.
+ */
+function allOf(
+  readings: Record<string, number | undefined>,
+): Record<string, number> | undefined {
+  const counts: Record<string, number> = {}
+  for (const [key, value] of Object.entries(readings)) {
+    if (value === undefined) return undefined
+    counts[key] = value
+  }
+  return counts
+}
+
+function contextCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  const entries = root.entries
+  if (!Array.isArray(entries)) return undefined
+
+  let bareReferences = 0
+  for (const entry of entries) {
+    const bare = lengthOf(asObject(entry)?.bareReferences)
+    if (bare === undefined) return undefined
+    bareReferences += bare
+  }
+
+  return allOf({
+    unresolvedCitations: lengthOf(asObject(root.citations)?.unresolved),
+    longEntries: lengthOf(root.length),
+    missingSections: lengthOf(root.missingSections),
+    indexDrift: lengthOf(root.indexDrift),
+    bareReferences,
+  })
+}
+
+function markdownCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  const entries = root.entries
+  if (!Array.isArray(entries)) return undefined
+
+  const depth = asObject(root.checkpoints)?.run
+  // A depth reading needs the checkpoint it is measured against. Counting zero
+  // without it reports a corpus nobody measured for depth as one with no file
+  // past the line.
+  if (entries.length > 0 && typeof depth !== 'number') return undefined
+
+  let bans = 0
+  let heavyBullets = 0
+  let heavyParagraphs = 0
+  let filesPastDepth = 0
+  let flatParagraphs = 0
+
+  for (const raw of entries) {
+    const entry = asObject(raw)
+    if (entry === undefined) return undefined
+
+    const entryBans = lengthOf(entry.bans)
+    const bullets = lengthOf(entry.heavyBullets)
+    const paragraphs = lengthOf(entry.heavyParagraphs)
+    const run = entry.longestRun
+    if (
+      entryBans === undefined ||
+      bullets === undefined ||
+      paragraphs === undefined ||
+      typeof run !== 'number'
+    ) {
+      return undefined
+    }
+
+    bans += entryBans
+    heavyBullets += bullets
+    heavyParagraphs += paragraphs
+    if (typeof depth === 'number' && run > depth) filesPastDepth += 1
+
+    // A file below the measuring floor carries no cadence key at all, which is
+    // a file that was not measured rather than one with no flat paragraph.
+    const flat = asObject(entry.cadence)?.flat
+    if (typeof flat === 'number') flatParagraphs += flat
+  }
+
+  return { bans, heavyBullets, heavyParagraphs, filesPastDepth, flatParagraphs }
+}
+
+/** The finding arrays `aitk claude skills audit` publishes, in its own order. */
+const SKILL_FINDINGS = [
+  'missingRequirement',
+  'nameMismatch',
+  'missingDescription',
+  'longDescription',
+  'readme',
+  'folderName',
+  'requirementSections',
+] as const
+
+function skillCounts(record: unknown): Record<string, number> | undefined {
+  const findings = asObject(asObject(record)?.findings)
+  if (findings === undefined) return undefined
+
+  return allOf(
+    Object.fromEntries(
+      SKILL_FINDINGS.map((key) => [key, lengthOf(findings[key])]),
+    ),
+  )
+}
+
+function boardCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  // The board keeps its untested rows apart from its findings on purpose, so
+  // they move no exit code. Folding them together here would undo that.
+  return allOf({
+    findings: lengthOf(root.findings),
+    untested: lengthOf(root.untested),
+  })
+}
+
+function findingsOnly(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  return allOf({ findings: lengthOf(root.findings) })
+}
+
+function commentCounts(record: unknown): Record<string, number> | undefined {
+  const snapshot = asObject(record)?.snapshot
+  if (!Array.isArray(snapshot)) return undefined
+
+  let degradationHits = 0
+  for (const language of snapshot) {
+    const hits = lengthOf(asObject(language)?.degradationHits)
+    if (hits === undefined) return undefined
+    degradationHits += hits
+  }
+
+  return { degradationHits }
+}
+
+function testOrderCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  // The unclassified count travels with the findings because it is the honest
+  // shape of this measure: a refactor and a new behavior cannot be told apart
+  // from history, and a findings count alone claims coverage it does not have.
+  return allOf({
+    findings: lengthOf(root.findings),
+    unclassified: lengthOf(root.unclassified),
+  })
+}
+
+/** The record kinds `aitk records validate` takes, and the corpus each reads. */
+const RECORD_KINDS: readonly (readonly [string, Corpus])[] = [
+  ['plans', 'per-machine'],
+  ['groundwork', 'per-machine'],
+  ['intake', 'per-machine'],
+  ['memory', 'per-machine'],
+  ['standards', 'tracked'],
+  ['teach', 'per-machine'],
+]
+
+/**
+ * Every audit the aggregate runs.
+ *
+ * `context`, `markdown`, and `skills` are the three that gate, which is exactly
+ * the set `scripts/core/verify.sh` already fails a push on. Adding a fourth
+ * here widens what fails a push without anyone deciding to, and the split this
+ * repository records gates a fact and reports a judgment.
+ *
+ * Each verb runs once in its fullest form. Running the gating half separately
+ * would walk the same tree twice for a number the full record already carries.
+ */
+export const AUDITS: readonly AuditSpec[] = [
+  {
+    id: 'context',
+    label: 'Context folders',
+    argv: ['context', 'audit', '--json'],
+    gatingExits: [EXIT_FINDINGS],
+    corpus: 'tracked',
+    counts: contextCounts,
+  },
+  {
+    id: 'markdown',
+    label: 'Markdown corpus',
+    argv: ['markdown', 'audit', '--json'],
+    // 3 is the empty ban set, which is the corpus walked with nothing looked
+    // for. That is a broken check rather than a clean tree, so it fails here
+    // the way it already fails the push.
+    gatingExits: [EXIT_FINDINGS, 3],
+    corpus: 'tracked',
+    counts: markdownCounts,
+  },
+  {
+    id: 'skills',
+    label: 'Skill corpora',
+    argv: ['claude', 'skills', 'audit', '--json'],
+    gatingExits: [EXIT_FINDINGS],
+    corpus: 'tracked',
+    counts: skillCounts,
+  },
+  {
+    id: 'tasks',
+    label: 'Task board',
+    argv: ['tasks', 'validate', '--json'],
+    gatingExits: [],
+    corpus: 'per-machine',
+    counts: boardCounts,
+  },
+  ...RECORD_KINDS.map(([kind, corpus]) => ({
+    id: `records-${kind}`,
+    label: `Records: ${kind}`,
+    argv: ['records', 'validate', kind, '--json'],
+    gatingExits: [],
+    corpus,
+    counts: findingsOnly,
+  })),
+  {
+    id: 'comments',
+    label: 'Comment census',
+    argv: ['comments', 'scan', '--json'],
+    gatingExits: [],
+    corpus: 'tracked',
+    counts: commentCounts,
+  },
+  {
+    id: 'test-order',
+    label: 'Test order',
+    argv: ['gov', 'test-order', '--json'],
+    gatingExits: [],
+    corpus: 'tracked',
+    counts: testOrderCounts,
+  },
+]
+
+export function auditFor(id: string): AuditSpec | undefined {
+  return AUDITS.find((audit) => audit.id === id)
+}
+
+export function isTracked(spec: AuditSpec): boolean {
+  return spec.corpus === 'tracked'
+}
+
+export function countsFor(
+  spec: AuditSpec,
+  record: unknown,
+): Record<string, number> | undefined {
+  return spec.counts(record)
+}
+
+/**
+ * Reads the reason a refusing verb published, so the aggregate names the cause
+ * rather than only the exit. Every command here puts it in `reason`, and the
+ * ones that also carry a sentence put that in `message`.
+ */
+function refusalReason(record: unknown, exitCode: number): string {
+  const root = asObject(record)
+  const reason = typeof root?.reason === 'string' ? root.reason : undefined
+  const message = typeof root?.message === 'string' ? root.message : undefined
+
+  if (reason !== undefined && message !== undefined) {
+    return `${reason}: ${message}`
+  }
+  return reason ?? message ?? `exited ${exitCode} with no reason on stdout`
+}
+
+/**
+ * The reasons a verb publishes when the folder it reads is simply not there.
+ *
+ * Typed against the unions those two verbs export rather than spelled as bare
+ * strings, so renaming a reason at its source fails this build. A literal here
+ * would go on matching nothing and quietly turn every expected absence back
+ * into an unmeasured audit.
+ */
+const ABSENT_REASONS: readonly (RecordRefusal | BoardRefusal)[] = [
+  'no-folder',
+  'no-board',
+]
+
+/**
+ * Whether a refusal is a folder this machine never created rather than a break.
+ *
+ * Every gitignored record folder is absent on a fresh clone and in CI, so six
+ * of the twelve refuse there on every run. Counting those as unmeasured pins
+ * the verdict at `incomplete` forever, and a signal that never changes is one
+ * nobody reads after the second time they see it.
+ *
+ * A tracked corpus gets no such allowance. That tree ships to targets, so a
+ * checkout that cannot find it is broken, and reading the absence as ordinary
+ * would report a pass over a corpus nobody measured.
+ */
+function isExpectedAbsence(spec: AuditSpec, record: unknown): boolean {
+  if (spec.corpus !== 'per-machine') return false
+
+  const reason = asObject(record)?.reason
+  return (
+    typeof reason === 'string' &&
+    (ABSENT_REASONS as readonly string[]).includes(reason)
+  )
+}
+
+/**
+ * Turns one verb's exit code and stdout into a result the aggregate can report.
+ *
+ * Unparseable output is `unmeasured` rather than clean, for the reason the
+ * verify pipeline already states about its own skipped stages: a run that does
+ * not report is a broken command, and skipping would report the pass the check
+ * exists to withhold.
+ */
+export function classify(
+  spec: AuditSpec,
+  exitCode: number,
+  stdout: string,
+): AuditResult {
+  const base = {
+    id: spec.id,
+    label: spec.label,
+    tracked: isTracked(spec),
+    exitCode,
+  }
+
+  let record: unknown
+  let parsed = true
+  try {
+    record = JSON.parse(stdout)
+  } catch {
+    parsed = false
+  }
+
+  if (exitCode === EXIT_REFUSED) {
+    return {
+      ...base,
+      status: isExpectedAbsence(spec, record) ? 'absent' : 'unmeasured',
+      reason: parsed
+        ? refusalReason(record, exitCode)
+        : `refused with no record on stdout`,
+    }
+  }
+
+  const counts = parsed ? spec.counts(record) : undefined
+
+  if (spec.gatingExits.includes(exitCode)) {
+    return { ...base, status: 'finding', ...(counts && { counts }) }
+  }
+
+  if (exitCode !== 0 && exitCode !== EXIT_FINDINGS) {
+    return {
+      ...base,
+      status: 'unmeasured',
+      reason: `exited ${exitCode}, which this verb does not document`,
+    }
+  }
+
+  if (counts === undefined) {
+    return {
+      ...base,
+      status: 'unmeasured',
+      reason: parsed
+        ? 'the record did not carry the keys this audit reads'
+        : 'stdout carried no JSON record',
+    }
+  }
+
+  if (exitCode === EXIT_FINDINGS) {
+    return { ...base, status: 'reported', counts }
+  }
+
+  const quiet = Object.values(counts).every((value) => value === 0)
+  return { ...base, status: quiet ? 'clean' : 'reported', counts }
+}

package/src/audits/run.ts

@@ -0,0 +1,202 @@
+import { join } from 'node:path'
+import { execa } from 'execa'
+import type { Delta } from '@/audits/baseline'
+import { type AuditResult, type AuditSpec, classify } from '@/audits/catalog'
+import { gitEnv } from '@/git-env'
+import { PROJECT_ROOT } from '@/project-root'
+
+/** At least one audit reported a finding that is a fact. */
+export const EXIT_FINDING = 2
+
+/** At least one audit did not report, so the run measured less than the set. */
+export const EXIT_UNMEASURED = 3
+
+export type Verdict =
+  /** Every audit reported and every count it produced was zero. */
+  | 'clean'
+  /** Every audit reported and at least one carries a judgment finding. */
+  | 'reported'
+  /** At least one audit carries a finding that is a fact. */
+  | 'findings'
+  /** At least one audit did not report, so this is not a pass. */
+  | 'incomplete'
+
+export interface SpawnResult {
+  readonly exitCode: number
+  readonly stdout: string
+}
+
+export type Spawn = (spec: AuditSpec) => Promise<SpawnResult>
+
+/**
+ * Runs each verb out of the checkout this CLI is executing from.
+ *
+ * `process.execPath` and the resolved `cli.ts` rather than a bare `aitk`, for
+ * the reason `verify.sh` already names: a globally installed binary resolves to
+ * the main checkout no matter which worktree is running, so the aggregate would
+ * measure the wrong tree and report a pass over a branch it never read.
+ *
+ * `reject: false` because a findings exit is the ordinary outcome for half of
+ * these verbs, and a throw there would be an error path for a working check.
+ */
+export function spawnAudit(root: string): Spawn {
+  const cli = join(PROJECT_ROOT, 'src', 'cli.ts')
+
+  return async (spec) => {
+    const result = await execa(process.execPath, [cli, ...spec.argv], {
+      cwd: root,
+      reject: false,
+      // `gitEnv` already returns the whole ambient environment minus git's
+      // resolution variables, so it replaces the environment rather than
+      // extending it. Spreading `process.env` alongside would put back the very
+      // variables it strips, and a hook's `GIT_DIR` would then point every
+      // history-reading verb at a repository nobody asked about.
+      env: { ...gitEnv(), AITK_NON_INTERACTIVE: '1' },
+      extendEnv: false,
+    })
+
+    return { exitCode: result.exitCode ?? 1, stdout: result.stdout }
+  }
+}
+
+/**
+ * Runs every audit and reads each one's own record shape.
+ *
+ * The verbs walk separate trees and share no state, so they run together
+ * rather than in sequence. Serially, the aggregate would be the slowest thing
+ * in the verify pipeline for no reason beyond the order they were written in.
+ *
+ * A spawn that throws becomes an `unmeasured` result rather than a rejection.
+ * One absent binary would otherwise take the whole aggregate down and report
+ * nothing about the audits that did run.
+ */
+export async function runAudits(
+  specs: readonly AuditSpec[],
+  spawn: Spawn,
+): Promise<AuditResult[]> {
+  return Promise.all(
+    specs.map(async (spec) => {
+      try {
+        const { exitCode, stdout } = await spawn(spec)
+        return classify(spec, exitCode, stdout)
+      } catch (error) {
+        return {
+          id: spec.id,
+          label: spec.label,
+          status: 'unmeasured' as const,
+          tracked: spec.corpus === 'tracked',
+          exitCode: 1,
+          reason: `could not be started: ${error instanceof Error ? error.message : String(error)}`,
+        }
+      }
+    }),
+  )
+}
+
+/**
+ * The single verdict over the set.
+ *
+ * `incomplete` outranks a quiet set on purpose. An aggregate reporting a pass
+ * over a tree it did not finish measuring is the failure this whole command
+ * exists against, and an empty set takes it for the same reason: nothing ran,
+ * so nothing passed.
+ *
+ * An `absent` per-machine corpus does not reach it. That folder is gitignored
+ * and missing on every fresh clone, so folding it in would pin the verdict at
+ * `incomplete` on every CI run and leave the word meaning nothing.
+ */
+export function verdictOf(results: readonly AuditResult[]): Verdict {
+  if (results.length === 0) return 'incomplete'
+  if (results.some((result) => result.status === 'finding')) return 'findings'
+  if (results.some((result) => result.status === 'unmeasured')) {
+    return 'incomplete'
+  }
+  if (results.some((result) => result.status === 'reported')) return 'reported'
+  return 'clean'
+}
+
+/**
+ * Exits non-zero only on a fact, which is the split this aggregate inherits
+ * rather than moves. A growing judgment count reports as a delta and fails
+ * nothing, because the standards behind those measures set no hard cap.
+ *
+ * An audit that did not report takes its own code rather than the findings one.
+ * The two mean opposite things to whoever reads the exit: a fact is a defect in
+ * the tree, and an unmeasured audit is a defect in the run.
+ */
+export function exitCodeFor(results: readonly AuditResult[]): number {
+  const verdict = verdictOf(results)
+  if (verdict === 'findings') return EXIT_FINDING
+  if (verdict === 'incomplete') return EXIT_UNMEASURED
+  return 0
+}
+
+export interface Summary {
+  readonly verdict: Verdict
+  /** Audits that reported, out of the whole set. */
+  readonly audited: number
+  /** Audits carrying a finding that is a fact. */
+  readonly facts: number
+  /** Audits that did not report at all. */
+  readonly unmeasured: number
+  /**
+   * Audits whose per-machine folder is not on this disk.
+   *
+   * Published rather than folded into `audited`, so a run stating twelve
+   * audits never implies twelve corpora were read.
+   */
+  readonly absent: number
+  /** Tracked counts that rose against the recorded floor. */
+  readonly grown: number
+  /** Tracked counts that fell against the recorded floor. */
+  readonly shrunk: number
+  /** Tracked audits with no recorded floor to compare against. */
+  readonly unrecorded: number
+}
+
+/**
+ * The flat reading a shell stage takes without parsing the nested record.
+ *
+ * Published rather than left to a consumer to derive, for the reason the
+ * context audit already gives about its own join: deriving it means restating
+ * which question each number answers, and one wrong restatement is growth
+ * reported against a measure that never moved. Every key here is unique across
+ * the whole record, so a grep for one reaches the top level alone.
+ */
+export function summarize(
+  results: readonly AuditResult[],
+  deltas: readonly Delta[],
+): Summary {
+  let grown = 0
+  let shrunk = 0
+  let unrecorded = 0
+
+  for (const delta of deltas) {
+    if (delta.kind === 'unrecorded') {
+      unrecorded += 1
+      continue
+    }
+    if (delta.kind !== 'compared') continue
+
+    for (const moved of delta.moved) {
+      if (moved.delta > 0) grown += 1
+      else shrunk += 1
+    }
+  }
+
+  const counting = (status: AuditResult['status']) =>
+    results.filter((result) => result.status === status).length
+
+  return {
+    verdict: verdictOf(results),
+    audited: results.filter(
+      (result) => result.status !== 'unmeasured' && result.status !== 'absent',
+    ).length,
+    facts: counting('finding'),
+    unmeasured: counting('unmeasured'),
+    absent: counting('absent'),
+    grown,
+    shrunk,
+    unrecorded,
+  }
+}

package/src/cli.ts

@@ -27,6 +27,7 @@ import { register as context } from '@/commands/context'
 import { register as markdown } from '@/commands/markdown'
 import { register as records } from '@/commands/records'
 import { register as sessions } from '@/commands/sessions'
+import { register as audits } from '@/commands/audits'
 import { PROJECT_ROOT } from '@/project-root'
 
 const GREY = '\x1b[0;90m'
@@ -63,6 +64,7 @@ function showHelp(): void {
     `${GREY}│${NC}    markdown [cmd]     ${GREY}# Report markdown against the attribute standards (audit)${NC}`,
     `${GREY}│${NC}    records [cmd]      ${GREY}# Session records under .claude/ (validate, push, pull)${NC}`,
     `${GREY}│${NC}    sessions [cmd]     ${GREY}# Resolve live sessions to worktree and branch (list)${NC}`,
+    `${GREY}│${NC}    audits [cmd]       ${GREY}# Run every health check as one set (run, list)${NC}`,
     `${GREY}│${NC}`,
     `${GREY}│${NC}  ${WHITE}Sandbox:${NC}`,
     `${GREY}│${NC}    aitk sandbox             ${GREY}# Interactive scenario picker${NC}`,
@@ -101,6 +103,7 @@ function showHelp(): void {
     `${GREY}│${NC}    aitk records validate plans`,
     `${GREY}│${NC}    aitk records push --json`,
     `${GREY}│${NC}    aitk sessions list --json`,
+    `${GREY}│${NC}    aitk audits run --json`,
     `${GREY}└${NC}`,
   ]
   console.log(lines.join('\n'))
@@ -159,5 +162,6 @@ context(program)
 markdown(program)
 records(program)
 sessions(program)
+audits(program)
 
 program.parse()