@erclx/aitk 0.108.0 → 0.110.0

package/src/audits/baseline.ts

@@ -0,0 +1,194 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import type { AuditResult } from '@/audits/catalog'
+
+/**
+ * Where the retained counts live, relative to the project root.
+ *
+ * Committed rather than per-machine, because the question this half answers is
+ * whether a number grew since anyone last looked, and a fresh checkout has to
+ * inherit that answer. A per-machine record makes every contributor's first run
+ * a first run.
+ *
+ * Under the project root rather than beside the aggregator in `src/`, because
+ * the numbers describe one repository's corpus and `src/` ships to every target
+ * that installs the CLI. A baseline in the package would hand a target this
+ * repository's counts to measure its own tree against.
+ */
+export const BASELINE_REL = join('.claude', 'audits', 'baseline.json')
+
+export interface Baseline {
+  /** The day the record was taken, as `YYYY-MM-DD`. */
+  readonly recordedAt: string
+  /** The commit the counts were read at, so a reader can reproduce them. */
+  readonly commit: string
+  readonly checks: Readonly<Record<string, Readonly<Record<string, number>>>>
+}
+
+export interface Stamp {
+  readonly recordedAt: string
+  readonly commit: string
+}
+
+export interface MovedCount {
+  readonly key: string
+  readonly from: number
+  readonly to: number
+  readonly delta: number
+}
+
+export type Delta =
+  | {
+      readonly id: string
+      readonly kind: 'compared'
+      readonly moved: readonly MovedCount[]
+      /** Keys the run reproduced exactly. */
+      readonly steady: readonly string[]
+      /** Keys the run produced that the baseline never recorded. */
+      readonly added: readonly { key: string; to: number }[]
+      /** Keys the baseline holds that this run did not produce. */
+      readonly dropped: readonly { key: string; from: number }[]
+    }
+  /** No recorded floor, so a zero delta would be indistinguishable from quiet. */
+  | { readonly id: string; readonly kind: 'unrecorded' }
+  /** Gitignored scratch, whose counts are one machine's and answer nobody else. */
+  | { readonly id: string; readonly kind: 'per-machine' }
+  /** The audit did not report, so there is nothing to compare. */
+  | { readonly id: string; readonly kind: 'unmeasured' }
+
+/**
+ * Builds the record a run leaves behind.
+ *
+ * Only a tracked corpus is retained. A gitignored record folder holds one
+ * machine's session scratch, so committing its counts writes a floor no other
+ * clone can reproduce, and every contributor would read a regression against a
+ * number that describes somebody else's disk.
+ *
+ * An audit that did not report is left out rather than written as zero. Zeros
+ * there record a clean corpus nobody measured, and the next run reads its real
+ * numbers as a regression against a floor that was never taken.
+ */
+export function baselineFrom(
+  results: readonly AuditResult[],
+  stamp: Stamp,
+): Baseline {
+  const checks: Record<string, Record<string, number>> = {}
+
+  for (const result of results) {
+    if (!result.tracked || result.counts === undefined) continue
+    checks[result.id] = { ...result.counts }
+  }
+
+  return { recordedAt: stamp.recordedAt, commit: stamp.commit, checks }
+}
+
+/**
+ * Compares this run against the recorded floor, one audit at a time.
+ *
+ * A first run reports `unrecorded` rather than a delta of zero. A zero delta
+ * against an absent baseline says the same thing as a corpus that did not move,
+ * and those are the two states this repository has already had to separate
+ * twice elsewhere.
+ */
+export function compareBaseline(
+  baseline: Baseline | undefined,
+  results: readonly AuditResult[],
+): Delta[] {
+  return results.map((result) => {
+    if (!result.tracked) return { id: result.id, kind: 'per-machine' as const }
+    if (result.counts === undefined) {
+      return { id: result.id, kind: 'unmeasured' as const }
+    }
+
+    const recorded = baseline?.checks[result.id]
+    if (recorded === undefined) {
+      return { id: result.id, kind: 'unrecorded' as const }
+    }
+
+    const moved: MovedCount[] = []
+    const steady: string[] = []
+    const added: { key: string; to: number }[] = []
+    const dropped: { key: string; from: number }[] = []
+
+    for (const [key, to] of Object.entries(result.counts)) {
+      const from = recorded[key]
+      if (from === undefined) {
+        added.push({ key, to })
+      } else if (from === to) {
+        steady.push(key)
+      } else {
+        moved.push({ key, from, to, delta: to - from })
+      }
+    }
+
+    for (const [key, from] of Object.entries(recorded)) {
+      if (!(key in result.counts)) dropped.push({ key, from })
+    }
+
+    return {
+      id: result.id,
+      kind: 'compared' as const,
+      moved,
+      steady,
+      added,
+      dropped,
+    }
+  })
+}
+
+function isBaseline(value: unknown): value is Baseline {
+  if (typeof value !== 'object' || value === null) return false
+  const record = value as Record<string, unknown>
+  return (
+    typeof record.recordedAt === 'string' &&
+    typeof record.commit === 'string' &&
+    typeof record.checks === 'object' &&
+    record.checks !== null &&
+    !Array.isArray(record.checks)
+  )
+}
+
+/**
+ * Reads the recorded floor, or `undefined` when none has been taken.
+ *
+ * An absent file and a broken one are different answers. Absent is a first run.
+ * Broken is a record someone hand-edited into a shape nothing can read, and
+ * reading that as absent would silently reset the floor the file exists to hold.
+ */
+export async function readBaseline(
+  root: string,
+): Promise<Baseline | undefined> {
+  const path = join(root, BASELINE_REL)
+
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf8')
+  } catch {
+    return undefined
+  }
+
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch {
+    throw new Error(`${BASELINE_REL} does not parse as JSON. Fix or delete it.`)
+  }
+
+  if (!isBaseline(parsed)) {
+    throw new Error(
+      `${BASELINE_REL} carries no recordedAt, commit, and checks. Fix or delete it.`,
+    )
+  }
+
+  return parsed
+}
+
+export async function writeBaseline(
+  root: string,
+  baseline: Baseline,
+): Promise<string> {
+  const path = join(root, BASELINE_REL)
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(path, `${JSON.stringify(baseline, null, 2)}\n`, 'utf8')
+  return path
+}

package/src/audits/catalog.ts

@@ -0,0 +1,464 @@
+import type { ValidateRefusal as RecordRefusal } from '@/records/validate'
+import type { ValidateRefusal as BoardRefusal } from '@/tasks/validate'
+
+/**
+ * The audits this repository already owns, and how to read each one's record.
+ *
+ * Every verb here ships its own `--json` shape with its own keys, and this
+ * module reads each shape rather than forcing a common envelope on them. Each
+ * record already has consumers, so an envelope would be a breaking change to
+ * every one of them bought for tidiness.
+ */
+
+/** How far a finding from this audit reaches. */
+export type Corpus =
+  /** Committed, so every clone measures the same files and a delta is shared. */
+  | 'tracked'
+  /** Gitignored session scratch, so the numbers are one machine's alone. */
+  | 'per-machine'
+
+export type AuditStatus =
+  /** The audit reported and every count it produced is zero. */
+  | 'clean'
+  /** The audit reported findings it does not gate on. */
+  | 'reported'
+  /** The audit reported a finding that is a fact, which fails the aggregate. */
+  | 'finding'
+  /**
+   * The per-machine corpus this audit reads is not on this disk, which is the
+   * ordinary state of a gitignored folder on a fresh clone and in CI.
+   */
+  | 'absent'
+  /** The audit did not report, so the aggregate measured less than the set. */
+  | 'unmeasured'
+
+export interface AuditSpec {
+  readonly id: string
+  readonly label: string
+  /** Arguments after `aitk`, always ending in `--json`. */
+  readonly argv: readonly string[]
+  /**
+   * Exit codes this verb sets on a finding that is a fact.
+   *
+   * Empty for every verb whose findings are judgments. The catalog decides
+   * this rather than the exit code, because a reporting verb sets 2 on
+   * findings it deliberately does not gate on, so the code alone cannot say
+   * whether a finding is a fact.
+   */
+  readonly gatingExits: readonly number[]
+  readonly corpus: Corpus
+  /**
+   * Pulls the counts worth retaining out of this verb's record.
+   *
+   * Returns `undefined` when the record does not carry the keys it reads,
+   * which is a measure that did not run rather than a corpus with nothing in
+   * it. Zero there would read as clean, which is the two-states-look-alike
+   * defect this repository has already fixed elsewhere.
+   */
+  readonly counts: (record: unknown) => Record<string, number> | undefined
+}
+
+export interface AuditResult {
+  readonly id: string
+  readonly label: string
+  readonly status: AuditStatus
+  readonly tracked: boolean
+  readonly exitCode: number
+  readonly counts?: Record<string, number>
+  /** Why the audit did not report, present only on `unmeasured`. */
+  readonly reason?: string
+}
+
+/** The exit code every command here sets when it refuses. */
+const EXIT_REFUSED = 1
+
+/** The exit code every command here sets when it carries findings. */
+const EXIT_FINDINGS = 2
+
+function asObject(value: unknown): Record<string, unknown> | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined
+  }
+  return value as Record<string, unknown>
+}
+
+function lengthOf(value: unknown): number | undefined {
+  return Array.isArray(value) ? value.length : undefined
+}
+
+/**
+ * Folds a set of per-key readings into one record, or `undefined` when any of
+ * them could not be read. One unreadable key means the record is not the shape
+ * the extractor was written against, and reporting the rest would publish a
+ * partial count under a name that claims to be whole.
+ */
+function allOf(
+  readings: Record<string, number | undefined>,
+): Record<string, number> | undefined {
+  const counts: Record<string, number> = {}
+  for (const [key, value] of Object.entries(readings)) {
+    if (value === undefined) return undefined
+    counts[key] = value
+  }
+  return counts
+}
+
+function contextCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  const entries = root.entries
+  if (!Array.isArray(entries)) return undefined
+
+  let bareReferences = 0
+  for (const entry of entries) {
+    const bare = lengthOf(asObject(entry)?.bareReferences)
+    if (bare === undefined) return undefined
+    bareReferences += bare
+  }
+
+  return allOf({
+    unresolvedCitations: lengthOf(asObject(root.citations)?.unresolved),
+    longEntries: lengthOf(root.length),
+    missingSections: lengthOf(root.missingSections),
+    indexDrift: lengthOf(root.indexDrift),
+    bareReferences,
+  })
+}
+
+function markdownCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  const entries = root.entries
+  if (!Array.isArray(entries)) return undefined
+
+  const depth = asObject(root.checkpoints)?.run
+  // A depth reading needs the checkpoint it is measured against. Counting zero
+  // without it reports a corpus nobody measured for depth as one with no file
+  // past the line.
+  if (entries.length > 0 && typeof depth !== 'number') return undefined
+
+  let bans = 0
+  let heavyBullets = 0
+  let heavyParagraphs = 0
+  let filesPastDepth = 0
+  let flatParagraphs = 0
+
+  for (const raw of entries) {
+    const entry = asObject(raw)
+    if (entry === undefined) return undefined
+
+    const entryBans = lengthOf(entry.bans)
+    const bullets = lengthOf(entry.heavyBullets)
+    const paragraphs = lengthOf(entry.heavyParagraphs)
+    const run = entry.longestRun
+    if (
+      entryBans === undefined ||
+      bullets === undefined ||
+      paragraphs === undefined ||
+      typeof run !== 'number'
+    ) {
+      return undefined
+    }
+
+    bans += entryBans
+    heavyBullets += bullets
+    heavyParagraphs += paragraphs
+    if (typeof depth === 'number' && run > depth) filesPastDepth += 1
+
+    // A file below the measuring floor carries no cadence key at all, which is
+    // a file that was not measured rather than one with no flat paragraph.
+    const flat = asObject(entry.cadence)?.flat
+    if (typeof flat === 'number') flatParagraphs += flat
+  }
+
+  return { bans, heavyBullets, heavyParagraphs, filesPastDepth, flatParagraphs }
+}
+
+/** The finding arrays `aitk claude skills audit` publishes, in its own order. */
+const SKILL_FINDINGS = [
+  'missingRequirement',
+  'nameMismatch',
+  'missingDescription',
+  'longDescription',
+  'readme',
+  'folderName',
+  'requirementSections',
+] as const
+
+function skillCounts(record: unknown): Record<string, number> | undefined {
+  const findings = asObject(asObject(record)?.findings)
+  if (findings === undefined) return undefined
+
+  return allOf(
+    Object.fromEntries(
+      SKILL_FINDINGS.map((key) => [key, lengthOf(findings[key])]),
+    ),
+  )
+}
+
+function boardCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  // The board keeps its untested rows apart from its findings on purpose, so
+  // they move no exit code. Folding them together here would undo that.
+  return allOf({
+    findings: lengthOf(root.findings),
+    untested: lengthOf(root.untested),
+  })
+}
+
+function findingsOnly(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  return allOf({ findings: lengthOf(root.findings) })
+}
+
+function commentCounts(record: unknown): Record<string, number> | undefined {
+  const snapshot = asObject(record)?.snapshot
+  if (!Array.isArray(snapshot)) return undefined
+
+  let degradationHits = 0
+  for (const language of snapshot) {
+    const hits = lengthOf(asObject(language)?.degradationHits)
+    if (hits === undefined) return undefined
+    degradationHits += hits
+  }
+
+  return { degradationHits }
+}
+
+function testOrderCounts(record: unknown): Record<string, number> | undefined {
+  const root = asObject(record)
+  if (root === undefined) return undefined
+
+  // The unclassified count travels with the findings because it is the honest
+  // shape of this measure: a refactor and a new behavior cannot be told apart
+  // from history, and a findings count alone claims coverage it does not have.
+  return allOf({
+    findings: lengthOf(root.findings),
+    unclassified: lengthOf(root.unclassified),
+  })
+}
+
+/** The record kinds `aitk records validate` takes, and the corpus each reads. */
+const RECORD_KINDS: readonly (readonly [string, Corpus])[] = [
+  ['plans', 'per-machine'],
+  ['groundwork', 'per-machine'],
+  ['intake', 'per-machine'],
+  ['memory', 'per-machine'],
+  ['standards', 'tracked'],
+  ['teach', 'per-machine'],
+]
+
+/**
+ * Every audit the aggregate runs.
+ *
+ * `context`, `markdown`, and `skills` are the three that gate, which is exactly
+ * the set `scripts/core/verify.sh` already fails a push on. Adding a fourth
+ * here widens what fails a push without anyone deciding to, and the split this
+ * repository records gates a fact and reports a judgment.
+ *
+ * Each verb runs once in its fullest form. Running the gating half separately
+ * would walk the same tree twice for a number the full record already carries.
+ */
+export const AUDITS: readonly AuditSpec[] = [
+  {
+    id: 'context',
+    label: 'Context folders',
+    argv: ['context', 'audit', '--json'],
+    gatingExits: [EXIT_FINDINGS],
+    corpus: 'tracked',
+    counts: contextCounts,
+  },
+  {
+    id: 'markdown',
+    label: 'Markdown corpus',
+    argv: ['markdown', 'audit', '--json'],
+    // 3 is the empty ban set, which is the corpus walked with nothing looked
+    // for. That is a broken check rather than a clean tree, so it fails here
+    // the way it already fails the push.
+    gatingExits: [EXIT_FINDINGS, 3],
+    corpus: 'tracked',
+    counts: markdownCounts,
+  },
+  {
+    id: 'skills',
+    label: 'Skill corpora',
+    argv: ['claude', 'skills', 'audit', '--json'],
+    gatingExits: [EXIT_FINDINGS],
+    corpus: 'tracked',
+    counts: skillCounts,
+  },
+  {
+    id: 'tasks',
+    label: 'Task board',
+    argv: ['tasks', 'validate', '--json'],
+    gatingExits: [],
+    corpus: 'per-machine',
+    counts: boardCounts,
+  },
+  ...RECORD_KINDS.map(([kind, corpus]) => ({
+    id: `records-${kind}`,
+    label: `Records: ${kind}`,
+    argv: ['records', 'validate', kind, '--json'],
+    gatingExits: [],
+    corpus,
+    counts: findingsOnly,
+  })),
+  {
+    id: 'comments',
+    label: 'Comment census',
+    argv: ['comments', 'scan', '--json'],
+    gatingExits: [],
+    corpus: 'tracked',
+    counts: commentCounts,
+  },
+  {
+    id: 'test-order',
+    label: 'Test order',
+    argv: ['gov', 'test-order', '--json'],
+    gatingExits: [],
+    corpus: 'tracked',
+    counts: testOrderCounts,
+  },
+]
+
+export function auditFor(id: string): AuditSpec | undefined {
+  return AUDITS.find((audit) => audit.id === id)
+}
+
+export function isTracked(spec: AuditSpec): boolean {
+  return spec.corpus === 'tracked'
+}
+
+export function countsFor(
+  spec: AuditSpec,
+  record: unknown,
+): Record<string, number> | undefined {
+  return spec.counts(record)
+}
+
+/**
+ * Reads the reason a refusing verb published, so the aggregate names the cause
+ * rather than only the exit. Every command here puts it in `reason`, and the
+ * ones that also carry a sentence put that in `message`.
+ */
+function refusalReason(record: unknown, exitCode: number): string {
+  const root = asObject(record)
+  const reason = typeof root?.reason === 'string' ? root.reason : undefined
+  const message = typeof root?.message === 'string' ? root.message : undefined
+
+  if (reason !== undefined && message !== undefined) {
+    return `${reason}: ${message}`
+  }
+  return reason ?? message ?? `exited ${exitCode} with no reason on stdout`
+}
+
+/**
+ * The reasons a verb publishes when the folder it reads is simply not there.
+ *
+ * Typed against the unions those two verbs export rather than spelled as bare
+ * strings, so renaming a reason at its source fails this build. A literal here
+ * would go on matching nothing and quietly turn every expected absence back
+ * into an unmeasured audit.
+ */
+const ABSENT_REASONS: readonly (RecordRefusal | BoardRefusal)[] = [
+  'no-folder',
+  'no-board',
+]
+
+/**
+ * Whether a refusal is a folder this machine never created rather than a break.
+ *
+ * Every gitignored record folder is absent on a fresh clone and in CI, so six
+ * of the twelve refuse there on every run. Counting those as unmeasured pins
+ * the verdict at `incomplete` forever, and a signal that never changes is one
+ * nobody reads after the second time they see it.
+ *
+ * A tracked corpus gets no such allowance. That tree ships to targets, so a
+ * checkout that cannot find it is broken, and reading the absence as ordinary
+ * would report a pass over a corpus nobody measured.
+ */
+function isExpectedAbsence(spec: AuditSpec, record: unknown): boolean {
+  if (spec.corpus !== 'per-machine') return false
+
+  const reason = asObject(record)?.reason
+  return (
+    typeof reason === 'string' &&
+    (ABSENT_REASONS as readonly string[]).includes(reason)
+  )
+}
+
+/**
+ * Turns one verb's exit code and stdout into a result the aggregate can report.
+ *
+ * Unparseable output is `unmeasured` rather than clean, for the reason the
+ * verify pipeline already states about its own skipped stages: a run that does
+ * not report is a broken command, and skipping would report the pass the check
+ * exists to withhold.
+ */
+export function classify(
+  spec: AuditSpec,
+  exitCode: number,
+  stdout: string,
+): AuditResult {
+  const base = {
+    id: spec.id,
+    label: spec.label,
+    tracked: isTracked(spec),
+    exitCode,
+  }
+
+  let record: unknown
+  let parsed = true
+  try {
+    record = JSON.parse(stdout)
+  } catch {
+    parsed = false
+  }
+
+  if (exitCode === EXIT_REFUSED) {
+    return {
+      ...base,
+      status: isExpectedAbsence(spec, record) ? 'absent' : 'unmeasured',
+      reason: parsed
+        ? refusalReason(record, exitCode)
+        : `refused with no record on stdout`,
+    }
+  }
+
+  const counts = parsed ? spec.counts(record) : undefined
+
+  if (spec.gatingExits.includes(exitCode)) {
+    return { ...base, status: 'finding', ...(counts && { counts }) }
+  }
+
+  if (exitCode !== 0 && exitCode !== EXIT_FINDINGS) {
+    return {
+      ...base,
+      status: 'unmeasured',
+      reason: `exited ${exitCode}, which this verb does not document`,
+    }
+  }
+
+  if (counts === undefined) {
+    return {
+      ...base,
+      status: 'unmeasured',
+      reason: parsed
+        ? 'the record did not carry the keys this audit reads'
+        : 'stdout carried no JSON record',
+    }
+  }
+
+  if (exitCode === EXIT_FINDINGS) {
+    return { ...base, status: 'reported', counts }
+  }
+
+  const quiet = Object.values(counts).every((value) => value === 0)
+  return { ...base, status: quiet ? 'clean' : 'reported', counts }
+}