@erclx/aitk 0.63.1 → 0.64.0

package/src/markdown/bans.ts

@@ -0,0 +1,241 @@
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { linesOutsideFences } from '@/markdown/scan'
+
+/**
+ * Headings the two standards state their closed sets under.
+ *
+ * Each standard says where its own bans sit and points at the other for the
+ * rest, so discovery anchors on the heading rather than on a bullet's position.
+ * A section renamed empties the set loudly at the report's legend, where a
+ * position-based read would narrow the check and still print a count.
+ */
+export const WORD_BAN_HEADING = '## Language'
+export const CHARACTER_BAN_HEADING = '## Punctuation'
+
+/** Bullet lead-ins the two closed sets are stated behind. */
+const BAN_LEAD = '- Do not use '
+const SPELLING_LEAD = '- Use American English spelling'
+
+/**
+ * Roots searched in order. The installed copy wins over the authoring root, so
+ * a target project measures against the standard it actually has rather than
+ * one only the toolkit carries.
+ */
+const STANDARD_ROOTS = ['.claude/standards', 'standards']
+
+const BACKTICKED = /`([^`]+)`/g
+const WORD = /^[a-z]+$/
+const SUFFIX = /^-[a-z]+$/
+
+export interface BanReport {
+  /** Single characters the mechanics standard bans outright. */
+  readonly characters: readonly string[]
+  /** Single lowercase words the prose standard bans outright. */
+  readonly words: readonly string[]
+  /** Spellings derived from the prose standard's own examples. */
+  readonly spellings: readonly string[]
+  /** Repo-relative paths of the standards read, in the order read. */
+  readonly sources: readonly string[]
+  /**
+   * Standards that resolved under neither root. Absent is a distinct state from
+   * empty: a scan with no terms finds nothing, and reporting that as a clean
+   * file claims the prose passed when nothing was looked for.
+   */
+  readonly missing: readonly string[]
+}
+
+function section(markdown: string, heading: string): string[] {
+  const lines = linesOutsideFences(markdown)
+  const start = lines.findIndex((line) => line.trim() === heading)
+  if (start === -1) return []
+
+  const body = lines.slice(start + 1)
+  const end = body.findIndex((line) => line.startsWith('## '))
+
+  return end === -1 ? body : body.slice(0, end)
+}
+
+function backticked(line: string): string[] {
+  return [...line.matchAll(BACKTICKED)].map((match) => match[1].trim())
+}
+
+/**
+ * Pulls the single lowercase words out of the `Do not use` bullets.
+ *
+ * A multi-word term is left out rather than matched loosely. The standard bans
+ * a pattern like `It's not X, it's Y` with a placeholder standing in for the
+ * rest of the sentence, and no literal match reaches it, so the phrase bans
+ * stay a reader's judgment and the report names them as unmeasured.
+ */
+export function parseWordBans(markdown: string): string[] {
+  const terms: string[] = []
+
+  for (const line of section(markdown, WORD_BAN_HEADING)) {
+    if (!line.startsWith(BAN_LEAD)) continue
+    for (const term of backticked(line)) {
+      if (WORD.test(term) && !terms.includes(term)) terms.push(term)
+    }
+  }
+
+  return terms
+}
+
+/**
+ * Pulls the single non-alphanumeric characters out of the `Do not use` bullets.
+ *
+ * Width is what separates the two ban classes in that section. The character
+ * bans are stated as one glyph each, and the parenthetical-aside ban quotes a
+ * whole clause, which is a shape no literal match should be built from.
+ */
+export function parseCharacterBans(markdown: string): string[] {
+  const terms: string[] = []
+
+  for (const line of section(markdown, CHARACTER_BAN_HEADING)) {
+    if (!line.startsWith(BAN_LEAD)) continue
+    for (const term of backticked(line)) {
+      if (
+        term.length === 1 &&
+        !/[a-z0-9]/i.test(term) &&
+        !terms.includes(term)
+      ) {
+        terms.push(term)
+      }
+    }
+  }
+
+  return terms
+}
+
+/**
+ * Derives the banned spellings by applying the standard's own suffix rules to
+ * the standard's own examples.
+ *
+ * The spelling rule is stated as a preference with examples rather than as a
+ * closed set, and a suffix pattern run over prose is what produced 46 of the
+ * 58 false positives measured during intake: `exercises`, `promises`, and
+ * `revised` all end in `-ise` and none is a British spelling. Transforming the
+ * listed American words into their British forms yields a closed set that
+ * matches whole words and reaches none of them, and an example added to the
+ * standard extends the check without a code edit.
+ */
+export function parseSpellingBans(markdown: string): string[] {
+  const line = section(markdown, WORD_BAN_HEADING).find((each) =>
+    each.startsWith(SPELLING_LEAD),
+  )
+  if (!line) return []
+
+  const terms = backticked(line)
+  const suffixes = terms.filter((term) => SUFFIX.test(term))
+  const examples = terms.filter((term) => WORD.test(term))
+
+  // Stated as `preferred` over `banned`, so the run of suffix terms pairs off
+  // in order. An odd count means the sentence was rewritten into a shape this
+  // cannot read, and pairing what is there would invent a rule from half of it.
+  if (suffixes.length === 0 || suffixes.length % 2 !== 0) return []
+
+  const pairs: { preferred: string; banned: string }[] = []
+  for (let index = 0; index < suffixes.length; index += 2) {
+    pairs.push({
+      preferred: suffixes[index].slice(1),
+      banned: suffixes[index + 1].slice(1),
+    })
+  }
+
+  const spellings: string[] = []
+  for (const example of examples) {
+    const pair = pairs.find((each) => example.endsWith(each.preferred))
+    if (!pair) continue
+
+    const banned = `${example.slice(0, -pair.preferred.length)}${pair.banned}`
+    if (!spellings.includes(banned)) spellings.push(banned)
+  }
+
+  return spellings
+}
+
+export interface StandardText {
+  /** Repo-relative path of the copy read, so a report says which root won. */
+  readonly source: string
+  readonly text: string
+}
+
+export interface Standards {
+  readonly markdown: StandardText | undefined
+  readonly prose: StandardText | undefined
+  /**
+   * Standards that resolved under neither root. Absent is a distinct state from
+   * empty: a scan with no terms finds nothing, and reporting that as a clean
+   * file claims the prose passed when nothing was looked for.
+   */
+  readonly missing: readonly string[]
+}
+
+async function readStandard(
+  root: string,
+  name: string,
+): Promise<StandardText | undefined> {
+  for (const standardRoot of STANDARD_ROOTS) {
+    const path = resolve(root, standardRoot, name)
+    const file = Bun.file(path)
+    if (!(await file.exists())) continue
+
+    return {
+      source: `${standardRoot}/${name}`,
+      text: await readFile(path, 'utf8'),
+    }
+  }
+
+  return undefined
+}
+
+/**
+ * Finds both standards, so one read serves the ban sets and the checkpoints.
+ *
+ * The texts are returned rather than the parsed sets alone because
+ * `structure.ts` reads its numbers out of the same `markdown.md`, and a second
+ * loader would open the file twice and could resolve a different root than
+ * this one did.
+ */
+export async function loadStandards(root: string): Promise<Standards> {
+  const [markdown, prose] = await Promise.all([
+    readStandard(root, 'markdown.md'),
+    readStandard(root, 'prose.md'),
+  ])
+
+  return {
+    markdown,
+    prose,
+    missing: [
+      markdown ? undefined : 'markdown.md',
+      prose ? undefined : 'prose.md',
+    ].filter((name): name is string => name !== undefined),
+  }
+}
+
+/**
+ * Reads both closed sets out of the standards stating them.
+ *
+ * Holding the lists in code was the alternative and it puts the bans in two
+ * places, where an author adding a banned word gets no enforcement until
+ * someone edits TypeScript. The trade is a reader of prose that a reformat can
+ * break, which `bans.test.ts` answers by asserting the parsed sets against the
+ * shipped standards.
+ */
+export function banReport(standards: Standards): BanReport {
+  const { markdown, prose } = standards
+
+  return {
+    characters: markdown ? parseCharacterBans(markdown.text) : [],
+    words: prose ? parseWordBans(prose.text) : [],
+    spellings: prose ? parseSpellingBans(prose.text) : [],
+    sources: [markdown?.source, prose?.source].filter(
+      (source): source is string => source !== undefined,
+    ),
+    missing: standards.missing,
+  }
+}
+
+export async function loadBans(root: string): Promise<BanReport> {
+  return banReport(await loadStandards(root))
+}

package/src/markdown/files.ts

@@ -0,0 +1,92 @@
+import { existsSync, statSync } from 'node:fs'
+import { relative, resolve } from 'node:path'
+import { listRepositoryFiles } from '@/git-files'
+
+const MARKDOWN = /\.md$/
+
+/**
+ * `unavailable` is a distinct state from an empty scope.
+ *
+ * A bare run takes its corpus from git, and a git that cannot answer would
+ * otherwise resolve zero files and report every one of them clean.
+ */
+export type FileScope =
+  | {
+      readonly kind: 'resolved'
+      readonly files: readonly string[]
+      /** Arguments matching no markdown file, which is a typo rather than a pass. */
+      readonly unmatched: readonly string[]
+    }
+  | { readonly kind: 'unavailable' }
+
+function isDirectory(path: string): boolean {
+  try {
+    return statSync(path).isDirectory()
+  } catch {
+    return false
+  }
+}
+
+/** Compiles the pattern once rather than per candidate. */
+function matchGlob(files: readonly string[], pattern: string): string[] {
+  const glob = new Bun.Glob(pattern)
+  return files.filter((each) => glob.match(each))
+}
+
+/**
+ * Resolves the arguments to a markdown file list, repo-relative and sorted.
+ *
+ * The corpus is what git lists rather than a directory walk, which is what
+ * keeps `node_modules` and the gitignored session-scratch folders out without
+ * naming either. A directory argument narrows that list by prefix and a
+ * pattern narrows it by match, so the three argument shapes read one corpus
+ * and a file only the working tree has is in scope on the branch adding it.
+ *
+ * An explicit file path is taken as given. A caller naming one file means it,
+ * and refusing a path git does not list would make the verb unusable against a
+ * gitignored draft a session wants measured before it commits.
+ */
+export async function resolveMarkdown(
+  root: string,
+  args: readonly string[],
+): Promise<FileScope> {
+  const listed = await listRepositoryFiles(root)
+  if (!listed) return { kind: 'unavailable' }
+
+  const markdown = listed.filter((rel) => MARKDOWN.test(rel))
+
+  if (args.length === 0) {
+    return { kind: 'resolved', files: markdown, unmatched: [] }
+  }
+
+  const files = new Set<string>()
+  const unmatched: string[] = []
+
+  for (const arg of args) {
+    const absolute = resolve(root, arg)
+    const rel = relative(root, absolute)
+
+    if (!isDirectory(absolute) && MARKDOWN.test(arg) && !arg.includes('*')) {
+      if (existsSync(absolute)) files.add(rel)
+      else unmatched.push(arg)
+      continue
+    }
+
+    // An empty `rel` is the root itself, where the prefix below matches nothing
+    // and would report the whole tree as an argument that resolved to no file.
+    const matched = isDirectory(absolute)
+      ? rel === ''
+        ? markdown
+        : markdown.filter((each) => each === rel || each.startsWith(`${rel}/`))
+      : matchGlob(markdown, arg)
+
+    if (matched.length === 0) {
+      unmatched.push(arg)
+      continue
+    }
+
+    for (const each of matched) files.add(each)
+  }
+
+  return { kind: 'resolved', files: [...files].sort(), unmatched }
+}

package/src/markdown/scan.ts

@@ -0,0 +1,183 @@
+/**
+ * A fence opens on a run of three or more backticks or tildes and closes on a
+ * run of the same character at least as long. Length matters: a ```` block
+ * displaying a ``` example closes on neither of the inner delimiters, and a
+ * walker toggling on any run reads the second inner fence as an opening and
+ * inverts the rest of the file.
+ */
+const FENCE = /^(`{3,}|~{3,})/
+
+/** Anchored at position 0, so a `---` block inside a fenced template is body. */
+const FRONTMATTER = /^---\n[\s\S]*?\n---\n?/
+
+/**
+ * Inline code, a link destination, and an autolink, the three spans holding
+ * text a reader is shown rather than told.
+ *
+ * The bans this masking serves are stated with backticked examples, so a
+ * standard quoting its own banned character would report itself without it. A
+ * link destination is masked because a query string carries a semicolon that no
+ * rewrite of the sentence can remove.
+ */
+const CODE_SPAN = /(`+)(?:(?!\1).)*\1/g
+const LINK_DESTINATION = /\]\([^)]*\)/g
+const AUTOLINK = /<[^>\s]+>/g
+
+export interface BodyLine {
+  readonly number: number
+  readonly text: string
+  /** True on a fence delimiter and on every line between one pair. */
+  readonly fenced: boolean
+}
+
+export type BanKind = 'character' | 'word' | 'spelling'
+
+export interface BanFinding {
+  readonly line: number
+  readonly column: number
+  readonly kind: BanKind
+  /** The term as the standard states it, so a report names what to look up. */
+  readonly term: string
+}
+
+export interface BanSets {
+  readonly characters: readonly string[]
+  readonly words: readonly string[]
+  readonly spellings: readonly string[]
+}
+
+/**
+ * Marks which lines sit inside a fence without dropping them.
+ *
+ * Each measure excludes a fence for its own reason and needs a different
+ * response. The depth measure skips a fenced line so an example cannot break
+ * the run around it, bullet folding treats one as a break so a bullet does not
+ * absorb the block below it, and the ban scan ignores it outright. Returning
+ * the mark rather than a filtered list is what lets one walk serve all three.
+ */
+function markFences(texts: readonly string[], offset: number): BodyLine[] {
+  const lines: BodyLine[] = []
+  let fence: string | undefined
+
+  for (const [index, text] of texts.entries()) {
+    const match = FENCE.exec(text.trim())
+    let fenced = false
+
+    if (fence) {
+      fenced = true
+      const closes =
+        match && match[1][0] === fence[0] && match[1].length >= fence.length
+      if (closes) fence = undefined
+    } else if (match) {
+      fenced = true
+      fence = match[1]
+    }
+
+    lines.push({ number: offset + index + 1, text, fenced })
+  }
+
+  return lines
+}
+
+/**
+ * Drops the frontmatter while keeping every surviving line's original number,
+ * so a finding points at the line an editor opens rather than at an offset into
+ * the body.
+ */
+export function bodyLines(source: string): BodyLine[] {
+  const match = source.match(FRONTMATTER)
+  const offset = match ? match[0].split('\n').length - 1 : 0
+
+  return markFences(
+    source
+      .slice(match ? match[0].length : 0)
+      .replace(/\n$/, '')
+      .split('\n'),
+    offset,
+  )
+}
+
+/**
+ * Drops every fenced block from raw text, frontmatter included as content.
+ *
+ * The record validators read whole files whose frontmatter is part of what they
+ * check, so this walks the source as given rather than through `bodyLines`.
+ */
+export function linesOutsideFences(text: string): string[] {
+  return markFences(text.split('\n'), 0)
+    .filter((line) => !line.fenced)
+    .map((line) => line.text)
+}
+
+/** Blanks a span while holding its width, so a column stays where it was. */
+function blank(match: string): string {
+  return ' '.repeat(match.length)
+}
+
+/**
+ * Replaces displayed spans with spaces of equal width.
+ *
+ * Equal width is what keeps a reported column pointing at the character an
+ * editor puts the cursor on, which a plain deletion would shift left by
+ * everything masked ahead of it on the line.
+ */
+export function maskDisplayed(text: string): string {
+  return text
+    .replace(CODE_SPAN, blank)
+    .replace(LINK_DESTINATION, blank)
+    .replace(AUTOLINK, blank)
+}
+
+function escape(term: string): string {
+  return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/**
+ * Finds every banned term outside a fence, a code span, and a link.
+ *
+ * A word ban matches on word boundaries and either casing, since the standard
+ * states each in lowercase and bans the word rather than a spelling of it. This
+ * is what separates the check from the pattern that produced most of the
+ * intake's false positives: `exercises` and `promises` end in the banned
+ * suffix and are not the banned words, and a closed set never reaches them.
+ */
+export function scanBans(
+  lines: readonly BodyLine[],
+  bans: BanSets,
+): BanFinding[] {
+  const found: BanFinding[] = []
+
+  const patterns: { kind: BanKind; term: string; pattern: RegExp }[] = [
+    ...bans.words.map((term) => ({
+      kind: 'word' as const,
+      term,
+      pattern: new RegExp(`\\b${escape(term)}\\b`, 'gi'),
+    })),
+    ...bans.spellings.map((term) => ({
+      kind: 'spelling' as const,
+      term,
+      pattern: new RegExp(`\\b${escape(term)}\\b`, 'gi'),
+    })),
+  ]
+
+  for (const line of lines) {
+    if (line.fenced) continue
+    const text = maskDisplayed(line.text)
+
+    for (const term of bans.characters) {
+      let column = text.indexOf(term)
+      while (column !== -1) {
+        found.push({ line: line.number, column, kind: 'character', term })
+        column = text.indexOf(term, column + term.length)
+      }
+    }
+
+    for (const { kind, term, pattern } of patterns) {
+      for (const match of text.matchAll(pattern)) {
+        found.push({ line: line.number, column: match.index, kind, term })
+      }
+    }
+  }
+
+  return found.sort((a, b) => a.line - b.line || a.column - b.column)
+}