@erclx/aitk 0.8.0 → 0.10.0

package/src/comments/scan.ts

@@ -0,0 +1,338 @@
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { listIgnored } from '@/git-ignore'
+
+export type Language = 'ts' | 'sh'
+
+export const LANGUAGES: readonly Language[] = ['ts', 'sh']
+
+const EXTENSIONS: Record<Language, readonly string[]> = {
+  ts: ['.ts', '.tsx'],
+  sh: ['.sh'],
+}
+
+/**
+ * Pruned by path segment rather than by glob, since `Bun.Glob` has no exclude.
+ * `fixtures` and `__fixtures__` are here because fixture trees hold content
+ * authored to be parsed rather than run, and counting it reports the harness
+ * instead of the source.
+ */
+const PRUNED_SEGMENTS = [
+  '.git',
+  'node_modules',
+  'dist',
+  'build',
+  'coverage',
+  'fixtures',
+  '__fixtures__',
+]
+
+export interface DegradationHit {
+  readonly file: string
+  readonly line: number
+  readonly term: string
+}
+
+export interface LanguageCount {
+  readonly language: Language
+  readonly files: number
+  readonly lines: number
+  readonly commentLines: number
+  readonly docBlocks: number
+  readonly inlineComments: number
+  readonly degradationHits: readonly DegradationHit[]
+}
+
+export interface ScanOptions {
+  readonly languages?: readonly Language[]
+  readonly vocabulary?: readonly string[]
+}
+
+/** One file's text paired with the repo-relative path a hit is reported under. */
+export interface SourceFile {
+  readonly path: string
+  readonly text: string
+}
+
+export function languageFor(path: string): Language | undefined {
+  for (const language of LANGUAGES) {
+    if (EXTENSIONS[language].some((ext) => path.endsWith(ext))) return language
+  }
+  return undefined
+}
+
+export function isPruned(relativePath: string): boolean {
+  return relativePath
+    .split('/')
+    .some((segment) => PRUNED_SEGMENTS.includes(segment))
+}
+
+/**
+ * Builds the matcher for one vocabulary term.
+ *
+ * Case sensitivity is derived from the term rather than from a second list:
+ * a term carrying an uppercase letter is a marker convention (`TODO`, `HACK`)
+ * and matches exactly, while an all-lowercase term is prose (`used to`) and
+ * matches either casing. Matching `FIXED` case-insensitively would hit every
+ * comment containing the word "fixed", which is why the split exists.
+ */
+function matcherFor(term: string): RegExp {
+  const escaped = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const leading = /^\w/.test(term) ? '(?<!\\w)' : ''
+  const trailing = /\w$/.test(term) ? '(?!\\w)' : ''
+  const flags = /[A-Z]/.test(term) ? '' : 'i'
+
+  return new RegExp(`${leading}${escaped}${trailing}`, flags)
+}
+
+function sweep(
+  text: string,
+  file: string,
+  line: number,
+  matchers: readonly { term: string; pattern: RegExp }[],
+  hits: DegradationHit[],
+): void {
+  for (const { term, pattern } of matchers) {
+    if (pattern.test(text)) hits.push({ file, line, term })
+  }
+}
+
+function splitLines(text: string): string[] {
+  const lines = text.split('\n')
+  if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop()
+  return lines
+}
+
+interface FileCount {
+  readonly lines: number
+  readonly commentLines: number
+  readonly docBlocks: number
+  readonly inlineComments: number
+}
+
+/**
+ * Counts one TypeScript file.
+ *
+ * A line counts as a comment only when its first non-whitespace token opens
+ * one, which is what keeps a URL in a string literal from reading as a `//`
+ * comment without an AST. The cost is that a `//` line inside a template
+ * literal counts, and a trailing comment after code does not.
+ */
+function countTypeScript(
+  file: SourceFile,
+  matchers: readonly { term: string; pattern: RegExp }[],
+  hits: DegradationHit[],
+): FileCount {
+  const lines = splitLines(file.text)
+  let commentLines = 0
+  let docBlocks = 0
+  let inlineComments = 0
+  let inBlock = false
+
+  for (const [index, line] of lines.entries()) {
+    const trimmed = line.trim()
+
+    if (inBlock) {
+      commentLines++
+      sweep(trimmed, file.path, index + 1, matchers, hits)
+      if (trimmed.includes('*/')) inBlock = false
+      continue
+    }
+
+    if (trimmed.startsWith('/*')) {
+      commentLines++
+      if (/^\/\*\*(?!\/)/.test(trimmed)) docBlocks++
+      sweep(trimmed, file.path, index + 1, matchers, hits)
+      if (trimmed.indexOf('*/', 2) === -1) inBlock = true
+      continue
+    }
+
+    if (trimmed.startsWith('//')) {
+      commentLines++
+      inlineComments++
+      sweep(trimmed, file.path, index + 1, matchers, hits)
+    }
+  }
+
+  return { lines: lines.length, commentLines, docBlocks, inlineComments }
+}
+
+const HEREDOC_OPENER = /<<-?\s*(['"]?)([A-Za-z_][A-Za-z0-9_]*)\1/
+
+/**
+ * Counts one bash file, skipping heredoc bodies.
+ *
+ * A heredoc body is data, and this repository's sandbox scenarios carry
+ * markdown inside one, where `#` opens a heading rather than a comment.
+ * Counting those inflated a measured 112 comment lines to 427. Body lines
+ * leave the denominator as well as the numerator, since a density that drops
+ * the numerator alone understates by however much data the file carries.
+ *
+ * The line-1 shebang is excluded. Every script has one, so counting it puts a
+ * floor under density that reports the file count rather than the discipline.
+ */
+function countBash(
+  file: SourceFile,
+  matchers: readonly { term: string; pattern: RegExp }[],
+  hits: DegradationHit[],
+): FileCount {
+  const lines = splitLines(file.text)
+  let counted = 0
+  let commentLines = 0
+  let inlineComments = 0
+  let delimiter: string | undefined
+
+  for (const [index, line] of lines.entries()) {
+    if (delimiter !== undefined) {
+      // `<<-` strips leading tabs from the terminator, and the plain form
+      // requires it at column zero. Trimming covers both without tracking
+      // which form opened the body.
+      if (line.trim() === delimiter) delimiter = undefined
+      continue
+    }
+
+    counted++
+    const trimmed = line.trim()
+
+    if (trimmed.startsWith('#')) {
+      if (!(index === 0 && trimmed.startsWith('#!'))) {
+        commentLines++
+        inlineComments++
+        sweep(trimmed, file.path, index + 1, matchers, hits)
+      }
+      continue
+    }
+
+    const opener = HEREDOC_OPENER.exec(line)
+    if (opener) delimiter = opener[2]
+  }
+
+  return { lines: counted, commentLines, docBlocks: 0, inlineComments }
+}
+
+export function countFile(
+  file: SourceFile,
+  language: Language,
+  vocabulary: readonly string[] = [],
+): LanguageCount {
+  const matchers = vocabulary.map((term) => ({
+    term,
+    pattern: matcherFor(term),
+  }))
+  const hits: DegradationHit[] = []
+
+  const counted =
+    language === 'ts'
+      ? countTypeScript(file, matchers, hits)
+      : countBash(file, matchers, hits)
+
+  return {
+    language,
+    files: 1,
+    lines: counted.lines,
+    commentLines: counted.commentLines,
+    docBlocks: counted.docBlocks,
+    inlineComments: counted.inlineComments,
+    degradationHits: hits,
+  }
+}
+
+/** Counts a set of already-read files, grouped into one entry per language. */
+export function countFiles(
+  files: readonly SourceFile[],
+  opts: ScanOptions = {},
+): LanguageCount[] {
+  const languages = opts.languages ?? LANGUAGES
+  const vocabulary = opts.vocabulary ?? []
+
+  return languages.map((language) => {
+    const totals = {
+      language,
+      files: 0,
+      lines: 0,
+      commentLines: 0,
+      docBlocks: 0,
+      inlineComments: 0,
+      degradationHits: [] as DegradationHit[],
+    }
+
+    for (const file of files) {
+      if (languageFor(file.path) !== language) continue
+      const count = countFile(file, language, vocabulary)
+      totals.files += count.files
+      totals.lines += count.lines
+      totals.commentLines += count.commentLines
+      totals.docBlocks += count.docBlocks
+      totals.inlineComments += count.inlineComments
+      totals.degradationHits.push(...count.degradationHits)
+    }
+
+    return totals
+  })
+}
+
+export function density(count: LanguageCount): number {
+  return count.lines === 0 ? 0 : count.commentLines / count.lines
+}
+
+/** Lists the scannable source files under `root`, honoring `.gitignore`. */
+export async function listSourceFiles(
+  root: string,
+  languages: readonly Language[] = LANGUAGES,
+): Promise<string[]> {
+  const glob = new Bun.Glob('**/*')
+  const candidates: string[] = []
+
+  for await (const rel of glob.scan({
+    cwd: root,
+    onlyFiles: true,
+    dot: true,
+  })) {
+    const normalized = rel.split('\\').join('/')
+    if (isPruned(normalized)) continue
+
+    const language = languageFor(normalized)
+    if (!language || !languages.includes(language)) continue
+
+    candidates.push(resolve(root, normalized))
+  }
+
+  candidates.sort()
+
+  const ignored = await listIgnored(root, candidates)
+  return ignored.size === 0
+    ? candidates
+    : candidates.filter((path) => !ignored.has(path))
+}
+
+/**
+ * Open descriptors allowed at once while reading a tree.
+ *
+ * A single `Promise.all` over every path opens one descriptor per file, which
+ * exhausts a default 1024 limit on any large repository and fails the whole
+ * scan with EMFILE. Reading in bounded batches keeps the parallelism that
+ * matters without letting the file count set the ceiling.
+ */
+const CONCURRENT_READS = 64
+
+/** Scans a working tree on disk. */
+export async function scanTree(
+  root: string,
+  opts: ScanOptions = {},
+): Promise<LanguageCount[]> {
+  const languages = opts.languages ?? LANGUAGES
+  const paths = await listSourceFiles(root, languages)
+  const files: SourceFile[] = []
+
+  for (let start = 0; start < paths.length; start += CONCURRENT_READS) {
+    const batch = await Promise.all(
+      paths.slice(start, start + CONCURRENT_READS).map(async (path) => ({
+        path: path.startsWith(`${root}/`) ? path.slice(root.length + 1) : path,
+        text: await readFile(path, 'utf8'),
+      })),
+    )
+    files.push(...batch)
+  }
+
+  return countFiles(files, opts)
+}

package/src/comments/trend.ts

@@ -0,0 +1,207 @@
+import { $ } from 'bun'
+import { gitEnv } from '@/git-env'
+import {
+  countFiles,
+  isPruned,
+  type Language,
+  LANGUAGES,
+  type LanguageCount,
+  languageFor,
+  type ScanOptions,
+  type SourceFile,
+} from '@/comments/scan'
+
+export interface TrendPoint {
+  readonly rev: string
+  readonly date: string
+  readonly languages: readonly LanguageCount[]
+}
+
+export const DEFAULT_POINTS = 6
+
+interface Commit {
+  readonly rev: string
+  readonly date: string
+}
+
+function parseCommits(text: string): Commit[] {
+  return text
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [rev, date] = line.split('\t')
+      return { rev, date }
+    })
+}
+
+/**
+ * Lists the commits from `since` to HEAD, oldest first and inclusive of
+ * `since` itself.
+ *
+ * A `since..HEAD` range excludes its own boundary, which drops the reading the
+ * caller is measuring against. The whole point of a trend is the before, so
+ * the boundary commit is prepended rather than left to the range operator.
+ *
+ * `--first-parent` keeps a merged branch's own commits out of the sample, so
+ * evenly spaced points land on the trunk rather than clustering inside
+ * whichever feature happened to carry the most commits.
+ */
+export async function listCommits(
+  root: string,
+  since: string,
+): Promise<Commit[]> {
+  const boundary =
+    await $`git -C ${root} log -1 --format=%H%x09%ad --date=short ${since}`
+      .env(gitEnv())
+      .quiet()
+      .nothrow()
+
+  if (boundary.exitCode !== 0) return []
+
+  const range =
+    await $`git -C ${root} log --first-parent --reverse --format=%H%x09%ad --date=short ${`${since}..HEAD`}`
+      .env(gitEnv())
+      .quiet()
+      .nothrow()
+
+  const commits = parseCommits(boundary.text())
+  if (range.exitCode === 0) commits.push(...parseCommits(range.text()))
+
+  return commits
+}
+
+/**
+ * Picks evenly spaced commits from `commits`, always keeping the newest.
+ *
+ * Sample selection matters more than sample size here. The reading the
+ * comment-discipline track needed came from four points spanning six months,
+ * so spacing across the window is what the arm optimizes for rather than
+ * density of coverage.
+ */
+export function spaceEvenly(
+  commits: readonly Commit[],
+  points: number,
+): Commit[] {
+  if (commits.length <= points) return [...commits]
+  if (points <= 1) return [commits[commits.length - 1]]
+
+  const picked: Commit[] = []
+  const step = (commits.length - 1) / (points - 1)
+
+  for (let index = 0; index < points; index++) {
+    picked.push(commits[Math.round(index * step)])
+  }
+
+  return picked
+}
+
+/**
+ * Reads every scannable blob at `rev` without checking anything out.
+ *
+ * `ls-tree` names the blobs and `cat-file --batch` streams their contents in
+ * one process, so a six-point trend costs six subprocesses rather than one per
+ * file per commit. Contents arrive as bytes, so the batch stream is walked by
+ * byte offset rather than split as text.
+ */
+export async function readRevision(
+  root: string,
+  rev: string,
+  languages: readonly Language[] = LANGUAGES,
+): Promise<SourceFile[]> {
+  const listed = await $`git -C ${root} ls-tree -r -z ${rev}`
+    .env(gitEnv())
+    .quiet()
+    .nothrow()
+  if (listed.exitCode !== 0) return []
+
+  const wanted: { oid: string; path: string }[] = []
+
+  for (const entry of listed.text().split('\0')) {
+    if (!entry) continue
+    const [meta, path] = entry.split('\t')
+    if (!path) continue
+
+    const [, type, oid] = meta.split(/\s+/)
+    if (type !== 'blob') continue
+    if (isPruned(path)) continue
+
+    const language = languageFor(path)
+    if (!language || !languages.includes(language)) continue
+
+    wanted.push({ oid, path })
+  }
+
+  if (wanted.length === 0) return []
+
+  const stdin = Buffer.from(`${wanted.map(({ oid }) => oid).join('\n')}\n`)
+  const batch = await $`git -C ${root} cat-file --batch < ${stdin}`
+    .env(gitEnv())
+    .quiet()
+    .nothrow()
+
+  if (batch.exitCode !== 0) return []
+
+  return parseBatch(Buffer.from(batch.arrayBuffer()), wanted)
+}
+
+/**
+ * Walks `git cat-file --batch` output, which frames each object as
+ * `<oid> SP <type> SP <size> LF <contents> LF`. The size header is the only
+ * safe way to find the next record, since contents may hold anything.
+ */
+function parseBatch(
+  buffer: Buffer,
+  wanted: readonly { oid: string; path: string }[],
+): SourceFile[] {
+  const files: SourceFile[] = []
+  let offset = 0
+
+  for (const { path } of wanted) {
+    const headerEnd = buffer.indexOf(0x0a, offset)
+    if (headerEnd === -1) break
+
+    const header = buffer.toString('utf8', offset, headerEnd)
+    const size = Number(header.split(' ')[2])
+    if (!Number.isFinite(size)) break
+
+    const start = headerEnd + 1
+    files.push({ path, text: buffer.toString('utf8', start, start + size) })
+    offset = start + size + 1
+  }
+
+  return files
+}
+
+/** Counts one revision's tree, reusing the same pass the snapshot arm runs. */
+export async function scanRevision(
+  root: string,
+  rev: string,
+  opts: ScanOptions = {},
+): Promise<LanguageCount[]> {
+  const files = await readRevision(root, rev, opts.languages ?? LANGUAGES)
+  return countFiles(files, opts)
+}
+
+export interface TrendOptions extends ScanOptions {
+  readonly since: string
+  readonly points?: number
+}
+
+/** Recomputes the series from git rather than reading a stored ledger. */
+export async function trend(
+  root: string,
+  opts: TrendOptions,
+): Promise<TrendPoint[]> {
+  const commits = await listCommits(root, opts.since)
+  const sampled = spaceEvenly(commits, opts.points ?? DEFAULT_POINTS)
+
+  // Each point is an independent pair of git reads, and the sample is bounded
+  // by `points`, so the whole series costs one revision's wall clock.
+  return Promise.all(
+    sampled.map(async ({ rev, date }) => ({
+      rev,
+      date,
+      languages: await scanRevision(root, rev, opts),
+    })),
+  )
+}

package/src/comments/vocabulary.ts

@@ -0,0 +1,85 @@
+import { existsSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+
+/**
+ * The heading a rule carries to publish its degradation list.
+ *
+ * Discovery is anchored on this rather than on a filename, because governance
+ * rules are numbered and a renumber would silently empty the vocabulary while
+ * the sweep kept reporting clean. A heading survives the rename.
+ */
+export const VOCABULARY_HEADING = '## Degradation vocabulary'
+
+/**
+ * Roots searched in order. The installed copy wins over the toolkit source, so
+ * a target project measures against the rule it actually has rather than one
+ * only the toolkit carries.
+ */
+const RULE_ROOTS = ['.claude/rules', 'governance/rules']
+
+/**
+ * Absent is a distinct state from empty.
+ *
+ * A sweep with no vocabulary finds nothing, and reporting that as zero hits
+ * claims the codebase is clean when nothing was actually looked for. The
+ * command reports it as skipped instead.
+ */
+export type Vocabulary =
+  | {
+      readonly kind: 'loaded'
+      readonly source: string
+      readonly terms: string[]
+    }
+  | { readonly kind: 'absent' }
+
+/** Pulls the backticked terms out of the bullets under the vocabulary heading. */
+export function parseVocabulary(markdown: string): string[] | undefined {
+  const lines = markdown.split('\n')
+  const start = lines.findIndex((line) => line.trim() === VOCABULARY_HEADING)
+  if (start === -1) return undefined
+
+  const terms: string[] = []
+
+  for (const line of lines.slice(start + 1)) {
+    if (line.startsWith('## ')) break
+    for (const match of line.matchAll(/`([^`]+)`/g)) {
+      const term = match[1].trim()
+      if (term && !terms.includes(term)) terms.push(term)
+    }
+  }
+
+  return terms
+}
+
+/**
+ * Finds the rule publishing the vocabulary under `root`.
+ *
+ * Reading the list out of the rule rather than hardcoding it is what keeps one
+ * definition when the rule installs into a target, the same way
+ * `.claude/hooks/standards-audit.sh` reads its bans out of `prose.md`.
+ */
+export async function loadVocabulary(root: string): Promise<Vocabulary> {
+  for (const ruleRoot of RULE_ROOTS) {
+    const dir = resolve(root, ruleRoot)
+    if (!existsSync(dir)) continue
+
+    const paths: string[] = []
+    for await (const rel of new Bun.Glob('**/*.md').scan({
+      cwd: dir,
+      onlyFiles: true,
+    })) {
+      paths.push(rel)
+    }
+    paths.sort()
+
+    for (const rel of paths) {
+      const parsed = parseVocabulary(await readFile(resolve(dir, rel), 'utf8'))
+      if (parsed && parsed.length > 0) {
+        return { kind: 'loaded', source: `${ruleRoot}/${rel}`, terms: parsed }
+      }
+    }
+  }
+
+  return { kind: 'absent' }
+}

package/src/git-env.ts

@@ -0,0 +1,36 @@
+/**
+ * Repository-resolution variables git reads from the environment.
+ *
+ * A git hook exports these into every process it runs, and they take
+ * precedence over `-C`. A command scoped to a subtree then silently reads
+ * whatever the hook's repository points at and reports figures for a tree
+ * nobody asked about, which is worse than failing because the output looks
+ * ordinary. `GIT_PREFIX` is here for the same reason: it re-anchors a relative
+ * pathspec to the directory the hook was invoked from.
+ */
+const RESOLUTION_VARS = [
+  'GIT_DIR',
+  'GIT_WORK_TREE',
+  'GIT_COMMON_DIR',
+  'GIT_INDEX_FILE',
+  'GIT_OBJECT_DIRECTORY',
+  'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+  'GIT_NAMESPACE',
+  'GIT_PREFIX',
+]
+
+/**
+ * Returns the ambient environment with git's repository-resolution variables
+ * removed, so `git -C <path>` resolves against `<path>` and nothing else.
+ *
+ * Read per call rather than captured at module load. A long-lived process can
+ * have these set after import, and a snapshot would then hand git the very
+ * variables this exists to strip.
+ */
+export function gitEnv(): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(process.env).filter(
+      ([key, value]) => value !== undefined && !RESOLUTION_VARS.includes(key),
+    ),
+  ) as Record<string, string>
+}

package/src/git-ignore.ts

@@ -0,0 +1,46 @@
+import { resolve } from 'node:path'
+import { $ } from 'bun'
+import { gitEnv } from '@/git-env'
+
+/**
+ * Reports which of `candidates` git ignores under `root`.
+ *
+ * Batched through one `check-ignore --stdin` rather than a call per path,
+ * since a whole-repo walk hands this thousands of candidates. Outside a git
+ * repo nothing is ignored, so a caller's segment prune becomes the only filter
+ * that applies.
+ */
+export async function listIgnored(
+  root: string,
+  candidates: string[],
+): Promise<Set<string>> {
+  if (candidates.length === 0) return new Set()
+
+  const isRepo = await $`git -C ${root} rev-parse --git-dir`
+    .env(gitEnv())
+    .quiet()
+    .nothrow()
+    .then((result) => result.exitCode === 0)
+
+  if (!isRepo) return new Set()
+
+  const stdin = Buffer.from(`${candidates.join('\n')}\n`)
+
+  const result = await $`git -C ${root} check-ignore --stdin < ${stdin}`
+    .env(gitEnv())
+    .quiet()
+    .nothrow()
+
+  // Exit 1 means nothing matched, which is a clean result rather than a
+  // failure. Anything above that is a real error and degrades to "ignores
+  // nothing" so a broken git never silently shrinks the scanned set.
+  if (result.exitCode > 1) return new Set()
+
+  return new Set(
+    result
+      .text()
+      .split('\n')
+      .filter(Boolean)
+      .map((path) => resolve(root, path)),
+  )
+}