@erclx/aitk 2.1.0 → 3.0.0

package/src/secrets/patterns.ts

@@ -0,0 +1,142 @@
+/**
+ * The rule set the secret scan keys on.
+ *
+ * Every pattern matches an issued value and none of them matches a word. That
+ * split is what lets the exclusion set start empty: a scan keyed on `password`,
+ * `secret`, or `token` fires on the environment reads, the workflow inputs, and
+ * the prose that name those things, and this repository ships all three. The
+ * cost is a credential no issuer stamps with a recognizable prefix, which this
+ * set does not reach and no exclusion would have helped with either.
+ *
+ * None of these sources matches itself, so this file is in scope like any
+ * other. Each literal prefix is followed here by a character class rather than
+ * by the class's own members, and `src/secrets/scan.test.ts` holds the check.
+ */
+export interface SecretPattern {
+  readonly id: string
+  readonly label: string
+  /** Carries the global flag, since a line may hold more than one value. */
+  readonly match: RegExp
+  /**
+   * Whether the matched text is the credential itself.
+   *
+   * A private key header names a block without carrying its bytes, so echoing
+   * it whole tells the reader what was found. Every other pattern matches the
+   * value, and reporting one in full would copy a live credential into a log.
+   */
+  readonly redact: boolean
+}
+
+export const PATTERNS: readonly SecretPattern[] = [
+  {
+    id: 'aws-access-key-id',
+    label: 'AWS access key id',
+    match: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g,
+    redact: true,
+  },
+  {
+    id: 'github-token',
+    label: 'GitHub token',
+    // Spelled as alternation rather than a character class, so the source
+    // carries the issued prefixes as themselves. A class reads as one
+    // pronounceable token to a spell checker and puts a nonsense word in a
+    // dictionary that is supposed to hold real terms.
+    match: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'github-fine-grained-token',
+    label: 'GitHub fine-grained token',
+    match: /\bgithub_pat_[A-Za-z0-9_]{50,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'google-api-key',
+    label: 'Google API key',
+    match: /\bAIza[0-9A-Za-z_-]{35}\b/g,
+    redact: true,
+  },
+  {
+    id: 'slack-token',
+    label: 'Slack token',
+    match: /\b(?:xoxa|xoxb|xoxp|xoxr|xoxs)-[0-9A-Za-z-]{12,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'slack-webhook',
+    label: 'Slack webhook',
+    match: /https:\/\/hooks\.slack\.com\/services\/T[0-9A-Za-z_/-]{20,}/g,
+    redact: true,
+  },
+  {
+    id: 'stripe-secret-key',
+    label: 'Stripe live key',
+    match: /\b(?:sk|rk)_live_[0-9A-Za-z]{20,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'anthropic-api-key',
+    label: 'Anthropic API key',
+    match: /\bsk-ant-[0-9A-Za-z_-]{24,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'openai-api-key',
+    label: 'OpenAI project key',
+    match: /\bsk-proj-[0-9A-Za-z_-]{24,}\b/g,
+    redact: true,
+  },
+  {
+    id: 'npm-token',
+    label: 'npm token',
+    match: /\bnpm_[0-9A-Za-z]{36}\b/g,
+    redact: true,
+  },
+  {
+    id: 'private-key-block',
+    label: 'Private key block',
+    match: /-----BEGIN(?: [A-Z0-9]+)* PRIVATE KEY-----/g,
+    redact: false,
+  },
+]
+
+export interface PatternHit {
+  readonly pattern: string
+  readonly label: string
+  /** One-based, so the report reads like every other file reference here. */
+  readonly column: number
+  /** What the report prints, redacted unless the pattern says otherwise. */
+  readonly preview: string
+}
+
+/**
+ * Shortens a matched value to its two ends.
+ *
+ * The ends are what a reader needs to find the credential in the file and to
+ * tell one match from another, and the middle is the part no report should
+ * carry. A value too short to have a middle is reported as its shape alone.
+ */
+function redact(value: string): string {
+  if (value.length <= 8) return '…'
+  return `${value.slice(0, 4)}…${value.slice(-4)}`
+}
+
+/** Every value on one line, ordered by where each starts. */
+export function matchLine(line: string): PatternHit[] {
+  const hits: PatternHit[] = []
+
+  for (const pattern of PATTERNS) {
+    for (const found of line.matchAll(pattern.match)) {
+      if (found.index === undefined) continue
+
+      hits.push({
+        pattern: pattern.id,
+        label: pattern.label,
+        column: found.index + 1,
+        preview: pattern.redact ? redact(found[0]) : found[0],
+      })
+    }
+  }
+
+  return hits.sort((left, right) => left.column - right.column)
+}

package/src/secrets/scan.ts

@@ -0,0 +1,134 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { listRepositoryFiles } from '@/git-files'
+import { isExempt } from '@/secrets/marker'
+import { matchLine } from '@/secrets/patterns'
+import { readShipEntries, selectShipped } from '@/secrets/shipped'
+
+export interface SecretFinding {
+  readonly file: string
+  /** One-based, matching the `file:line` form a reader clicks. */
+  readonly line: number
+  readonly column: number
+  readonly pattern: string
+  readonly label: string
+  readonly preview: string
+}
+
+/**
+ * Why a scan produced no corpus, which is never the same as a clean one.
+ *
+ * Three of these mean this tree publishes nothing, and `no-files-field` means
+ * the opposite: a publish would pack everything and this check read none of it.
+ * They are separate reasons because the aggregate answers them differently,
+ * and folding them together is what let one message deny the case another
+ * comment named.
+ */
+export type ScanRefusal =
+  | 'no-manifest'
+  | 'no-publish'
+  | 'no-files-field'
+  | 'no-git'
+  | 'no-shipped-files'
+
+export type SecretScan =
+  | {
+      readonly kind: 'scanned'
+      /** Files opened, so a report can state what the verdict covers. */
+      readonly files: number
+      readonly skipped: number
+      /**
+       * Everything git lists, so the report states its own bound.
+       *
+       * A count of what passed reads as a verdict on the repository unless the
+       * run also says how much of it the corpus left out.
+       */
+      readonly listed: number
+      readonly findings: readonly SecretFinding[]
+    }
+  | { readonly kind: 'refused'; readonly reason: ScanRefusal }
+
+/**
+ * Whether the bytes are something a line scanner should not read.
+ *
+ * A NUL byte rather than an extension list, since the shipped tree carries
+ * fonts and images under names this check has no reason to enumerate, and a
+ * list would go stale the first time a format was added. Decoded text holds no
+ * NUL, so the test costs one scan and never rejects source.
+ */
+export function isBinary(text: string): boolean {
+  return text.includes('\0')
+}
+
+/** Every finding in one file's text, with the marker already applied. */
+export function scanText(file: string, text: string): SecretFinding[] {
+  const lines = text.split('\n')
+  const findings: SecretFinding[] = []
+
+  for (const [index, line] of lines.entries()) {
+    const hits = matchLine(line)
+    if (hits.length === 0 || isExempt(lines, index)) continue
+
+    for (const hit of hits) {
+      findings.push({ file, line: index + 1, ...hit })
+    }
+  }
+
+  return findings
+}
+
+/**
+ * Scans the tree this repository publishes, not the tree it holds.
+ *
+ * A refusal rather than an empty result wherever the corpus cannot be built.
+ * Zero findings over zero files reads in the report exactly like zero findings
+ * over the whole shipped tree, and the two mean opposite things, which is the
+ * split every other audit here already draws.
+ */
+export async function scanShippedTree(root: string): Promise<SecretScan> {
+  const declared = await readShipEntries(root)
+  if (declared.kind !== 'entries') {
+    return { kind: 'refused', reason: declared.kind }
+  }
+
+  const listed = await listRepositoryFiles(root)
+  if (listed === undefined) return { kind: 'refused', reason: 'no-git' }
+
+  const paths = selectShipped(listed, declared.entries)
+  if (paths.length === 0) {
+    return { kind: 'refused', reason: 'no-shipped-files' }
+  }
+
+  const findings: SecretFinding[] = []
+  let scanned = 0
+  let skipped = 0
+
+  for (const path of paths) {
+    let text: string
+    try {
+      text = await readFile(join(root, path), 'utf8')
+    } catch {
+      // A listed path that will not open is a symlink pointing outside the
+      // tree or a file removed since git answered. Counted rather than
+      // reported, so the run still states that it measured less than it listed.
+      skipped += 1
+      continue
+    }
+
+    if (isBinary(text)) {
+      skipped += 1
+      continue
+    }
+
+    scanned += 1
+    findings.push(...scanText(path, text))
+  }
+
+  return {
+    kind: 'scanned',
+    files: scanned,
+    skipped,
+    listed: listed.length,
+    findings,
+  }
+}

package/src/secrets/shipped.ts

@@ -0,0 +1,111 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+/**
+ * The corpus is the package's own `files` field rather than a list kept here.
+ *
+ * That field is what npm packs, so it is already the single statement of which
+ * trees leave this repository, and a second list beside it would answer the
+ * same question and drift. It also carries the negations the publish already
+ * makes, so the sandbox tree, the eval tree, and every test file are out of
+ * scope by the same rule that keeps them out of the tarball rather than by an
+ * exclusion this check invented.
+ *
+ * What it does not cover is the plugin, which a marketplace install reads live
+ * from `claude/` rather than from a tarball. That folder is a `files` entry
+ * too, and its `standards` and `snippets` symlinks resolve into trees the field
+ * lists in their own right, so both routes land inside the same corpus.
+ */
+export type ShipEntries =
+  /** The field declares a corpus, which is what this check reads. */
+  | { readonly kind: 'entries'; readonly entries: readonly string[] }
+  /** No manifest at all, so nothing is published from this tree. */
+  | { readonly kind: 'no-manifest' }
+  /** The manifest declares it is never published, so there is no shipped tree. */
+  | { readonly kind: 'no-publish' }
+  /**
+   * A manifest that publishes and declares no corpus.
+   *
+   * npm packs the whole tree in that case, so this is the package that ships
+   * the most rather than one that ships nothing. This check reads a declared
+   * corpus and does not stand in an undeclared one, so the caller reports the
+   * shipped tree as unread rather than as empty.
+   */
+  | { readonly kind: 'no-files-field' }
+
+export async function readShipEntries(root: string): Promise<ShipEntries> {
+  let manifest: unknown
+  try {
+    manifest = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'))
+  } catch {
+    return { kind: 'no-manifest' }
+  }
+
+  const record = manifest as { files?: unknown; private?: unknown } | null
+
+  // The one field that separates a project publishing nothing from one
+  // publishing everything, which the `files` field alone cannot tell apart.
+  if (record?.private === true) return { kind: 'no-publish' }
+
+  const files = record?.files
+  if (!Array.isArray(files)) return { kind: 'no-files-field' }
+
+  const entries = files.filter(
+    (entry): entry is string => typeof entry === 'string',
+  )
+
+  return entries.length === 0
+    ? { kind: 'no-files-field' }
+    : { kind: 'entries', entries }
+}
+
+/**
+ * Root files npm packs whether or not the `files` field names them.
+ *
+ * They leave the repository on every publish, so a corpus built from the field
+ * alone would let a credential in any of them ship unreported. Matched by stem
+ * against any extension, since each is packed under whichever one it carries.
+ */
+const ALWAYS_PACKED = ['package.json', 'readme', 'license', 'licence', 'notice']
+
+function isAlwaysPacked(path: string): boolean {
+  if (path.includes('/')) return false
+
+  const stem = path.toLowerCase().split('.')[0] ?? ''
+  return (
+    ALWAYS_PACKED.includes(path.toLowerCase()) || ALWAYS_PACKED.includes(stem)
+  )
+}
+
+/** Whether the entry reaches this path, as a directory prefix or as a glob. */
+function covers(entry: string, path: string): boolean {
+  if (entry.includes('*')) return new Bun.Glob(entry).match(path)
+
+  return path === entry || path.startsWith(`${entry}/`)
+}
+
+/**
+ * Narrows a repository listing to what the package publishes.
+ *
+ * Negations are collected first and applied to every candidate, since npm
+ * reads the field as one set rather than in order, and an entry's position in
+ * the array says nothing about what it overrides.
+ */
+export function selectShipped(
+  paths: readonly string[],
+  entries: readonly string[],
+): string[] {
+  const included = entries.filter((entry) => !entry.startsWith('!'))
+  const excluded = entries
+    .filter((entry) => entry.startsWith('!'))
+    .map((entry) => entry.slice(1))
+
+  return paths
+    .filter(
+      (path) =>
+        (isAlwaysPacked(path) ||
+          included.some((entry) => covers(entry, path))) &&
+        !excluded.some((entry) => covers(entry, path)),
+    )
+    .sort()
+}

package/src/sync/stamp.ts

@@ -53,7 +53,7 @@ export interface Stamp {
 }
 
 export function stampPath(target: string): string {
-  return join(target, '.claude', 'aitk.json')
+  return join(target, '.claude', 'aitk', 'config.json')
 }
 
 export function hashContent(content: Buffer | string): string {