@erclx/aitk 2.1.0 → 2.2.0

package/src/secrets/scan.ts

@@ -0,0 +1,134 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { listRepositoryFiles } from '@/git-files'
+import { isExempt } from '@/secrets/marker'
+import { matchLine } from '@/secrets/patterns'
+import { readShipEntries, selectShipped } from '@/secrets/shipped'
+
+export interface SecretFinding {
+  readonly file: string
+  /** One-based, matching the `file:line` form a reader clicks. */
+  readonly line: number
+  readonly column: number
+  readonly pattern: string
+  readonly label: string
+  readonly preview: string
+}
+
+/**
+ * Why a scan produced no corpus, which is never the same as a clean one.
+ *
+ * Three of these mean this tree publishes nothing, and `no-files-field` means
+ * the opposite: a publish would pack everything and this check read none of it.
+ * They are separate reasons because the aggregate answers them differently,
+ * and folding them together is what let one message deny the case another
+ * comment named.
+ */
+export type ScanRefusal =
+  | 'no-manifest'
+  | 'no-publish'
+  | 'no-files-field'
+  | 'no-git'
+  | 'no-shipped-files'
+
+export type SecretScan =
+  | {
+      readonly kind: 'scanned'
+      /** Files opened, so a report can state what the verdict covers. */
+      readonly files: number
+      readonly skipped: number
+      /**
+       * Everything git lists, so the report states its own bound.
+       *
+       * A count of what passed reads as a verdict on the repository unless the
+       * run also says how much of it the corpus left out.
+       */
+      readonly listed: number
+      readonly findings: readonly SecretFinding[]
+    }
+  | { readonly kind: 'refused'; readonly reason: ScanRefusal }
+
+/**
+ * Whether the bytes are something a line scanner should not read.
+ *
+ * A NUL byte rather than an extension list, since the shipped tree carries
+ * fonts and images under names this check has no reason to enumerate, and a
+ * list would go stale the first time a format was added. Decoded text holds no
+ * NUL, so the test costs one scan and never rejects source.
+ */
+export function isBinary(text: string): boolean {
+  return text.includes('\0')
+}
+
+/** Every finding in one file's text, with the marker already applied. */
+export function scanText(file: string, text: string): SecretFinding[] {
+  const lines = text.split('\n')
+  const findings: SecretFinding[] = []
+
+  for (const [index, line] of lines.entries()) {
+    const hits = matchLine(line)
+    if (hits.length === 0 || isExempt(lines, index)) continue
+
+    for (const hit of hits) {
+      findings.push({ file, line: index + 1, ...hit })
+    }
+  }
+
+  return findings
+}
+
+/**
+ * Scans the tree this repository publishes, not the tree it holds.
+ *
+ * A refusal rather than an empty result wherever the corpus cannot be built.
+ * Zero findings over zero files reads in the report exactly like zero findings
+ * over the whole shipped tree, and the two mean opposite things, which is the
+ * split every other audit here already draws.
+ */
+export async function scanShippedTree(root: string): Promise<SecretScan> {
+  const declared = await readShipEntries(root)
+  if (declared.kind !== 'entries') {
+    return { kind: 'refused', reason: declared.kind }
+  }
+
+  const listed = await listRepositoryFiles(root)
+  if (listed === undefined) return { kind: 'refused', reason: 'no-git' }
+
+  const paths = selectShipped(listed, declared.entries)
+  if (paths.length === 0) {
+    return { kind: 'refused', reason: 'no-shipped-files' }
+  }
+
+  const findings: SecretFinding[] = []
+  let scanned = 0
+  let skipped = 0
+
+  for (const path of paths) {
+    let text: string
+    try {
+      text = await readFile(join(root, path), 'utf8')
+    } catch {
+      // A listed path that will not open is a symlink pointing outside the
+      // tree or a file removed since git answered. Counted rather than
+      // reported, so the run still states that it measured less than it listed.
+      skipped += 1
+      continue
+    }
+
+    if (isBinary(text)) {
+      skipped += 1
+      continue
+    }
+
+    scanned += 1
+    findings.push(...scanText(path, text))
+  }
+
+  return {
+    kind: 'scanned',
+    files: scanned,
+    skipped,
+    listed: listed.length,
+    findings,
+  }
+}

package/src/secrets/shipped.ts

@@ -0,0 +1,111 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+/**
+ * The corpus is the package's own `files` field rather than a list kept here.
+ *
+ * That field is what npm packs, so it is already the single statement of which
+ * trees leave this repository, and a second list beside it would answer the
+ * same question and drift. It also carries the negations the publish already
+ * makes, so the sandbox tree, the eval tree, and every test file are out of
+ * scope by the same rule that keeps them out of the tarball rather than by an
+ * exclusion this check invented.
+ *
+ * What it does not cover is the plugin, which a marketplace install reads live
+ * from `claude/` rather than from a tarball. That folder is a `files` entry
+ * too, and its `standards` and `snippets` symlinks resolve into trees the field
+ * lists in their own right, so both routes land inside the same corpus.
+ */
+export type ShipEntries =
+  /** The field declares a corpus, which is what this check reads. */
+  | { readonly kind: 'entries'; readonly entries: readonly string[] }
+  /** No manifest at all, so nothing is published from this tree. */
+  | { readonly kind: 'no-manifest' }
+  /** The manifest declares it is never published, so there is no shipped tree. */
+  | { readonly kind: 'no-publish' }
+  /**
+   * A manifest that publishes and declares no corpus.
+   *
+   * npm packs the whole tree in that case, so this is the package that ships
+   * the most rather than one that ships nothing. This check reads a declared
+   * corpus and does not stand in an undeclared one, so the caller reports the
+   * shipped tree as unread rather than as empty.
+   */
+  | { readonly kind: 'no-files-field' }
+
+export async function readShipEntries(root: string): Promise<ShipEntries> {
+  let manifest: unknown
+  try {
+    manifest = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'))
+  } catch {
+    return { kind: 'no-manifest' }
+  }
+
+  const record = manifest as { files?: unknown; private?: unknown } | null
+
+  // The one field that separates a project publishing nothing from one
+  // publishing everything, which the `files` field alone cannot tell apart.
+  if (record?.private === true) return { kind: 'no-publish' }
+
+  const files = record?.files
+  if (!Array.isArray(files)) return { kind: 'no-files-field' }
+
+  const entries = files.filter(
+    (entry): entry is string => typeof entry === 'string',
+  )
+
+  return entries.length === 0
+    ? { kind: 'no-files-field' }
+    : { kind: 'entries', entries }
+}
+
+/**
+ * Root files npm packs whether or not the `files` field names them.
+ *
+ * They leave the repository on every publish, so a corpus built from the field
+ * alone would let a credential in any of them ship unreported. Matched by stem
+ * against any extension, since each is packed under whichever one it carries.
+ */
+const ALWAYS_PACKED = ['package.json', 'readme', 'license', 'licence', 'notice']
+
+function isAlwaysPacked(path: string): boolean {
+  if (path.includes('/')) return false
+
+  const stem = path.toLowerCase().split('.')[0] ?? ''
+  return (
+    ALWAYS_PACKED.includes(path.toLowerCase()) || ALWAYS_PACKED.includes(stem)
+  )
+}
+
+/** Whether the entry reaches this path, as a directory prefix or as a glob. */
+function covers(entry: string, path: string): boolean {
+  if (entry.includes('*')) return new Bun.Glob(entry).match(path)
+
+  return path === entry || path.startsWith(`${entry}/`)
+}
+
+/**
+ * Narrows a repository listing to what the package publishes.
+ *
+ * Negations are collected first and applied to every candidate, since npm
+ * reads the field as one set rather than in order, and an entry's position in
+ * the array says nothing about what it overrides.
+ */
+export function selectShipped(
+  paths: readonly string[],
+  entries: readonly string[],
+): string[] {
+  const included = entries.filter((entry) => !entry.startsWith('!'))
+  const excluded = entries
+    .filter((entry) => entry.startsWith('!'))
+    .map((entry) => entry.slice(1))
+
+  return paths
+    .filter(
+      (path) =>
+        (isAlwaysPacked(path) ||
+          included.some((entry) => covers(entry, path))) &&
+        !excluded.some((entry) => covers(entry, path)),
+    )
+    .sort()
+}