clckernel 1.2.19 → 2.0.1

package/tools/scan_secrets.py

@@ -3,6 +3,16 @@
 Secrets & API Key Scanner
 Detects hardcoded secrets, API keys, tokens, and private keys in changed files.
 Self-contained: uses only Python stdlib (re, pathlib, subprocess, json).
+
+Rules (sdds/secret-patterns) are tried in order; the first match labels the
+line (one finding per line). They mirror RULES in scan_secrets.js exactly,
+same regex sources and flags. The two generic rules skip placeholder values
+(see _is_placeholder):
+  - Quoted Key Secret: a quoted JSON/YAML key ending in a secret word, with a
+    quoted value of 8+ non-space characters holding a letter and a digit;
+  - Dotenv Secret: in `.env`, `.env.*` and `*.env` files (not `.env.example`,
+    `.env.sample`, `.env.template`), `NAME=value` where a `_` segment of NAME
+    is a secret word and the value is not numeric-only or a boolean/null word.
 """
 from __future__ import annotations
 
@@ -10,33 +20,120 @@ import json
 import os
 import pathlib
 import re
-import subprocess
 import sys
+from typing import Callable, List, NamedTuple, Optional
+
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
+
+SECRET_KEY_WORDS = (
+    r"api[_-]?key|apikey|secret[_-]?key|api[_-]?secret|client[_-]?secret|secret|"
+    r"access[_-]?token|auth[_-]?token|refresh[_-]?token|token|password|passwd|private[_-]?key"
+)
+DOTENV_SECRET_NAME = re.compile(
+    r"(?:^|_)(?:API_?KEY|KEY|SECRET|TOKEN|PASSWORD|PASSWD|PWD|PASS|CREDENTIALS?|DSN|PRIVATE)(?:_|$)",
+    re.IGNORECASE,
+)
+PLACEHOLDER_WORDS = (
+    "xxx", "your", "changeme", "change_me", "change-me", "placeholder", "example",
+    "dummy", "replace", "insert", "redacted", "fake", "sample", "todo", "...", "***",
+)
+NON_SECRET_WORDS = ("true", "false", "yes", "no", "on", "off", "null", "none")
+TEMPLATE_SEGMENTS = ("example", "sample", "template")
 
-# One finding per matching line: (line_no, label, matched_snippet)
-PATTERNS: list[tuple[str, str]] = [
-    (r"-----BEGIN (?:RSA |EC )?PRIVATE KEY-----", "Private Key"),
-    (r"AKIA[0-9A-Z]{16}", "AWS Access Key"),
-    (
-        r"(?:api[_-]?key|apikey|secret[_-]?key|api[_-]?secret)\s*[=:]\s*['\"]"
-        r"[A-Za-z0-9_\-]{16,}['\"]",
-        "API Key",
-    ),
-    (
-        r"(?:password|passwd|pwd)\s*[=:]\s*['\"][^'\"]{6,}['\"]",
-        "Hardcoded Password",
+
+def _is_placeholder(value: str) -> bool:
+    """True for values that are obviously not a real secret."""
+    v = value.lower()
+    if v.startswith("$") or "${" in v or "$(" in v or "{{" in v:
+        return True
+    if "<" in v and ">" in v:
+        return True
+    if re.match(r"^(.)\1*$", v):
+        return True
+    return any(w in v for w in PLACEHOLDER_WORDS)
+
+
+def _dotenv_value(raw: str) -> str:
+    """The value of a dotenv assignment: quotes removed, or an unquoted value without its ` #` comment."""
+    quoted = re.match(r"^([\"'])([^\n]*?)\1", raw)
+    if quoted:
+        return quoted.group(2)
+    comment = re.search(r"\s#", raw)
+    return (raw if comment is None else raw[: comment.start()]).strip()
+
+
+def _is_dotenv_file(filename: str) -> bool:
+    """`.env`, `.env.<suffix>` or `<name>.env`, unless a dot segment marks it as a template."""
+    base = os.path.basename(filename or "")
+    if not (base == ".env" or base.startswith(".env.") or (base.endswith(".env") and len(base) > 4)):
+        return False
+    return not any(seg in TEMPLATE_SEGMENTS for seg in base.lower().split("."))
+
+
+def _accept_quoted(m: re.Match) -> bool:
+    value = m.group(3)
+    return bool(re.search(r"[A-Za-z]", value)) and bool(re.search(r"[0-9]", value)) and not _is_placeholder(value)
+
+
+def _accept_dotenv(m: re.Match) -> bool:
+    if not DOTENV_SECRET_NAME.search(m.group(1)):
+        return False
+    value = _dotenv_value(m.group(2).strip())
+    if not value or re.match(r"^[0-9]+$", value) or value.lower() in NON_SECRET_WORDS:
+        return False
+    return not _is_placeholder(value)
+
+
+class Rule(NamedTuple):
+    regex: "re.Pattern[str]"
+    label: str
+    accept: Optional[Callable[[re.Match], bool]] = None
+    dotenv_only: bool = False
+
+
+def _rule(source: str, label: str, ignore_case: bool = False, **kw) -> Rule:
+    return Rule(re.compile(source, re.IGNORECASE if ignore_case else 0), label, **kw)
+
+
+# One finding per matching line: {lineNo, col, label, snippet}; the snippet is redacted.
+RULES: List[Rule] = [
+    _rule(r"-----BEGIN (?:RSA |EC )?PRIVATE KEY-----", "Private Key"),
+    _rule(r"AKIA[0-9A-Z]{16}", "AWS Access Key"),
+    _rule(
+        r"(?:api[_-]?key|apikey|secret[_-]?key|api[_-]?secret)\s*[=:]\s*['\"][A-Za-z0-9_\-]{16,}['\"]",
+        "API Key", ignore_case=True,
     ),
-    (
-        r"(?:secret|token|auth[_-]?token|bearer)\s*[=:]\s*['\"]"
-        r"[A-Za-z0-9_\-\.]{16,}['\"]",
-        "Secret/Token",
+    _rule(r"(?:password|passwd|pwd)\s*[=:]\s*['\"][^'\"]{6,}['\"]", "Hardcoded Password", ignore_case=True),
+    _rule(
+        r"(?:secret|token|auth[_-]?token|bearer)\s*[=:]\s*['\"][A-Za-z0-9_\-\.]{16,}['\"]",
+        "Secret/Token", ignore_case=True,
     ),
-    (r"ghp_[A-Za-z0-9]{36}", "GitHub Personal Access Token"),
-    (r"sk-[A-Za-z0-9]{20,}", "OpenAI/Stripe API Key"),
-    (r"eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", "JWT Token"),
-    (
+    _rule(r"ghp_[A-Za-z0-9]{36}", "GitHub Personal Access Token"),
+    _rule(r"sk-[A-Za-z0-9]{20,}", "OpenAI API Key"),
+    _rule(r"(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}", "Stripe API Key"),
+    _rule(r"eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", "JWT Token"),
+    _rule(
         r"(?:SUPABASE|NEXT_PUBLIC|VITE)_\w*_KEY\s*=\s*['\"][^'\"]{8,}['\"]",
-        "Framework Secret Key",
+        "Framework Secret Key", ignore_case=True,
+    ),
+    _rule(r"sk-ant-[A-Za-z0-9_-]{20,}", "Anthropic API Key"),
+    _rule(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{20,}", "OpenAI Project Key"),
+    _rule(r"AIza[0-9A-Za-z_-]{35}", "Google API Key"),
+    _rule(r"xox[abprs]-[0-9]+-[0-9A-Za-z-]{8,}", "Slack Token"),
+    _rule(r"gh[ousr]_[A-Za-z0-9]{36}", "GitHub Token"),
+    _rule(r"github_pat_[A-Za-z0-9_]{40,}", "GitHub Fine-Grained PAT"),
+    _rule(
+        r"([\"'])[A-Za-z0-9_.-]*(?:" + SECRET_KEY_WORDS + r")\1\s*:\s*([\"'])([^\"'\s]{8,})\2",
+        "Quoted Key Secret", ignore_case=True, accept=_accept_quoted,
+    ),
+    _rule(
+        r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([^\n]*)$",
+        "Dotenv Secret", accept=_accept_dotenv, dotenv_only=True,
     ),
 ]
 
@@ -48,45 +145,56 @@ SKIP_EXTENSIONS = frozenset(
     }
 )
 
-
-def _get_changed_files(target_dir: str) -> list[str] | None:
-    """Return list of changed files from git diff, or None on failure."""
-    for ref in ("HEAD~1", "--cached"):
-        try:
-            args = ["git", "diff", "--name-only"]
-            if ref == "--cached":
-                args.append("--cached")
-            else:
-                args.append(ref)
-            out = subprocess.check_output(
-                args, cwd=target_dir,
-                stderr=subprocess.DEVNULL, text=True,
-            )
-            files = [f.strip() for f in out.splitlines() if f.strip()]
-            if files:
-                return files
-        except (subprocess.CalledProcessError, FileNotFoundError):
-            continue
-    return None
+REDACTED = "…[REDACTED]"
 
 
-def _compile_patterns() -> list[tuple[re.Pattern, str]]:
-    """Compile all secret-detection patterns once."""
-    return [(re.compile(p), label) for p, label in PATTERNS]
-
-
-def check(target_dir: str | None = None) -> dict:
+def _redact(line: str, rules: List[Rule]) -> str:
+    """
+    The line with every match of every applicable rule cut to its first 4
+    characters plus a marker, stripped and capped at 120 characters. Filters
+    are ignored here: over-redaction is safe. The full matched value never
+    leaves the guard.
+    """
+    out = line
+    for rule in rules:
+        out = rule.regex.sub(lambda m: m.group(0)[:4] + REDACTED, out)
+    return out.strip()[:120]
+
+
+def find_secrets(text: str, filename: str) -> List[dict]:
+    """Pure matcher: the findings for `text` as the content of `filename` (repo-relative path)."""
+    dotenv = _is_dotenv_file(filename)
+    rules = [r for r in RULES if dotenv or not r.dotenv_only]
+    findings: List[dict] = []
+    # Split on "\n" only, like the JS guard, so line numbers and snippets match.
+    for line_no, line in enumerate(str(text).split("\n"), start=1):
+        for rule in rules:
+            m = rule.regex.search(line)
+            if m and (rule.accept is None or rule.accept(m)):
+                findings.append({
+                    "lineNo": line_no,
+                    "col": m.start(),
+                    "label": rule.label,
+                    "snippet": _redact(line, rules),
+                })
+                break  # one finding per line
+    return findings
+
+
+def check(target_dir: Optional[str] = None) -> dict:
     """
     Scan changed files for secrets and API keys.
 
     Returns:
         dict with keys: exit_code, skipped, name, data
-        data.findings maps filepath -> list of {lineNo, label, snippet}
+        data.findings maps filepath -> list of {lineNo, col, label, snippet}
     """
     dir_path = target_dir or os.getcwd()
 
-    files = _get_changed_files(dir_path)
-    if files is None:
+    # The snapshot being verified: the index, or HEAD when nothing is staged.
+    changes = _changes.changed_files(dir_path)
+    mode, files = changes["mode"], changes["files"]
+    if mode is None:
         return {
             "exit_code": 0, "skipped": True, "name": "scan_secrets",
             "data": {"findings": {}, "message": "No git diff available"},
@@ -98,35 +206,18 @@ def check(target_dir: str | None = None) -> dict:
             "data": {"findings": {}, "message": "No changed files to scan"},
         }
 
-    compiled = _compile_patterns()
-    findings: dict[str, list[dict]] = {}
+    findings: dict = {}
 
     for filepath in files:
         ext = pathlib.Path(filepath).suffix
         if ext in SKIP_EXTENSIONS:
             continue
 
-        full_path = pathlib.Path(dir_path) / filepath
-        if not full_path.exists():
+        content = _changes.read_file(dir_path, filepath, mode)
+        if content is None:
             continue
 
-        try:
-            content = full_path.read_text(encoding="utf-8", errors="replace")
-        except (OSError, PermissionError):
-            continue
-
-        file_hits: list[dict] = []
-        for line_no, line in enumerate(content.splitlines(), start=1):
-            for regex, label in compiled:
-                m = regex.search(line)
-                if m:
-                    file_hits.append({
-                        "lineNo": line_no,
-                        "label": label,
-                        "snippet": line.strip()[:120],
-                    })
-                    break  # one finding per line
-
+        file_hits = find_secrets(content, filepath)
         if file_hits:
             findings[filepath] = file_hits