@matt82198/aesop 0.1.0-beta.3 → 0.1.0-beta.5

package/tools/scanner_selftest.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""
+Regression harness for secret_scan.py — validates scanner behavior across TP/FP cases.
+
+Usage: python scanner_selftest.py [--temp-dir DIR]
+
+Runs test cases against the scanner and reports pass/fail. Uses system temp by default.
+
+# secretscan: allow-pattern-docs
+"""
+import os
+import sys
+import subprocess
+import tempfile
+import shutil
+from pathlib import Path
+
+
+def run_test(name, file_content, filename, expected_exit, expected_keyword):
+    """Generate a test file, run scanner, verify exit code and output."""
+    # Create temp directory for this test
+    test_dir = Path(test_temp_dir) / name
+    test_dir.mkdir(parents=True, exist_ok=True)
+
+    test_file = test_dir / filename
+    test_file.write_text(file_content)
+
+    # Run secret_scan.py
+    try:
+        result = subprocess.run(
+            [sys.executable, scanner_path, str(test_file)],
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+    except subprocess.TimeoutExpired:
+        # Retry once after 30s as per instructions
+        import time
+        time.sleep(30)
+        try:
+            result = subprocess.run(
+                [sys.executable, scanner_path, str(test_file)],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+        except Exception as e:
+            return False, f"TIMEOUT/ERROR: {e}"
+    except Exception as e:
+        return False, f"ERROR: {e}"
+
+    # Check exit code
+    if result.returncode != expected_exit:
+        return False, f"Exit {result.returncode} (expected {expected_exit}). Output: {result.stdout} {result.stderr}"
+
+    # Check for expected keyword in output
+    output = result.stdout + result.stderr
+    if expected_keyword and expected_keyword not in output:
+        return False, f"Keyword '{expected_keyword}' not in output: {output}"
+
+    return True, "OK"
+
+
+def main():
+    global test_temp_dir, scanner_path
+
+    # Paths
+    scripts_dir = Path(__file__).parent
+    scanner_path = scripts_dir / "secret_scan.py"
+
+    if not scanner_path.exists():
+        print(f"ERROR: {scanner_path} not found")
+        return 1
+
+    # Create temp directory in system temp
+    with tempfile.TemporaryDirectory(prefix="scanner_selftest_") as tmpdir:
+        test_temp_dir = Path(tmpdir)
+
+        # Test cases: (name, file_content, filename, expected_exit, expected_keyword_in_output)
+        cases = [
+            (
+                "tp_ghp_token",
+                f"{'# GitHub token:'}\\ntoken = '{'ghp_' + 'a' * 32}'\\n",
+                "test.py",
+                1,
+                "token"
+            ),
+            (
+                "tp_akia_key",
+                f"{'# AWS key:'}\\naccess_key = '{'AKIA' + 'A' * 16}'\\n",
+                "config.py",
+                1,
+                "key"
+            ),
+            (
+                "tp_password_literal",
+                f"password = '{'hunter2secret99'}'\\n",
+                "app.py",
+                1,
+                "password"
+            ),
+            (
+                "tp_env_secret",
+                f"{'SECRET_KEY=realvalue123456'}\\n",
+                ".env",
+                1,
+                "FOUND"
+            ),
+            (
+                "tp_credentials_filename",
+                "some data here\n",
+                ".credentials.json",
+                1,
+                "FOUND"
+            ),
+            (
+                "fp_java_orderTokens",
+                "for (String oToken : orderTokens) {\\n    process(oToken);\\n}\\n",
+                "OrderProcessor.java",
+                0,
+                ""
+            ),
+            (
+                "fp_python_fstring",
+                "result = f\"txn:{inv.txn_id}\"\\n",
+                "invoice.py",
+                0,
+                ""
+            ),
+            (
+                "fp_clean_file",
+                "#!/usr/bin/env python3\\ndef hello():\\n    return 'world'\\n",
+                "clean.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                # Pragma is scoped to doc-shaped rules ONLY: a GitHub token is a
+                # fatal class and must block even with the pragma present.
+                "tp_pragma_github_token_still_fatal",
+                f"# secretscan: allow-pattern-docs\\ntoken = '{'ghp_' + 'b' * 32}'\\n",
+                "docs.py",
+                1,
+                "HIGH"
+            ),
+            (
+                # Fatal class: PEM private key + pragma must still exit 1.
+                # Header fragment-assembled so this source never holds the shape.
+                "tp_pragma_pem_still_fatal",
+                "# secretscan: allow-pattern-docs\\n"
+                + "-----BEGIN RSA " + "PRIVATE" + " KEY-----" + "\\n"
+                + "MIIEdummy1234567890abcdef\\n"
+                + "-----END RSA " + "PRIVATE" + " KEY-----" + "\\n",
+                "keydoc.txt",
+                1,
+                "pem_private_key"
+            ),
+            (
+                # Doc-shaped rule: generic assignment IS softened by pragma.
+                "tp_pragma_generic_softened",
+                "# secretscan: allow-pattern-docs\\n"
+                + "pass" + "word = \"this_is_a_doc_sample_value_12345\"\\n",
+                "pattern_docs.py",
+                0,
+                "ALLOWED-DOC"
+            ),
+            (
+                # Doc-shaped rule: env access IS softened by pragma.
+                "tp_pragma_env_access_softened",
+                "# secretscan: allow-pattern-docs\\n"
+                + "v = " + "os.envi" + "ron['API_" + "TOKEN']" + "\\n",
+                "env_docs.py",
+                0,
+                "ALLOWED-DOC"
+            ),
+            (
+                "fp_java_baseurl_constant",
+                'private static final String BASE_URL = "https://api.example.com";\\n',
+                "Config.java",
+                0,
+                "CLEAN"
+            ),
+            (
+                "fp_python_qbo_version",
+                "QBO_MINOR_VERSION = 65\\n",
+                "version.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                "fp_python_recv_batchsize",
+                "RECV_BATCH_SIZE = 200\\n",
+                "processor.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                "fp_env_amount_limit",
+                "AMOUNT_LIMIT=99999999\\n",
+                ".env",
+                0,
+                "CLEAN"
+            ),
+            (
+                "tp_env_secret_key_envfile",
+                "SECRET_KEY=realvalue123456\\n",
+                ".env",
+                1,
+                "FOUND"
+            ),
+            (
+                "fp_connection_localhost_fixture",
+                "# Test fixture: localhost DB\\ndb_url = 'postgresql://user:testpass@localhost:5432/testdb'\\n",
+                "test_db.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                "fp_connection_127_fixture",
+                "# Test fixture: loopback address\\ndb_url = 'mysql://admin:changeme@127.0.0.1:3306/test'\\n",
+                "test_config.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                "fp_connection_example_fixture",
+                "# Example URL from docs\\nurl = 'postgres://user:password@example.com:5432/db'\\n",
+                "docs_example.py",
+                0,
+                "CLEAN"
+            ),
+            (
+                # Fixture fragment-assembled so this selftest source never holds a
+                # contiguous connection-string shape (fatal class, pragma-immune).
+                "tp_connection_real_credentials",
+                "# Real production DB\\ndb_url = '"
+                + "postgresql:" + "//produser:RealPassword123"
+                + "@db.prod.example.com:5432/production" + "'\\n",
+                "config_prod.py",
+                1,
+                "connection_string"
+            ),
+            (
+                # Large-file bypass closed: >1MB file with a key in the first 1MB
+                # must still block (bounded FATAL_RULES scan; key runtime-assembled).
+                "tp_large_file_with_key",
+                "k = '" + "AKIA" + "A" * 16 + "'\n" + "x" * (1024 * 1024 + 100) + "\n",
+                "bigfile.txt",
+                1,
+                "aws_access_key"
+            ),
+            (
+                # Binary bypass closed: an embedded null byte no longer exempts the
+                # file; latin-1 decode + FATAL_RULES scan must still block.
+                "tp_binary_file_with_key",
+                "\x00binaryjunk\n" + "v = '" + "AKIA" + "A" * 16 + "'\n",
+                "blob.bin",
+                1,
+                "aws_access_key"
+            ),
+            (
+                # Large clean file: exit 0, with SKIPPED-LARGE note on stderr.
+                "fp_large_clean_file",
+                "x" * (1024 * 1024 + 100) + "\n",
+                "bigclean.log",
+                0,
+                "SKIPPED-LARGE"
+            ),
+        ]
+
+        results = []
+        for name, content, filename, exp_exit, exp_kw in cases:
+            passed, msg = run_test(name, content, filename, exp_exit, exp_kw)
+            results.append((name, passed, msg))
+            status = "PASS" if passed else "FAIL"
+            print(f"{status:5} {name:30} {msg}")
+
+        # Self-scan invariant: the scanner must scan its OWN source CLEAN with
+        # zero pragma reliance (fatal-class pattern literals are runtime-assembled).
+        try:
+            r = subprocess.run(
+                [sys.executable, str(scanner_path), str(scanner_path)],
+                capture_output=True, text=True, timeout=10,
+            )
+            self_ok = (r.returncode == 0 and "CLEAN" in r.stdout
+                       and "ALLOWED-DOC" not in r.stdout)
+            msg = "OK" if self_ok else f"Exit {r.returncode}. Output: {r.stdout} {r.stderr}"
+        except Exception as e:
+            self_ok, msg = False, f"ERROR: {e}"
+        results.append(("self_scan_clean_no_pragma", self_ok, msg))
+        print(f"{'PASS' if self_ok else 'FAIL':5} {'self_scan_clean_no_pragma':30} {msg}")
+
+        # Summary
+        passed_count = sum(1 for _, p, _ in results if p)
+        total_count = len(results)
+        print(f"\nSELFTEST: {passed_count}/{total_count} passed")
+
+        # Exit 0 only if all pass
+        return 0 if passed_count == total_count else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/secret_scan.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# secretscan: allow-pattern-docs
 """
 secret_scan.py — Pre-push secret/credential detection gate.
 
@@ -11,14 +10,19 @@ Modes:
 Exit codes: 0=clean, 1=findings, 2=usage error
 Output: one line per finding or summary (never prints full secrets)
 
-Pragma escape hatch (defensive only):
+Pragma escape hatch (STRICTLY SCOPED to doc-shaped rules):
   If the literal string 'secretscan: allow-pattern-docs' appears in a file's first 10 lines
-  (any comment syntax: #, //, <!-- -->), pattern-based findings in that file are reported
-  as ALLOWED-DOC and do NOT cause exit 1. Filename-based findings (credential files like
-  .credentials.json, *.pem, id_rsa*, *.p12) stay fatal regardless. Use only for deliberate
-  pattern documentation; the pragma appears in git diffs and is a reviewable act.
+  (any comment syntax: #, //, <!-- -->), findings from the two DOC-SHAPED rules ONLY
+  (generic_secret_assignment, env_access) are reported as ALLOWED-DOC and do not cause
+  exit 1. Fatal classes (PEM private keys, AWS/GitHub/Slack/OpenAI-Anthropic tokens,
+  connection strings) and filename-based findings stay fatal REGARDLESS of the pragma.
+  The pragma appears in git diffs and is a reviewable act.
 
-NOTE: Public repo version has NO vault allowlist. All secrets block unless permitted by pragma.
+Self-scan invariant: this file must scan CLEAN with NO pragma. Any pattern literal that
+would match its own regex is runtime-assembled from fragments (see pem_private_key) so
+the pattern text never appears contiguously in this source.
+
+NOTE: Public repo version has NO vault allowlist.
 """
 
 import argparse
@@ -30,8 +34,10 @@ from pathlib import Path
 
 
 # Regex patterns for secret detection (case-insensitive where sensible)
+# NOTE: pem_private_key is runtime-assembled from fragments so this source file
+# never contains a contiguous PEM-header shape (self-scan invariant, no pragma).
 PATTERNS = {
-    "pem_private_key": (r"-----BEGIN .* PRIVATE KEY-----", re.IGNORECASE),
+    "pem_private_key": (r"-----BEGIN .* " + "PRIVATE" + " KEY-----", re.IGNORECASE),
     "aws_access_key": (r"AKIA[0-9A-Z]{16}", 0),
     "aws_secret_pattern": (
         r"aws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*[^\s\$\<\{]",
@@ -86,7 +92,7 @@ def has_pragma(filepath):
 
 
 def is_binary_file(filepath):
-    """Check if file is binary (contains null bytes)."""
+    """Check if file is binary (contains null bytes in first 8KB)."""
     try:
         with open(filepath, "rb") as f:
             return b"\x00" in f.read(8192)
@@ -95,17 +101,25 @@ def is_binary_file(filepath):
 
 
 def should_skip_file(filepath):
-    """Check if file should be skipped (binary, too large, in .git/)."""
-    if ".git" in filepath.parts or ".git" in str(filepath):
+    """Check if file should be skipped entirely (.git/, node_modules/, __pycache__, .pyc, .pyo)."""
+    # Skip .git directories — match a path COMPONENT, not a substring
+    # (".git" in str(path) also matched ".gitignore" and skipped scanning it).
+    if ".git" in filepath.parts:
         return True
 
-    try:
-        stat = filepath.stat()
-        if stat.st_size > 1024 * 1024:  # >1MB
-            return True
-        if is_binary_file(filepath):
-            return True
-    except Exception:
+    # Skip node_modules — third-party deps, always git-ignored, never our code.
+    # CI installs them via `npm ci` for the dashboard build, so a whole-tree
+    # scan would otherwise walk thousands of package files (README example
+    # connection strings, files literally named token.js → false positives).
+    if "node_modules" in filepath.parts:
+        return True
+
+    # Skip __pycache__ directories
+    if "__pycache__" in filepath.parts:
+        return True
+
+    # Skip compiled Python artifacts
+    if filepath.name.endswith(".pyc") or filepath.name.endswith(".pyo"):
         return True
 
     return False
@@ -142,14 +156,36 @@ def scan_file(filepath):
     """
     Scan a single file for secrets.
     Returns list of (line_num, rule, match_str, is_fatal).
-    is_fatal=True for credential filenames; is_fatal=False if pragma present and rule-based.
+    is_fatal=True for credential filenames and fatal rule categories;
+    is_fatal=False only if pragma present AND rule is in SOFTENED_BY_PRAGMA.
+
+    Large files (>1MB) and binary files are scanned for FATAL_RULES patterns:
+    - Large files: scan first 1MB; emit SKIPPED-LARGE to stderr if file is larger
+    - Binary files: decode as latin-1; emit SKIPPED-BINARY to stderr if not fully scanned
     """
+    # Rules that CAN be softened by pragma (doc-shaped rules only)
+    SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
+
+    # Rules that are ALWAYS fatal, pragma never applies
+    FATAL_RULES = {
+        "pem_private_key",
+        "aws_access_key",
+        "github_token",
+        "slack_token",
+        "openai_anthropic_key",
+        "connection_string",
+    }
+
+    SIZE_THRESHOLD = 1024 * 1024  # 1MB
+    MAX_READ_SIZE = 2 * 1024 * 1024  # 2MB max to read
+    SCAN_PREFIX_SIZE = 1024 * 1024  # Scan first 1MB of large files
+
     findings = []
 
     if should_skip_file(filepath):
         return findings
 
-    # Check for pragma (applies only to rule-based findings, not filename findings)
+    # Check for pragma (applies only to specific rule-based findings, not filename findings)
     has_file_pragma = has_pragma(filepath)
 
     # Check if filename matches credential patterns (always fatal, pragma does NOT apply)
@@ -162,24 +198,95 @@ def scan_file(filepath):
             break
 
     try:
-        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
-            for line_num, line in enumerate(f, start=1):
-                for rule_name, (pattern, flags) in PATTERNS.items():
-                    # Skip env_assignment rule if not an .env-like file
-                    if rule_name == "env_assignment" and not is_env_file(filepath):
+        # Check file size and binary status
+        stat = filepath.stat()
+        file_size = stat.st_size
+        is_binary = is_binary_file(filepath)
+        is_large = file_size > SIZE_THRESHOLD
+
+        if is_large:
+            # Scan first 1MB of large file for FATAL_RULES
+            with open(filepath, "rb") as f:
+                content = f.read(SCAN_PREFIX_SIZE)
+
+            # Decode as latin-1 (handles any binary content)
+            try:
+                content_str = content.decode("latin-1")
+            except Exception:
+                content_str = content.decode("utf-8", errors="ignore")
+
+            # Emit skip notice to stderr
+            print(f"SKIPPED-LARGE {filepath} (scanned first {SCAN_PREFIX_SIZE // 1024}KB)", file=sys.stderr)
+
+            # Scan content for FATAL_RULES only
+            for line_num, line in enumerate(content_str.split("\n"), start=1):
+                for rule_name in FATAL_RULES:
+                    if rule_name not in PATTERNS:
                         continue
-
+                    pattern, flags = PATTERNS[rule_name]
+                    matches = re.finditer(pattern, line, flags)
+                    for match in matches:
+                        match_str = match.group(0)
+                        if is_placeholder(match_str):
+                            continue
+                        findings.append((line_num, rule_name, match_str, True))
+
+        elif is_binary:
+            # Scan binary file as latin-1 for FATAL_RULES only
+            with open(filepath, "rb") as f:
+                content = f.read(MAX_READ_SIZE)
+
+            # Decode as latin-1 (preserves all bytes)
+            try:
+                content_str = content.decode("latin-1")
+            except Exception:
+                content_str = content.decode("utf-8", errors="ignore")
+
+            # Emit skip notice to stderr
+            print(f"SKIPPED-BINARY {filepath} (scanned via latin-1)", file=sys.stderr)
+
+            # Scan content for FATAL_RULES only
+            for line_num, line in enumerate(content_str.split("\n"), start=1):
+                for rule_name in FATAL_RULES:
+                    if rule_name not in PATTERNS:
+                        continue
+                    pattern, flags = PATTERNS[rule_name]
                     matches = re.finditer(pattern, line, flags)
                     for match in matches:
                         match_str = match.group(0)
-
-                        # Skip if it's a placeholder
                         if is_placeholder(match_str):
                             continue
+                        findings.append((line_num, rule_name, match_str, True))
+
+        else:
+            # Normal text file: scan all rules
+            with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+                for line_num, line in enumerate(f, start=1):
+                    for rule_name, (pattern, flags) in PATTERNS.items():
+                        # Skip env_assignment rule if not an .env-like file
+                        if rule_name == "env_assignment" and not is_env_file(filepath):
+                            continue
 
-                        # Mark as non-fatal if pragma present (rule-based only)
-                        is_fatal = not has_file_pragma
-                        findings.append((line_num, rule_name, match_str, is_fatal))
+                        matches = re.finditer(pattern, line, flags)
+                        for match in matches:
+                            match_str = match.group(0)
+
+                            # Skip if it's a placeholder
+                            if is_placeholder(match_str):
+                                continue
+
+                            # Determine fatality based on rule category
+                            if rule_name in FATAL_RULES:
+                                # These are always fatal, pragma never applies
+                                is_fatal = True
+                            elif has_file_pragma and rule_name in SOFTENED_BY_PRAGMA:
+                                # Only these rules can be softened by pragma
+                                is_fatal = False
+                            else:
+                                # Other rules are fatal unless pragma and softened
+                                is_fatal = not has_file_pragma if rule_name in SOFTENED_BY_PRAGMA else True
+
+                            findings.append((line_num, rule_name, match_str, is_fatal))
 
     except Exception:
         pass