@matt82198/aesop 0.1.0 → 0.3.1

package/tools/secret_scan.py

@@ -8,7 +8,7 @@ Modes:
   secret_scan.py --history [--repo PATH]             Scan all blobs in git history
   secret_scan.py PATH [PATH...]                      Scan files/dirs directly (recurse dirs)
 
-Exit codes: 0=clean, 1=findings, 2=usage error
+Exit codes: 0=clean, 1=findings, 2=error (file unreadable/git failure/scan error)
 Output: one line per finding or summary (never prints full secrets)
 
 Pragma escape hatch (STRICTLY SCOPED to doc-shaped rules):
@@ -123,12 +123,41 @@ FATAL_RULES = {
     "connection_string",
 }
 
+# USER-APPROVED EXEMPTION (2026-07-22): redaction-pattern source files.
+# A regex whose PURPOSE is redacting connection strings necessarily contains a
+# connection-string-SHAPED pattern in its source text. connection_string
+# findings in these exact repo-relative paths are downgraded to non-fatal
+# ALLOWED-REDACTION-SOURCE (still reported, never silent). Every other rule
+# stays fully fatal in these files; connection_string stays fatal everywhere
+# else, including pragma'd files. Additions to this set are a security-gate
+# change and require explicit user approval.
+# Residual risk (accepted): matching is by trailing path components, so a
+# nested copy at */bench/sample_transcripts_judgment.py inherits the
+# exemption for this ONE rule; all other rules still apply there.
+REDACTION_SOURCE_FILES = {"bench/sample_transcripts_judgment.py"}
+
+
+def _is_redaction_source(scan_path):
+    """True when scan_path's trailing components match an exempted file."""
+    if not scan_path:
+        return False
+    parts = str(scan_path).replace("\\", "/").split("/")
+    for entry in REDACTION_SOURCE_FILES:
+        entry_parts = entry.split("/")
+        if parts[-len(entry_parts):] == entry_parts:
+            return True
+    return False
+
 
-def _classify_finding(rule_name, has_file_pragma):
+def _classify_finding(rule_name, has_file_pragma, scan_path=None):
     """Shared fatal/softened decision, used by every scan_* variant (disk-file,
     blob, large-file-chunked) so the pragma contract can't drift between them.
-    FATAL_RULES are always fatal. SOFTENED_BY_PRAGMA rules are fatal unless the
-    pragma is present. Everything else (e.g. env_assignment) is always fatal."""
+    FATAL_RULES are always fatal — except the single user-approved
+    connection_string downgrade for REDACTION_SOURCE_FILES (see above).
+    SOFTENED_BY_PRAGMA rules are fatal unless the pragma is present.
+    Everything else (e.g. env_assignment) is always fatal."""
+    if rule_name == "connection_string" and _is_redaction_source(scan_path):
+        return False
     if rule_name in FATAL_RULES:
         return True
     if rule_name in SOFTENED_BY_PRAGMA:
@@ -145,6 +174,12 @@ class GitScanError(Exception):
     printed CLEAN and exited 0 on an unresolvable range)."""
 
 
+class ScanError(Exception):
+    """Raised when a file cannot be scanned due to read/permission/OS errors.
+    Callers MUST treat this as fail-CLOSED (block / exit 2) -- a file the
+    scanner cannot read must block the push, not sail through as CLEAN."""
+
+
 def has_pragma(filepath):
     """Check if file has 'secretscan: allow-pattern-docs' pragma in first 10 lines."""
     try:
@@ -230,6 +265,9 @@ def scan_file(filepath):
     Large files (>1MB) and binary files are scanned for FATAL_RULES patterns:
     - Large files: scan first 1MB; emit SKIPPED-LARGE to stderr if file is larger
     - Binary files: decode as latin-1; emit SKIPPED-BINARY to stderr if not fully scanned
+
+    Raises ScanError if file cannot be read (permission denied, vanished, etc.)
+    to fail CLOSED rather than reporting the file clean.
     """
     SIZE_THRESHOLD = 1024 * 1024  # 1MB
     MAX_READ_SIZE = 2 * 1024 * 1024  # 2MB max to read
@@ -240,6 +278,7 @@ def scan_file(filepath):
         return findings
 
     # Check for pragma (applies only to specific rule-based findings, not filename findings)
+    # has_pragma() already swallows its own exceptions, so safe to call
     has_file_pragma = has_pragma(filepath)
 
     # Check if filename matches credential patterns (always fatal, pragma does NOT apply)
@@ -251,13 +290,21 @@ def scan_file(filepath):
             )
             break
 
+    # File stat and type detection — not wrapped in try/except so errors propagate
     try:
-        # Check file size and binary status
         stat = filepath.stat()
         file_size = stat.st_size
+    except (OSError, PermissionError, FileNotFoundError) as e:
+        raise ScanError(f"Cannot stat {filepath}: {e}")
+
+    try:
         is_binary = is_binary_file(filepath)
-        is_large = file_size > SIZE_THRESHOLD
+    except (OSError, PermissionError, FileNotFoundError) as e:
+        raise ScanError(f"Cannot check if binary {filepath}: {e}")
+
+    is_large = file_size > SIZE_THRESHOLD
 
+    try:
         if is_binary:
             # Binary file (any size): scan as-is for FATAL_RULES only
             with open(filepath, "rb") as f:
@@ -289,31 +336,26 @@ def scan_file(filepath):
             # Large text file: scan entire file in chunks for all rules
             print(f"SKIPPED-LARGE {filepath} (scanned in chunks)", file=sys.stderr)
             line_num = 0
-            try:
-                with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
-                    # Read in 1MB chunks to avoid loading entire large file into memory
-                    for chunk in iter(lambda: f.read(1024 * 1024), ""):
-                        for chunk_line in chunk.split("\n"):
-                            line_num += 1
-                            for rule_name, (pattern, flags) in PATTERNS.items():
-                                # Skip env_assignment rule if not an .env-like file
-                                if rule_name == "env_assignment" and not is_env_file(filepath):
-                                    continue
+            with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+                # Read in 1MB chunks to avoid loading entire large file into memory
+                for chunk in iter(lambda: f.read(1024 * 1024), ""):
+                    for chunk_line in chunk.split("\n"):
+                        line_num += 1
+                        for rule_name, (pattern, flags) in PATTERNS.items():
+                            # Skip env_assignment rule if not an .env-like file
+                            if rule_name == "env_assignment" and not is_env_file(filepath):
+                                continue
 
-                                matches = re.finditer(pattern, chunk_line, flags)
-                                for match in matches:
-                                    match_str = match.group(0)
+                            matches = re.finditer(pattern, chunk_line, flags)
+                            for match in matches:
+                                match_str = match.group(0)
 
-                                    # Skip if it's a placeholder
-                                    if is_placeholder(match_str):
-                                        continue
+                                # Skip if it's a placeholder
+                                if is_placeholder(match_str):
+                                    continue
 
-                                    is_fatal = _classify_finding(rule_name, has_file_pragma)
-                                    findings.append((line_num, rule_name, match_str, is_fatal))
-            except (IOError, OSError) as e:
-                # FAIL CLOSED: if we cannot fully scan a large text file, exit with error
-                print(f"FATAL: Cannot fully scan large text file {filepath}: {e}", file=sys.stderr)
-                sys.exit(1)
+                                is_fatal = _classify_finding(rule_name, has_file_pragma, filepath)
+                                findings.append((line_num, rule_name, match_str, is_fatal))
 
         else:
             # Normal small text file: scan all rules
@@ -332,11 +374,12 @@ def scan_file(filepath):
                             if is_placeholder(match_str):
                                 continue
 
-                            is_fatal = _classify_finding(rule_name, has_file_pragma)
+                            is_fatal = _classify_finding(rule_name, has_file_pragma, filepath)
                             findings.append((line_num, rule_name, match_str, is_fatal))
 
-    except Exception:
-        pass
+    except (OSError, PermissionError, FileNotFoundError, IOError) as e:
+        # FAIL CLOSED: if file cannot be read, raise error (not return clean)
+        raise ScanError(f"Cannot scan file {filepath}: {e}")
 
     return findings
 
@@ -424,7 +467,7 @@ def scan_blob(label, content_bytes):
                 match_str = match.group(0)
                 if is_placeholder(match_str):
                     continue
-                is_fatal = _classify_finding(rule_name, has_file_pragma)
+                is_fatal = _classify_finding(rule_name, has_file_pragma, label)
                 findings.append((line_num, rule_name, match_str, is_fatal))
 
     return findings
@@ -486,6 +529,7 @@ def get_staged_files(repo_path):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
             timeout=10,
         )
     except Exception as e:
@@ -513,6 +557,7 @@ def get_range_files(repo_path, commit_range):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
             timeout=10,
         )
     except Exception as e:
@@ -528,7 +573,11 @@ def get_range_files(repo_path, commit_range):
 
 
 def get_history_files(repo_path):
-    """Get all file contents from git history via git log -p."""
+    """Get all file contents from git history via git log -p.
+
+    Raises GitScanError if git log fails (corrupt repo, git crash, etc.).
+    This mirrors get_range_files() and get_staged_files() which already
+    fail closed on git errors."""
     files_content = []
     try:
         # Use git log -p to get full diff history
@@ -537,45 +586,108 @@ def get_history_files(repo_path):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
+            errors='replace',
             timeout=60,
         )
-        if result.returncode == 0:
-            # Parse the git log output: each file appears as +++ b/path/to/file followed by its content
-            current_file = None
-            current_content = []
-            for line in result.stdout.split("\n"):
-                if line.startswith("+++ b/"):
-                    if current_file and current_content:
-                        files_content.append((current_file, "\n".join(current_content)))
-                    current_file = line[6:]  # Remove "+++ b/"
-                    current_content = []
-                elif current_file and line.startswith("+") and not line.startswith("+++"):
-                    # Content line (added), strip the leading +
-                    current_content.append(line[1:])
-                elif current_file and line.startswith(" "):
-                    # Context line, keep it as-is (strip leading space)
-                    current_content.append(line[1:])
+    except Exception as e:
+        raise GitScanError(f"git log raised {e!r}")
+
+    if result.returncode != 0:
+        raise GitScanError(
+            f"git log failed (rc={result.returncode}): {result.stderr.strip()}"
+        )
+
+    # Parse the git log output: each file appears as +++ b/path/to/file followed by its content
+    current_file = None
+    current_content = []
+    for line in result.stdout.split("\n"):
+        if line.startswith("+++ b/"):
             if current_file and current_content:
                 files_content.append((current_file, "\n".join(current_content)))
-    except Exception:
-        pass
+            current_file = line[6:]  # Remove "+++ b/"
+            current_content = []
+        elif current_file and line.startswith("+") and not line.startswith("+++"):
+            # Content line (added), strip the leading +
+            current_content.append(line[1:])
+        elif current_file and line.startswith(" "):
+            # Context line, keep it as-is (strip leading space)
+            current_content.append(line[1:])
+    if current_file and current_content:
+        files_content.append((current_file, "\n".join(current_content)))
+
     return files_content
 
 
+def _git_committable_set(dir_path):
+    """Return the set of resolved absolute paths that git tracks or would
+    consider committable (tracked + untracked-but-not-ignored) for the repo
+    containing `dir_path`, or None if dir_path is not inside a git work tree
+    or git is unavailable/errors.
+
+    Built from `git ls-files --cached --others --exclude-standard`, which lists
+    exactly the tracked files plus untracked files that are NOT git-ignored.
+    A tracked file is ALWAYS included (even a force-added one that matches an
+    ignore rule), so filtering by this set never skips a committable file --
+    only ephemeral, git-ignored runtime files (which can never be pushed and
+    therefore can't leak a secret through git) are excluded.
+    """
+    try:
+        top = subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            cwd=str(dir_path), capture_output=True, text=True, encoding='utf-8', timeout=10,
+        )
+        if top.returncode != 0:
+            return None
+        toplevel = Path(top.stdout.strip())
+        listing = subprocess.run(
+            ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+            cwd=str(toplevel), capture_output=True, text=True, encoding='utf-8', timeout=60,
+        )
+        if listing.returncode != 0:
+            return None
+    except Exception:
+        return None
+
+    committable = set()
+    for rel in listing.stdout.split("\0"):
+        if rel:
+            committable.add((toplevel / rel).resolve())
+    return committable
+
+
 def scan_paths(paths):
-    """Recursively scan paths (files and directories)."""
+    """Recursively scan paths (files and directories).
+
+    For a DIRECTORY argument inside a git work tree, git-ignored files are
+    skipped: a full-tree scan (e.g. CI's `secret_scan.py .`) must not flag
+    ephemeral, git-ignored runtime files -- such as state/.ui-session-token --
+    that can never be committed or pushed and therefore cannot leak a secret
+    through git. Filtering uses the tracked+untracked-not-ignored set from
+    `git ls-files`, so a committable file is never skipped. When a directory is
+    not inside a git repo (or git errors), NO filtering is applied and every
+    file is walked (fail OPEN toward more scanning, never less). Explicitly
+    named FILE arguments are always scanned, even if git-ignored.
+    """
     files_to_scan = []
     for path_str in paths:
         path = Path(path_str).resolve()
         if path.is_file():
             files_to_scan.append(path)
         elif path.is_dir():
-            files_to_scan.extend(path.rglob("*"))
+            candidates = [p for p in path.rglob("*") if p.is_file()]
+            committable = _git_committable_set(path)
+            if committable is not None:
+                candidates = [p for p in candidates if p.resolve() in committable]
+            files_to_scan.extend(candidates)
     return [p for p in files_to_scan if p.is_file()]
 
 
-def scan_content(content):
-    """Scan raw content for secrets (used by history scanning)."""
+def scan_content(content, scan_path=None):
+    """Scan raw content for secrets (used by history scanning). scan_path
+    (the blob's repo-relative path, when known) feeds the REDACTION_SOURCE_FILES
+    exemption so committed redaction-regex source does not turn every future
+    --history / prepublish scan permanently red."""
     findings = []
     for line_num, line in enumerate(content.split("\n"), start=1):
         for rule_name, (pattern, flags) in PATTERNS.items():
@@ -584,7 +696,8 @@ def scan_content(content):
                 match_str = match.group(0)
                 if is_placeholder(match_str):
                     continue
-                findings.append((line_num, rule_name, match_str, True))
+                is_fatal = _classify_finding(rule_name, False, scan_path)
+                findings.append((line_num, rule_name, match_str, is_fatal))
     return findings
 
 
@@ -627,6 +740,7 @@ def main():
     all_findings = []
     fatal_findings = []
     allowed_doc_count = 0
+    allowed_redaction_count = 0
     file_count = 0
 
     if args.staged:
@@ -655,7 +769,10 @@ def main():
                 if is_fatal:
                     fatal_findings.append((label, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     elif args.range:
         # Scan the COMMITTED blob at the TIP of the range for each changed
@@ -685,15 +802,24 @@ def main():
                 if is_fatal:
                     fatal_findings.append((label, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     elif args.history:
         # History scanning mode: unaffected by the blob-scan fix above, since
         # it already walks committed diff content via `git log -p`.
-        history_files = get_history_files(args.repo)
+        try:
+            history_files = get_history_files(args.repo)
+        except GitScanError as e:
+            print(f"FATAL: could not scan git history: {e}", file=sys.stderr)
+            print("Failing CLOSED: refusing to report CLEAN when git history cannot be read.", file=sys.stderr)
+            sys.exit(2)
+
         file_count = len(set(f for f, _ in history_files))
         for filepath, content in history_files:
-            findings = scan_content(content)
+            findings = scan_content(content, scan_path=filepath)
             for line_num, rule, match_str, is_fatal in findings:
                 all_findings.append((filepath, line_num, rule, match_str, is_fatal))
                 if is_fatal:
@@ -703,13 +829,21 @@ def main():
         files = scan_paths(args.paths)
         file_count = len(files)
         for filepath in files:
-            findings = scan_file(filepath)
+            try:
+                findings = scan_file(filepath)
+            except ScanError as e:
+                print(f"FATAL: cannot scan file {filepath}: {e}", file=sys.stderr)
+                print("Failing CLOSED: refusing to report CLEAN when a file cannot be read.", file=sys.stderr)
+                sys.exit(2)
             for line_num, rule, match_str, is_fatal in findings:
                 all_findings.append((filepath, line_num, rule, match_str, is_fatal))
                 if is_fatal:
                     fatal_findings.append((filepath, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     # Output findings
     for filepath, line_num, rule, match_str, is_fatal in all_findings:
@@ -717,18 +851,26 @@ def main():
         if is_fatal:
             print(f"HIGH {filepath}:{line_num} {rule} ({masked})")
         else:
-            print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
+            if rule == "connection_string":
+                print(f"ALLOWED-REDACTION-SOURCE {filepath}:{line_num} {rule} ({masked})")
+            else:
+                print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
 
     # Summary and exit
     if len(fatal_findings) == 0:
-        if allowed_doc_count == 0:
+        if allowed_doc_count == 0 and allowed_redaction_count == 0:
             if args.history:
                 print(f"CLEAN: git history scanned")
             else:
                 print(f"CLEAN: {file_count} files scanned")
         else:
             pragma_file_count = len(set(f for f, _, _, _, is_fatal in all_findings if not is_fatal))
-            print(f"CLEAN: scanned ({allowed_doc_count} allowed-doc findings in {pragma_file_count} pragma files)")
+            parts = []
+            if allowed_doc_count:
+                parts.append(f"{allowed_doc_count} allowed-doc findings in {pragma_file_count} pragma files")
+            if allowed_redaction_count:
+                parts.append(f"{allowed_redaction_count} allowed-redaction-source findings")
+            print(f"CLEAN: scanned ({'; '.join(parts)})")
         sys.exit(0)
     else:
         print(f"FOUND: {len(fatal_findings)} secret(s)")

package/tools/self_stats.py

@@ -502,6 +502,26 @@ class StatsCounter:
         data = json.loads(self.json())
         data["generated_at"] = datetime.now(timezone.utc).isoformat()
         data["loc"] = self.git.lines_of_code
+
+        # Add cost economics metrics (requires cost_econ module)
+        try:
+            from cost_econ import calculate_economics, get_metric_honesty_caveats
+            # Infer state directory from repo root or config
+            repo_path = Path(self.git.repo_root)
+            state_dir = repo_path / "state"
+
+            economics = calculate_economics(
+                repo_root=str(repo_path),
+                state_dir=str(state_dir) if state_dir.exists() else None,
+                config_file=None
+            )
+
+            data["economics"] = economics
+            data["economics_caveats"] = get_metric_honesty_caveats()
+        except Exception:
+            # Graceful fallback: if cost_econ unavailable, skip economics metrics
+            pass
+
         return data
 
     def save_stats(self, output_file: str = "stats.json") -> None: