@rulemetric/proxy 0.2.2

package/addon/secret_redactor.py

@@ -0,0 +1,130 @@
+"""Redact secrets from proxy snapshot content before storage.
+
+Scans text fields for common secret patterns (API keys, tokens, passwords,
+connection strings) and replaces them with redaction placeholders. This
+prevents credentials accidentally included in LLM conversations from being
+persisted in the RuleMetric database.
+
+Runs in-process with zero external dependencies — pure regex, no ML models.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Secret patterns — ordered by specificity (most specific first)
+# ---------------------------------------------------------------------------
+
+# Each tuple: (name, compiled regex, group index to redact)
+# The regex should match the full secret value including any prefix.
+
+_SECRET_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    # API keys with known prefixes
+    ("anthropic_api_key", re.compile(r"sk-ant-[a-zA-Z0-9_-]{20,}")),
+    ("openai_api_key", re.compile(r"sk-[a-zA-Z0-9]{20,}")),
+    ("github_token", re.compile(r"gh[ps]_[a-zA-Z0-9]{36,}")),
+    ("github_fine_grained", re.compile(r"github_pat_[a-zA-Z0-9_]{20,}")),
+    ("aws_access_key", re.compile(r"AKIA[0-9A-Z]{16}")),
+    ("aws_secret_key", re.compile(r"(?:aws_secret_access_key|AWS_SECRET_ACCESS_KEY)\s*[=:]\s*['\"]?([a-zA-Z0-9/+=]{40})['\"]?")),
+    ("slack_token", re.compile(r"xox[baprs]-[a-zA-Z0-9-]+")),
+    ("stripe_key", re.compile(r"[sr]k_(live|test)_[a-zA-Z0-9]{20,}")),
+    ("npm_token", re.compile(r"npm_[a-zA-Z0-9]{36,}")),
+    ("supabase_key", re.compile(r"eyJ[a-zA-Z0-9_-]{100,}\.[a-zA-Z0-9_-]{100,}\.[a-zA-Z0-9_-]{40,}")),
+
+    # Connection strings with embedded credentials
+    ("postgres_url", re.compile(r"postgres(?:ql)?://[^:]+:[^@\s]+@[^\s\"']+")),
+    ("mysql_url", re.compile(r"mysql://[^:]+:[^@\s]+@[^\s\"']+")),
+    ("mongodb_url", re.compile(r"mongodb(?:\+srv)?://[^:]+:[^@\s]+@[^\s\"']+")),
+    ("redis_url", re.compile(r"redis://:[^@\s]+@[^\s\"']+")),
+
+    # Generic key=value patterns for known env var names
+    ("env_secret", re.compile(
+        r"(?:DATABASE_URL|JWT_SECRET|SESSION_SECRET|ENCRYPTION_KEY|"
+        r"SUPABASE_SERVICE_ROLE_KEY|PRIVATE_KEY|CLIENT_SECRET)"
+        r"\s*[=:]\s*['\"]?([^\s'\"]{8,})['\"]?"
+    )),
+
+    # Bearer tokens in inline examples
+    ("bearer_token", re.compile(r"Bearer\s+[a-zA-Z0-9_.-]{40,}")),
+
+    # Private keys (PEM format)
+    ("private_key_pem", re.compile(
+        r"-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----[\s\S]{20,}?-----END (?:RSA |EC |DSA )?PRIVATE KEY-----"
+    )),
+]
+
+_REDACTED = "[REDACTED]"
+
+
+def redact_text(text: str) -> tuple[str, list[dict[str, str]]]:
+    """Redact secrets from a text string.
+
+    Returns:
+        (redacted_text, findings) where findings is a list of
+        {"pattern": name, "sample": first_4_chars} for each match.
+    """
+    if not text:
+        return text, []
+
+    findings: list[dict[str, str]] = []
+
+    for name, pattern in _SECRET_PATTERNS:
+        matches = list(pattern.finditer(text))
+        for match in reversed(matches):  # reverse to preserve indices
+            matched = match.group(0)
+            # Keep a tiny sample for debugging (first 4 chars + "...")
+            sample = matched[:4] + "..." if len(matched) > 4 else matched
+            findings.append({"pattern": name, "sample": sample})
+            text = text[:match.start()] + _REDACTED + text[match.end():]
+
+    return text, findings
+
+
+def redact_snapshot(snapshot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, str]]]:
+    """Redact secrets from all text fields in a proxy snapshot.
+
+    Mutates the snapshot in place and returns (snapshot, all_findings).
+    """
+    all_findings: list[dict[str, str]] = []
+
+    # Redact system_prompt
+    if snapshot.get("system_prompt"):
+        snapshot["system_prompt"], findings = redact_text(snapshot["system_prompt"])
+        all_findings.extend(findings)
+
+    # Redact messages (simplified content strings)
+    for msg in snapshot.get("messages", []):
+        if isinstance(msg, dict) and msg.get("content"):
+            msg["content"], findings = redact_text(msg["content"])
+            all_findings.extend(findings)
+
+    # Redact messages_raw (full API message objects with nested content blocks)
+    for msg in snapshot.get("messages_raw", []):
+        if isinstance(msg, dict):
+            content = msg.get("content")
+            if isinstance(content, str):
+                msg["content"], findings = redact_text(content)
+                all_findings.extend(findings)
+            elif isinstance(content, list):
+                for block in content:
+                    if isinstance(block, dict) and isinstance(block.get("text"), str):
+                        block["text"], findings = redact_text(block["text"])
+                        all_findings.extend(findings)
+
+    # Redact system_blocks
+    for block in snapshot.get("system_blocks", []):
+        if isinstance(block, dict) and isinstance(block.get("text"), str):
+            block["text"], findings = redact_text(block["text"])
+            all_findings.extend(findings)
+
+    # Redact response content if present
+    resp = snapshot.get("response", {})
+    if isinstance(resp, dict):
+        for key in ("text", "content"):
+            if isinstance(resp.get(key), str):
+                resp[key], findings = redact_text(resp[key])
+                all_findings.extend(findings)
+
+    return snapshot, all_findings