@masyv/secretscan 0.1.0

package/core/src/patterns/builtin.rs

@@ -0,0 +1,366 @@
+//! 50+ built-in secret patterns covering all major providers and formats.
+
+use super::Pattern;
+use crate::Severity;
+use regex::Regex;
+
+macro_rules! pat {
+    ($id:expr, $name:expr, $sev:expr, $re:expr) => {
+        Pattern {
+            id: $id,
+            name: $name,
+            severity: $sev,
+            regex: Regex::new($re).expect(concat!("invalid regex for ", $id)),
+            min_entropy: None,
+            context_keywords: &[],
+        }
+    };
+    ($id:expr, $name:expr, $sev:expr, $re:expr, entropy: $e:expr) => {
+        Pattern {
+            id: $id,
+            name: $name,
+            severity: $sev,
+            regex: Regex::new($re).expect(concat!("invalid regex for ", $id)),
+            min_entropy: Some($e),
+            context_keywords: &[],
+        }
+    };
+}
+
+pub fn all_patterns() -> Vec<Pattern> {
+    vec![
+        // ── Anthropic ────────────────────────────────────────────────────────
+        pat!(
+            "anthropic_api_key",
+            "Anthropic API Key",
+            Severity::Critical,
+            r"sk-ant-[a-zA-Z0-9\-_]{80,}"
+        ),
+
+        // ── OpenAI ───────────────────────────────────────────────────────────
+        pat!(
+            "openai_api_key",
+            "OpenAI API Key",
+            Severity::Critical,
+            r"sk-[a-zA-Z0-9]{48}"
+        ),
+        pat!(
+            "openai_org",
+            "OpenAI Organization ID",
+            Severity::Medium,
+            r"org-[a-zA-Z0-9]{24}"
+        ),
+
+        // ── AWS ───────────────────────���────────────────────────────���─────────
+        pat!(
+            "aws_access_key",
+            "AWS Access Key ID",
+            Severity::Critical,
+            r"(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"
+        ),
+        pat!(
+            "aws_secret_key",
+            "AWS Secret Access Key",
+            Severity::Critical,
+            r#"(?i)aws[_\-\s]?secret[_\-\s]?(?:access[_\-\s]?)?key["':\s=]+([A-Za-z0-9/+=]{40})"#,
+            entropy: 4.0
+        ),
+        pat!(
+            "aws_session_token",
+            "AWS Session Token",
+            Severity::High,
+            r#"(?i)aws[_\-\s]?session[_\-\s]?token["':\s=]+([A-Za-z0-9/+=]{100,})"#
+        ),
+
+        // ── GitHub ───────────────────────────────────────────────────────────
+        pat!(
+            "github_pat",
+            "GitHub Personal Access Token",
+            Severity::Critical,
+            r"gh[pousr]_[A-Za-z0-9_]{36,255}"
+        ),
+        pat!(
+            "github_oauth",
+            "GitHub OAuth Token",
+            Severity::High,
+            r"gho_[A-Za-z0-9_]{36}"
+        ),
+        pat!(
+            "github_app_token",
+            "GitHub App Token",
+            Severity::High,
+            r"(?:ghu|ghs)_[A-Za-z0-9_]{36}"
+        ),
+
+        // ── GitLab ───────────────────────────────────────────���───────────────
+        pat!(
+            "gitlab_pat",
+            "GitLab Personal Access Token",
+            Severity::High,
+            r"glpat-[a-zA-Z0-9\-_]{20}"
+        ),
+        pat!(
+            "gitlab_pipeline",
+            "GitLab Pipeline Trigger Token",
+            Severity::Medium,
+            r"glptt-[a-f0-9]{40}"
+        ),
+
+        // ── Stripe ────────────────────────��─────────────────────────────��────
+        pat!(
+            "stripe_secret",
+            "Stripe Secret Key",
+            Severity::Critical,
+            r"sk_live_[a-zA-Z0-9]{24,}"
+        ),
+        pat!(
+            "stripe_restricted",
+            "Stripe Restricted Key",
+            Severity::High,
+            r"rk_live_[a-zA-Z0-9]{24,}"
+        ),
+        pat!(
+            "stripe_test",
+            "Stripe Test Key",
+            Severity::Low,
+            r"sk_test_[a-zA-Z0-9]{24,}"
+        ),
+
+        // ── Google ─────────────────────────���──────────────────────────���──────
+        pat!(
+            "google_api_key",
+            "Google API Key",
+            Severity::High,
+            r"AIza[0-9A-Za-z\-_]{35}"
+        ),
+        pat!(
+            "google_oauth",
+            "Google OAuth Client Secret",
+            Severity::High,
+            r"GOCSPX-[a-zA-Z0-9\-_]{28}"
+        ),
+        pat!(
+            "google_service_account",
+            "Google Service Account Key",
+            Severity::Critical,
+            r#""type"\s*:\s*"service_account""#
+        ),
+
+        // ── Slack ────────────────────────────────────────────────────────���───
+        pat!(
+            "slack_bot_token",
+            "Slack Bot Token",
+            Severity::High,
+            r"xoxb-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24}"
+        ),
+        pat!(
+            "slack_user_token",
+            "Slack User Token",
+            Severity::High,
+            r"xoxp-[0-9]{10,13}-[0-9]{10,13}-[0-9]{10,13}-[a-f0-9]{32}"
+        ),
+        pat!(
+            "slack_webhook",
+            "Slack Incoming Webhook",
+            Severity::Medium,
+            r"https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[a-zA-Z0-9]+"
+        ),
+        pat!(
+            "slack_app_token",
+            "Slack App Token",
+            Severity::High,
+            r"xapp-\d-[A-Z0-9]+-\d+-[a-f0-9]+"
+        ),
+
+        // ── npm ────────────────────────────���────────────────────────────��────
+        pat!(
+            "npm_token",
+            "npm Access Token",
+            Severity::High,
+            r"npm_[a-zA-Z0-9]{36}"
+        ),
+
+        // ── SendGrid ────────────────────────────��────────────────────────────
+        pat!(
+            "sendgrid_api_key",
+            "SendGrid API Key",
+            Severity::High,
+            r"SG\.[a-zA-Z0-9\-_]{22}\.[a-zA-Z0-9\-_]{43}"
+        ),
+
+        // ── Twilio ───────────────────────────────────────────────────────────
+        pat!(
+            "twilio_account_sid",
+            "Twilio Account SID",
+            Severity::Medium,
+            r"AC[a-f0-9]{32}"
+        ),
+        pat!(
+            "twilio_auth_token",
+            "Twilio Auth Token",
+            Severity::High,
+            r"((?i)twilio[_\-\s]?auth[_\-\s]?token[':\s=]+([a-f0-9]{32}))"
+        ),
+
+        // ── Cloudflare ───────────────────────────────────────────────────────
+        pat!(
+            "cloudflare_api_key",
+            "Cloudflare API Key",
+            Severity::High,
+            r"((?i)cloudflare[_\-\s]?(?:api[_\-\s]?)?(?:key|token)[':\s=]+([a-zA-Z0-9_\-]{37,40}))"
+        ),
+        pat!(
+            "cloudflare_global_key",
+            "Cloudflare Global API Key",
+            Severity::Critical,
+            r"[0-9a-f]{37}"
+        ),
+
+        // ── Azure ────────────────────────────────────────────────────────────
+        pat!(
+            "azure_connection_string",
+            "Azure Storage Connection String",
+            Severity::Critical,
+            r"DefaultEndpointsProtocol=https;AccountName=[^;]+;AccountKey=[^;]+"
+        ),
+        pat!(
+            "azure_sas_token",
+            "Azure SAS Token",
+            Severity::High,
+            r"sv=\d{4}-\d{2}-\d{2}&s[a-z]=.*?&sig=[A-Za-z0-9%+/=]+"
+        ),
+
+        // ── Database URLs ─────────────────────────────────────────────────────
+        pat!(
+            "postgres_url",
+            "PostgreSQL Connection URL with credentials",
+            Severity::Critical,
+            r"postgres(?:ql)?://[^:]+:[^@\s]{3,}@[^\s]+"
+        ),
+        pat!(
+            "mysql_url",
+            "MySQL Connection URL with credentials",
+            Severity::Critical,
+            r"mysql://[^:]+:[^@\s]{3,}@[^\s]+"
+        ),
+        pat!(
+            "mongodb_url",
+            "MongoDB Connection URL with credentials",
+            Severity::Critical,
+            r"mongodb(?:\+srv)?://[^:]+:[^@\s]{3,}@[^\s]+"
+        ),
+        pat!(
+            "redis_url",
+            "Redis URL with password",
+            Severity::High,
+            r"redis://[^:]*:[^@\s]{3,}@[^\s]+"
+        ),
+
+        // ── Private Keys ──────────────────────────────────────────────────────
+        pat!(
+            "private_key_pem",
+            "PEM Private Key",
+            Severity::Critical,
+            r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY(?:\s+BLOCK)?-----"
+        ),
+        pat!(
+            "certificate",
+            "PEM Certificate",
+            Severity::Low,
+            r"-----BEGIN CERTIFICATE-----"
+        ),
+
+        // ── JWT ─────────────────────────���─────────────────────────────��───────
+        pat!(
+            "jwt_token",
+            "JSON Web Token",
+            Severity::Medium,
+            r"ey[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+"
+        ),
+
+        // ── Heroku ───────────────────────────────────────────────────────────��
+        pat!(
+            "heroku_api_key",
+            "Heroku API Key",
+            Severity::High,
+            r"((?i)heroku[_\-\s]?(?:api[_\-\s]?)?key[':\s=]+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))"
+        ),
+
+        // ── Vercel ──────────────────────────────────���────────────────────���────
+        pat!(
+            "vercel_token",
+            "Vercel Access Token",
+            Severity::High,
+            r"((?i)vercel[_\-\s]?token[':\s=]+([a-zA-Z0-9]{24}))"
+        ),
+
+        // ── Datadog ────────────────────────────────���────────────────────────��─
+        pat!(
+            "datadog_api_key",
+            "Datadog API Key",
+            Severity::High,
+            r"((?i)(?:datadog|dd)[_\-\s]?(?:api[_\-\s]?)?key[':\s=]+([a-f0-9]{32}))"
+        ),
+
+        // ── Mailgun ───────────────────────────────────────────────────────────
+        pat!(
+            "mailgun_api_key",
+            "Mailgun API Key",
+            Severity::High,
+            r"key-[a-f0-9]{32}"
+        ),
+
+        // ── Shopify ─────────────────────��─────────────────────────���───────────
+        pat!(
+            "shopify_token",
+            "Shopify Access Token",
+            Severity::High,
+            r"shpat_[a-fA-F0-9]{32}"
+        ),
+        pat!(
+            "shopify_shared_secret",
+            "Shopify Shared Secret",
+            Severity::High,
+            r"shpss_[a-fA-F0-9]{32}"
+        ),
+
+        // ── Discord ───────────────────────────────────────────────��───────────
+        pat!(
+            "discord_bot_token",
+            "Discord Bot Token",
+            Severity::High,
+            r"[MN][a-zA-Z0-9\-_]{23,25}\.[a-zA-Z0-9\-_]{6}\.[a-zA-Z0-9\-_]{27,38}"
+        ),
+        pat!(
+            "discord_webhook",
+            "Discord Webhook URL",
+            Severity::Medium,
+            r"https://discord(?:app)?\.com/api/webhooks/[0-9]+/[a-zA-Z0-9_\-]+"
+        ),
+
+        // ── HuggingFace ───────────────────────────────────────���───────────────
+        pat!(
+            "huggingface_token",
+            "HuggingFace Access Token",
+            Severity::High,
+            r"hf_[a-zA-Z0-9]{34,}"
+        ),
+
+        // ── Env-file patterns ─────────────────────────────────────────────────
+        pat!(
+            "env_secret",
+            "Environment Variable with Secret",
+            Severity::Medium,
+            r#"(?i)(?:SECRET|PASSWORD|PASSWD|PWD|API_KEY|AUTH_TOKEN|PRIVATE_KEY|ACCESS_TOKEN)\s*=\s*["']?([A-Za-z0-9+/=_\-!@#$%^&*]{8,})["']?"#,
+            entropy: 3.5
+        ),
+
+        // ── Generic credential patterns ───────────────────────────��───────────
+        pat!(
+            "basic_auth",
+            "HTTP Basic Auth Credentials",
+            Severity::High,
+            r"https?://[A-Za-z0-9_\-\.]+:[A-Za-z0-9_\-\.!@#$%^&*]{4,}@"
+        ),
+    ]
+}

package/core/src/patterns/entropy.rs

@@ -0,0 +1,129 @@
+//! Shannon entropy analysis for detecting high-entropy strings that look like secrets.
+
+use crate::{Finding, Severity};
+use regex::Regex;
+use once_cell::sync::Lazy;
+
+/// Minimum entropy (bits per character) to flag as suspicious.
+const HIGH_ENTROPY_THRESHOLD: f64 = 4.5;
+/// Minimum length for entropy analysis.
+const MIN_SECRET_LENGTH: usize = 20;
+/// Maximum length (avoid flagging long base64 payloads like JWT bodies).
+const MAX_SECRET_LENGTH: usize = 200;
+
+/// Compute Shannon entropy of a string in bits per character.
+pub fn shannon(s: &str) -> f64 {
+    if s.is_empty() {
+        return 0.0;
+    }
+    let len = s.len() as f64;
+    let mut freq = [0u32; 256];
+    for b in s.bytes() {
+        freq[b as usize] += 1;
+    }
+    freq.iter()
+        .filter(|&&c| c > 0)
+        .map(|&c| {
+            let p = c as f64 / len;
+            -p * p.log2()
+        })
+        .sum()
+}
+
+/// Regex for candidate high-entropy tokens: alphanumeric + common secret chars.
+static CANDIDATE_RE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[A-Za-z0-9+/=_\-]{20,200}").unwrap()
+});
+
+/// Context keywords that, if nearby, raise the flag from INFO to MEDIUM.
+static CONTEXT_KW_RE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?i)(secret|password|passwd|pwd|token|key|api[_\-]?key|credential|auth|private|bearer|access[_\-]?key)")
+        .unwrap()
+});
+
+/// Scan for generic high-entropy strings.
+pub fn scan_high_entropy(text: &str) -> Vec<Finding> {
+    let mut findings = Vec::new();
+
+    for m in CANDIDATE_RE.find_iter(text) {
+        let s = m.as_str();
+        if s.len() < MIN_SECRET_LENGTH || s.len() > MAX_SECRET_LENGTH {
+            continue;
+        }
+
+        let e = shannon(s);
+        if e < HIGH_ENTROPY_THRESHOLD {
+            continue;
+        }
+
+        // Skip if it looks like a base64-encoded UUID or hash (common false positive)
+        if looks_like_hash(s) {
+            continue;
+        }
+
+        // Determine severity by context
+        let context_start = m.start().saturating_sub(80);
+        let context_end = (m.end() + 80).min(text.len());
+        let context = &text[context_start..context_end];
+        let severity = if CONTEXT_KW_RE.is_match(context) {
+            Severity::Medium
+        } else {
+            Severity::Low
+        };
+
+        let hash = blake3::hash(s.as_bytes());
+        let fingerprint = format!("{:.8}", hash.to_hex());
+
+        findings.push(Finding {
+            pattern_id: "high_entropy",
+            pattern_name: "High-entropy string",
+            severity,
+            matched: s.to_string(),
+            redacted: format!("[REDACTED:high_entropy:{}]", fingerprint),
+            fingerprint,
+            offset: m.start(),
+            length: s.len(),
+        });
+    }
+
+    findings
+}
+
+/// Heuristic: looks like a hex hash (md5/sha256/blake3) — lower false-positive risk.
+fn looks_like_hash(s: &str) -> bool {
+    let hex_chars = s.chars().filter(|c| c.is_ascii_hexdigit()).count();
+    let ratio = hex_chars as f64 / s.len() as f64;
+    // If >90% hex digits and length is 32, 40, 56, or 64 — it's probably a hash
+    ratio > 0.90 && matches!(s.len(), 32 | 40 | 56 | 64)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn entropy_of_repeated_char() {
+        assert!(shannon("aaaaaaaaaa") < 1.0);
+    }
+
+    #[test]
+    fn entropy_of_random_string() {
+        // A real API key-like string should have high entropy
+        let s = "xK9mP2nQ8rL5vT1wJ4hB7cF0dA3sE6uI";
+        assert!(shannon(s) > 4.0);
+    }
+
+    #[test]
+    fn hash_not_flagged() {
+        // SHA256-like hex string should not be flagged
+        let s = "a3f5c2d1e4b6a7f8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2";
+        assert!(looks_like_hash(s));
+    }
+
+    #[test]
+    fn high_entropy_detected() {
+        let text = "api_key = xK9mP2nQ8rL5vT1wJ4hB7cF0dA3sE6uI";
+        let findings = scan_high_entropy(text);
+        assert!(!findings.is_empty());
+    }
+}

package/core/src/patterns/mod.rs

@@ -0,0 +1,83 @@
+pub mod builtin;
+pub mod entropy;
+
+use crate::{Finding, Severity};
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+/// A compiled secret pattern.
+pub struct Pattern {
+    pub id: &'static str,
+    pub name: &'static str,
+    pub severity: Severity,
+    pub regex: Regex,
+    /// Minimum entropy threshold (bits/char). None = skip entropy check.
+    pub min_entropy: Option<f64>,
+    /// Context keywords that raise confidence (e.g. "password", "secret")
+    pub context_keywords: &'static [&'static str],
+}
+
+impl Pattern {
+    pub fn scan(&self, text: &str) -> Vec<Finding> {
+        let mut findings = Vec::new();
+        for m in self.regex.find_iter(text) {
+            let matched = m.as_str();
+
+            // Entropy gate
+            if let Some(min_e) = self.min_entropy {
+                if entropy::shannon(matched) < min_e {
+                    continue;
+                }
+            }
+
+            // Build short fingerprint (last 8 hex chars of blake3)
+            let hash = blake3::hash(matched.as_bytes());
+            let fingerprint = format!("{:.8}", hash.to_hex());
+
+            findings.push(Finding {
+                pattern_id: self.id,
+                pattern_name: self.name,
+                severity: self.severity,
+                matched: matched.to_string(),
+                redacted: format!("[REDACTED:{}:{}]", self.id, fingerprint),
+                fingerprint,
+                offset: m.start(),
+                length: matched.len(),
+            });
+        }
+        findings
+    }
+}
+
+/// All compiled built-in patterns, initialised once.
+pub static PATTERNS: Lazy<Vec<Pattern>> = Lazy::new(builtin::all_patterns);
+
+/// Scan text against all registered patterns.
+pub fn scan_all(text: &str) -> Vec<Finding> {
+    let mut findings: Vec<Finding> = PATTERNS.iter().flat_map(|p| p.scan(text)).collect();
+
+    // Add entropy-based generic detection
+    findings.extend(entropy::scan_high_entropy(text));
+
+    // Sort by offset, deduplicate overlapping findings (keep highest severity)
+    findings.sort_by_key(|f| f.offset);
+    dedup_overlapping(findings)
+}
+
+fn dedup_overlapping(findings: Vec<Finding>) -> Vec<Finding> {
+    let mut result: Vec<Finding> = Vec::new();
+    for f in findings {
+        if let Some(last) = result.last_mut() {
+            let last_end = last.offset + last.length;
+            if f.offset < last_end {
+                // Overlapping — keep the higher severity one
+                if f.severity > last.severity {
+                    *last = f;
+                }
+                continue;
+            }
+        }
+        result.push(f);
+    }
+    result
+}

package/core/src/redact/mod.rs

@@ -0,0 +1,69 @@
+//! Redaction engine — replaces detected secrets in text with safe placeholders.
+
+use crate::{Finding, ScanResult};
+
+/// Apply all findings to the text, replacing each matched secret with its
+/// redacted placeholder. Edits are applied in reverse order so byte offsets
+/// stay valid.
+pub fn apply_redactions(text: &str, findings: &[Finding]) -> String {
+    let mut result = text.to_string();
+    // Sort descending by offset so we don't invalidate later offsets
+    let mut sorted = findings.to_vec();
+    sorted.sort_by(|a, b| b.offset.cmp(&a.offset));
+
+    for f in &sorted {
+        let end = f.offset + f.length;
+        if end <= result.len() {
+            result.replace_range(f.offset..end, &f.redacted);
+        }
+    }
+    result
+}
+
+/// Full scan + redact pipeline. Returns the redacted text and scan metadata.
+pub fn scan_and_redact(
+    text: &str,
+    tool_name: Option<&str>,
+    db_path: &std::path::Path,
+    session_id: &str,
+) -> ScanResult {
+    use crate::patterns;
+    use crate::store::Store;
+
+    let findings = patterns::scan_all(text);
+
+    // Filter out allowlisted findings
+    let store = Store::open(db_path).ok();
+    let active_findings: Vec<Finding> = if let Some(ref s) = store {
+        findings
+            .into_iter()
+            .filter(|f| !s.is_allowed(&f.fingerprint).unwrap_or(false))
+            .collect()
+    } else {
+        findings
+    };
+
+    // Record findings to DB
+    if let Some(ref s) = store {
+        for f in &active_findings {
+            let _ = s.record_finding(f, tool_name, session_id);
+        }
+        let _ = s.record_scan(
+            session_id,
+            tool_name,
+            text.len(),
+            active_findings.len(),
+            active_findings.len(),
+        );
+    }
+
+    let redacted_text = apply_redactions(text, &active_findings);
+    let clean = active_findings.is_empty();
+
+    ScanResult {
+        original_len: text.len(),
+        redacted_text,
+        findings: active_findings,
+        clean,
+    }
+}