osintkit 0.1.0

package/osintkit/modules/breach.py

@@ -0,0 +1,82 @@
+"""Breach exposure check via multiple sources."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.config import APIKeys
+from osintkit.modules import ModuleError
+
+
+async def run_breach_exposure(inputs: Dict[str, Any], api_keys: APIKeys) -> List[Dict]:
+    """Check breach exposure with fallback chain: HIBP -> BreachDirectory -> LeakCheck."""
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    if api_keys.hibp:
+        try:
+            return await _check_hibp(email, api_keys.hibp)
+        except ModuleError:
+            pass
+
+    if api_keys.breachdirectory:
+        try:
+            return await _check_breachdirectory(email, api_keys.breachdirectory)
+        except ModuleError:
+            pass
+
+    if api_keys.leakcheck:
+        try:
+            return await _check_leakcheck(email, api_keys.leakcheck)
+        except ModuleError:
+            pass
+
+    return []
+
+
+async def _check_hibp(email: str, api_key: str) -> List[Dict]:
+    url = f"https://haveibeenpwned.com/api/v3/breaches?email={email}"
+    headers = {"hibp-api-key": api_key, "user-agent": "osintkit-CLI"}
+    resp = await httpx.AsyncClient(timeout=30, headers=headers).get(url)
+    if resp.status_code == 404:
+        return []
+    if resp.status_code != 200:
+        raise ModuleError(f"HIBP error: {resp.status_code}")
+    findings = []
+    for b in resp.json():
+        findings.append({"source": "hibp", "type": "breach",
+            "data": {"breach": b.get("Name"), "domain": b.get("Domain"),
+                "date": b.get("BreachDate"), "classes": b.get("DataClasses", [])},
+            "confidence": 0.95, "url": f"https://haveibeenpwned.com/breach/{b.get('Name')}"})
+    return findings
+
+
+async def _check_breachdirectory(email: str, api_key: str) -> List[Dict]:
+    url = "https://breachdirectory.p.rapidapi.com/"
+    headers = {"X-RapidAPI-Key": api_key, "X-RapidAPI-Host": "breachdirectory.p.rapidapi.com"}
+    resp = await httpx.AsyncClient(timeout=30, headers=headers).get(url, params={"func": "auto", "term": email})
+    if resp.status_code != 200:
+        raise ModuleError(f"BreachDirectory error: {resp.status_code}")
+    findings = []
+    for b in resp.json().get("result", []):
+        findings.append({"source": "breachdirectory", "type": "breach",
+            "data": {"breach": b.get("breach"), "fields": b.get("fields", [])},
+            "confidence": 0.85, "url": None})
+    return findings
+
+
+async def _check_leakcheck(email: str, api_key: str) -> List[Dict]:
+    url = f"https://leakcheck.io/api/public?check={email}"
+    headers = {"X-API-Key": api_key}
+    resp = await httpx.AsyncClient(timeout=30, headers=headers).get(url)
+    if resp.status_code != 200:
+        raise ModuleError(f"LeakCheck error: {resp.status_code}")
+    data = resp.json()
+    if not data.get("found"):
+        return []
+    findings = []
+    for s in data.get("sources", []):
+        findings.append({"source": "leakcheck", "type": "breach",
+            "data": {"breach": s.get("name"), "date": s.get("date")},
+            "confidence": 0.80, "url": None})
+    return findings

package/osintkit/modules/brokers.py

@@ -0,0 +1,56 @@
+"""Data broker search via Google CSE or direct HTTP."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.config import APIKeys
+from osintkit.modules import ModuleError
+
+BROKER_SITES = ["whitepages.com", "spokeo.com", "instantcheckmate.com",
+    "truepeoplesearch.com", "fastpeoplesearch.com", "familytreenow.com"]
+
+
+async def run_data_brokers(inputs: Dict[str, Any], api_keys: APIKeys) -> List[Dict]:
+    """Search data brokers with fallback: Google CSE -> direct HTTP."""
+    name = inputs.get("name")
+    if not name:
+        return []
+
+    if api_keys.google_cse_key and api_keys.google_cse_cx:
+        try:
+            return await _search_google_cse(name, api_keys.google_cse_key, api_keys.google_cse_cx)
+        except ModuleError:
+            pass
+
+    return await _search_direct_http(name)
+
+
+async def _search_google_cse(name: str, api_key: str, cx: str) -> List[Dict]:
+    url = "https://www.googleapis.com/customsearch/v1"
+    findings = []
+    client = httpx.AsyncClient(timeout=30)
+    for site in BROKER_SITES[:3]:
+        resp = await client.get(url, params={"key": api_key, "cx": cx, "q": f"{name} site:{site}"})
+        if resp.status_code == 200:
+            for item in resp.json().get("items", []):
+                findings.append({"source": "google_cse", "type": "data_broker",
+                    "data": {"site": site, "title": item.get("title"), "snippet": item.get("snippet")},
+                    "confidence": 0.85, "url": item.get("link")})
+    return findings
+
+
+async def _search_direct_http(name: str) -> List[Dict]:
+    slug = name.replace(" ", "-").lower()
+    findings = []
+    client = httpx.AsyncClient(timeout=15, follow_redirects=True)
+    for site in BROKER_SITES:
+        url = f"https://{site}/person/{slug}"
+        try:
+            resp = await client.get(url)
+            if resp.status_code == 200:
+                findings.append({"source": "direct_http", "type": "data_broker_potential",
+                    "data": {"site": site, "name": name, "url": url},
+                    "confidence": 0.4, "url": url})
+        except httpx.HTTPError:
+            pass
+    return findings

package/osintkit/modules/certs.py

@@ -0,0 +1,42 @@
+"""Certificate transparency check via crt.sh."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.modules import ModuleError
+
+COMMON_DOMAINS = ["gmail.com", "yahoo.com", "hotmail.com", "outlook.com", "icloud.com"]
+
+
+async def run_cert_transparency(inputs: Dict[str, Any]) -> List[Dict]:
+    """Query certificate transparency logs via crt.sh."""
+    email = inputs.get("email")
+    if not email or "@" not in email:
+        return []
+    
+    domain = email.split("@")[1]
+    if domain in COMMON_DOMAINS:
+        return []
+
+    url = f"https://crt.sh/?q={domain}&output=json"
+    try:
+        resp = await httpx.AsyncClient(timeout=30).get(url)
+        if resp.status_code == 404:
+            return []
+        if resp.status_code != 200:
+            raise ModuleError(f"crt.sh error: {resp.status_code}")
+
+        certs = resp.json()
+        findings = []
+        seen = set()
+        for cert in certs:
+            for name in cert.get("name_value", "").split("\n"):
+                name = name.strip()
+                if name and name not in seen:
+                    seen.add(name)
+                    findings.append({"source": "crtsh", "type": "ssl_cert",
+                        "data": {"domain": name, "issuer": cert.get("issuer_name")},
+                        "confidence": 0.8, "url": f"https://crt.sh/?q={name}"})
+        return findings
+    except httpx.HTTPError as e:
+        raise ModuleError(f"crt.sh failed: {e}")

package/osintkit/modules/dark_web.py

@@ -0,0 +1,51 @@
+"""Dark web search via Intelbase or Ahmia."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.config import APIKeys
+from osintkit.modules import ModuleError
+
+
+async def run_dark_web(inputs: Dict[str, Any], api_keys: APIKeys) -> List[Dict]:
+    """Search dark web with fallback: Intelbase -> Ahmia."""
+    query = inputs.get("email") or inputs.get("username") or inputs.get("name")
+    if not query:
+        return []
+
+    if api_keys.intelbase:
+        try:
+            return await _search_intelbase(query, api_keys.intelbase)
+        except ModuleError:
+            pass
+
+    try:
+        return await _search_ahmia(query)
+    except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError,
+            httpx.RemoteProtocolError, Exception):
+        return []
+
+
+async def _search_intelbase(query: str, api_key: str) -> List[Dict]:
+    url = "https://api.intelbase.is/v1/darkweb/search"
+    headers = {"Authorization": f"Bearer {api_key}"}
+    resp = await httpx.AsyncClient(timeout=60, headers=headers).post(url, json={"query": query})
+    if resp.status_code != 200:
+        raise ModuleError(f"Intelbase error: {resp.status_code}")
+    findings = []
+    for r in resp.json().get("results", []):
+        findings.append({"source": "intelbase", "type": "dark_web",
+            "data": {"title": r.get("title"), "url": r.get("url"), "snippet": r.get("snippet")},
+            "confidence": 0.7, "url": r.get("url")})
+    return findings
+
+
+async def _search_ahmia(query: str) -> List[Dict]:
+    url = f"https://ahmia.fi/search/?q={query}"
+    resp = await httpx.AsyncClient(timeout=30).get(url)
+    if resp.status_code != 200:
+        raise ModuleError(f"Ahmia error: {resp.status_code}")
+    # Ahmia returns HTML - return query info
+    return [{"source": "ahmia", "type": "dark_web_search",
+        "data": {"query": query, "search_url": url, "note": "Manual review recommended"},
+        "confidence": 0.5, "url": url}]

package/osintkit/modules/gravatar.py

@@ -0,0 +1,50 @@
+"""Gravatar profile lookup via MD5 email hash."""
+
+import hashlib
+from typing import Any, Dict, List
+
+import httpx
+
+
+async def run_gravatar(inputs: Dict[str, Any]) -> List[Dict]:
+    """Check Gravatar for an email address profile.
+
+    Uses MD5 hash of the email (lowercase, stripped) to query the Gravatar API.
+    Returns [] if no email or no profile found.
+    """
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    email_normalized = email.strip().lower()
+    email_hash = hashlib.md5(email_normalized.encode()).hexdigest()
+    url = f"https://www.gravatar.com/{email_hash}.json"
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(url)
+
+        if response.status_code != 200:
+            return []
+
+        data = response.json()
+        entry = data.get("entry", [{}])[0]
+
+        name = entry.get("name", {})
+        display_name = entry.get("displayName", "")
+        formatted_name = name.get("formatted", "") if isinstance(name, dict) else str(name)
+        profile_url = entry.get("profileUrl", f"https://www.gravatar.com/{email_hash}")
+
+        return [{
+            "source": "gravatar",
+            "type": "email_profile",
+            "data": {
+                "hash": email_hash,
+                "display_name": display_name,
+                "formatted_name": formatted_name,
+            },
+            "url": profile_url,
+        }]
+
+    except Exception:
+        return []

package/osintkit/modules/harvester.py

@@ -0,0 +1,56 @@
+"""Web presence enumeration via theHarvester."""
+
+import asyncio
+import logging
+import shutil
+import tempfile
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+from osintkit.modules import ModuleError
+
+logger = logging.getLogger(__name__)
+
+COMMON_PROVIDERS = ["gmail.com", "yahoo.com", "hotmail.com", "outlook.com", "icloud.com"]
+
+
+async def run_web_presence(inputs: Dict[str, Any], timeout_seconds: int) -> List[Dict]:
+    """Run web presence enumeration using theHarvester."""
+    email = inputs.get("email")
+    if not email or "@" not in email:
+        return []
+    
+    domain = email.split("@")[1]
+    if domain in COMMON_PROVIDERS:
+        return []
+
+    if not shutil.which("theHarvester"):
+        raise ModuleError("theHarvester not installed")
+
+    tmp_base = tempfile.mkdtemp(prefix="osintkit_harvester_")
+    output_file = Path(tmp_base) / f"harvester_{domain}.json"
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "theHarvester", "-d", domain, "-b", "all", "-f", str(output_file),
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+        )
+        await asyncio.wait_for(proc.communicate(), timeout=timeout_seconds + 30)
+
+        findings = []
+        if output_file.exists():
+            try:
+                with open(output_file) as f:
+                    data = json.load(f)
+                for e in data.get("emails", []):
+                    findings.append({"source": "harvester", "type": "associated_email",
+                        "data": {"email": e, "domain": domain}, "confidence": 0.7, "url": None})
+                for h in data.get("hosts", []):
+                    findings.append({"source": "harvester", "type": "associated_host",
+                        "data": {"host": h}, "confidence": 0.6, "url": f"http://{h}"})
+            except Exception as e:
+                raise ModuleError(str(e))
+        shutil.rmtree(tmp_base, ignore_errors=True)
+        return findings
+    except asyncio.TimeoutError:
+        raise ModuleError("theHarvester timed out")

package/osintkit/modules/hibp.py

@@ -0,0 +1,40 @@
+"""Password exposure check via HIBP PwnedPasswords."""
+
+import hashlib
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.modules import ModuleError
+
+
+async def run_password_exposure(inputs: Dict[str, Any]) -> List[Dict]:
+    """Check email breach exposure via HIBP.
+
+    Note: HIBP email breach lookup requires a paid API key (hibp field in
+    config.yaml).  The k-anonymity PwnedPasswords endpoint only accepts
+    password hashes, not email addresses.  Without a key this module returns
+    [] and logs a message so the scanner does not report a failure.
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+    logger.info(
+        "HIBP email lookup requires API key — configure hibp key for results"
+    )
+    return []
+
+
+async def check_password_hash(password: str) -> int:
+    """Check how many times password appears in breaches."""
+    sha1_hash = hashlib.sha1(password.encode()).hexdigest().upper()
+    prefix, suffix = sha1_hash[:5], sha1_hash[5:]
+    
+    url = f"https://api.pwnedpasswords.com/range/{prefix}"
+    try:
+        resp = await httpx.AsyncClient(timeout=15).get(url)
+        for line in resp.text.splitlines():
+            h, count = line.split(":")
+            if h == suffix:
+                return int(count)
+        return 0
+    except httpx.HTTPError as e:
+        raise ModuleError(f"HIBP failed: {e}")

package/osintkit/modules/hibp_kanon.py

@@ -0,0 +1,66 @@
+"""HIBP k-anonymity password exposure check using SHA1 prefix API."""
+
+import hashlib
+import logging
+from typing import Any, Dict, List
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+async def run_hibp_kanon(inputs: Dict[str, Any]) -> List[Dict]:
+    """Check HIBP pwnedpasswords k-anonymity endpoint.
+
+    Demonstrates the SHA1 prefix pattern: hashes the email address as a proxy
+    value and queries the first 5 characters of the SHA1 against the pwnedpasswords
+    range endpoint. This does NOT check actual passwords — it checks whether the
+    email's SHA1 hash happens to appear in the leaked password corpus (non-standard
+    but illustrates the k-anonymity pattern without exposing the full hash).
+
+    Returns count if a matching suffix is found in the range response.
+    """
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    try:
+        sha1 = hashlib.sha1(email.strip().lower().encode()).hexdigest().upper()
+        prefix = sha1[:5]
+        suffix = sha1[5:]
+
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                f"https://api.pwnedpasswords.com/range/{prefix}",
+                headers={"Add-Padding": "true"},
+            )
+
+        if response.status_code != 200:
+            return []
+
+        count = 0
+        for line in response.text.splitlines():
+            parts = line.strip().split(":")
+            if len(parts) == 2 and parts[0].upper() == suffix:
+                try:
+                    count = int(parts[1])
+                except ValueError:
+                    pass
+                break
+
+        if count == 0:
+            return []
+
+        logger.info(f"hibp_kanon: email SHA1 prefix {prefix} found {count} times in pwned passwords")
+        return [{
+            "source": "hibp_kanon",
+            "type": "password_exposure",
+            "data": {
+                "hash_prefix": prefix,
+                "count": count,
+                "note": "Email SHA1 hash matched in pwnedpasswords range (non-standard pattern)",
+            },
+        }]
+
+    except Exception:
+        return []

package/osintkit/modules/holehe.py

@@ -0,0 +1,39 @@
+"""Email account enumeration via Holehe."""
+
+import asyncio
+import shutil
+from typing import Any, Dict, List
+
+from osintkit.modules import ModuleError
+
+
+async def run_email_accounts(inputs: Dict[str, Any], timeout_seconds: int) -> List[Dict]:
+    """Run email account enumeration using Holehe."""
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    if not shutil.which("holehe"):
+        raise ModuleError("Holehe not installed. Install with: pip install holehe")
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "holehe", "--email", email,
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_seconds + 10)
+
+        output = stdout.decode()
+        findings = []
+        for line in output.splitlines():
+            if "+" in line or "used" in line.lower():
+                parts = line.strip().split()
+                if parts:
+                    findings.append({
+                        "source": "holehe", "type": "email_account",
+                        "data": {"platform": parts[0], "email": email},
+                        "confidence": 0.8, "url": None
+                    })
+        return findings
+    except asyncio.TimeoutError:
+        raise ModuleError("Holehe timed out")

package/osintkit/modules/libphonenumber_info.py

@@ -0,0 +1,79 @@
+"""Offline phone number analysis using the phonenumbers library."""
+
+import logging
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+async def run_libphonenumber(inputs: Dict[str, Any]) -> List[Dict]:
+    """Parse and analyse a phone number using the phonenumbers library.
+
+    Returns carrier, region, number type, validity, and E.164 format.
+    Returns [] if no phone input, if phonenumbers is not installed,
+    or if the number cannot be parsed.
+    """
+    phone = inputs.get("phone")
+    if not phone:
+        return []
+
+    try:
+        import phonenumbers
+        from phonenumbers import carrier, geocoder, number_type, is_valid_number, format_number
+        from phonenumbers import PhoneNumberFormat, PhoneNumberType
+    except ImportError:
+        logger.warning("phonenumbers library not installed; skipping phone analysis")
+        return []
+
+    try:
+        parsed = phonenumbers.parse(phone, None)
+    except phonenumbers.NumberParseException:
+        try:
+            # Retry with a default region hint
+            parsed = phonenumbers.parse(phone, "US")
+        except phonenumbers.NumberParseException:
+            logger.warning(f"Could not parse phone number: {phone!r}")
+            return []
+
+    valid = is_valid_number(parsed)
+    e164 = format_number(parsed, PhoneNumberFormat.E164)
+
+    try:
+        carrier_name = carrier.name_for_number(parsed, "en") or "unknown"
+    except Exception:
+        carrier_name = "unknown"
+
+    try:
+        region = geocoder.description_for_number(parsed, "en") or "unknown"
+    except Exception:
+        region = "unknown"
+
+    ntype_int = number_type(parsed)
+    type_map = {
+        PhoneNumberType.MOBILE: "mobile",
+        PhoneNumberType.FIXED_LINE: "fixed_line",
+        PhoneNumberType.FIXED_LINE_OR_MOBILE: "fixed_line_or_mobile",
+        PhoneNumberType.TOLL_FREE: "toll_free",
+        PhoneNumberType.PREMIUM_RATE: "premium_rate",
+        PhoneNumberType.SHARED_COST: "shared_cost",
+        PhoneNumberType.VOIP: "voip",
+        PhoneNumberType.PERSONAL_NUMBER: "personal_number",
+        PhoneNumberType.PAGER: "pager",
+        PhoneNumberType.UAN: "uan",
+        PhoneNumberType.VOICEMAIL: "voicemail",
+        PhoneNumberType.UNKNOWN: "unknown",
+    }
+    number_type_str = type_map.get(ntype_int, "unknown")
+
+    return [{
+        "source": "libphonenumber",
+        "type": "phone_info",
+        "data": {
+            "carrier": carrier_name,
+            "region": region,
+            "number_type": number_type_str,
+            "is_valid": valid,
+            "e164_format": e164,
+        },
+        "url": None,
+    }]

package/osintkit/modules/paste.py

@@ -0,0 +1,55 @@
+"""Paste site search via Intelbase or psbdmp."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.config import APIKeys
+from osintkit.modules import ModuleError
+
+
+async def run_paste_sites(inputs: Dict[str, Any], api_keys: APIKeys) -> List[Dict]:
+    """Search paste sites with fallback: Intelbase -> psbdmp.ws."""
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    if api_keys.intelbase:
+        try:
+            return await _search_intelbase_paste(email, api_keys.intelbase)
+        except ModuleError:
+            pass
+
+    try:
+        return await _search_psbdmp(email)
+    except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError,
+            httpx.RemoteProtocolError, Exception):
+        return []
+
+
+async def _search_intelbase_paste(email: str, api_key: str) -> List[Dict]:
+    url = "https://api.intelbase.is/v1/paste/search"
+    headers = {"Authorization": f"Bearer {api_key}"}
+    resp = await httpx.AsyncClient(timeout=60, headers=headers).post(url, json={"query": email})
+    if resp.status_code != 200:
+        raise ModuleError(f"Intelbase error: {resp.status_code}")
+    findings = []
+    for r in resp.json().get("results", []):
+        findings.append({"source": "intelbase", "type": "paste",
+            "data": {"paste_id": r.get("id"), "site": r.get("site"), "date": r.get("date")},
+            "confidence": 0.75, "url": r.get("url")})
+    return findings
+
+
+async def _search_psbdmp(email: str) -> List[Dict]:
+    url = f"https://psbdmp.ws/api/v3/search/{email}"
+    resp = await httpx.AsyncClient(timeout=30).get(url)
+    if resp.status_code == 404:
+        return []
+    if resp.status_code != 200:
+        raise ModuleError(f"psbdmp error: {resp.status_code}")
+    findings = []
+    for p in resp.json().get("data", []):
+        findings.append({"source": "psbdmp", "type": "paste",
+            "data": {"paste_id": p.get("id"), "date": p.get("date")},
+            "confidence": 0.65, "url": f"https://psbdmp.ws/{p.get('id')}"})
+    return findings

package/osintkit/modules/phone.py

@@ -0,0 +1,32 @@
+"""Phone validation via NumVerify."""
+
+import httpx
+from typing import Any, Dict, List
+
+from osintkit.config import APIKeys
+from osintkit.modules import ModuleError
+
+
+async def run_phone(inputs: Dict[str, Any], api_keys: APIKeys) -> List[Dict]:
+    """Validate phone number using NumVerify (requires key)."""
+    phone = inputs.get("phone")
+    if not phone or not api_keys.numverify:
+        return []
+    
+    clean = phone.lstrip("+").replace(" ", "").replace("-", "").replace("(", "").replace(")", "")
+    url = "https://apilayer.net/api/validate"
+    
+    resp = await httpx.AsyncClient(timeout=30).get(url,
+        params={"access_key": api_keys.numverify, "number": clean, "format": 1})
+    
+    if resp.status_code != 200:
+        raise ModuleError(f"NumVerify error: {resp.status_code}")
+    
+    data = resp.json()
+    if not data.get("valid"):
+        return []
+    
+    return [{"source": "numverify", "type": "phone_info",
+        "data": {"number": phone, "country": data.get("country_name"),
+            "carrier": data.get("carrier"), "line_type": data.get("line_type")},
+        "confidence": 0.9, "url": None}]