osintkit 0.1.0

package/osintkit/modules/sherlock.py

@@ -0,0 +1,48 @@
+"""Sherlock social profile enumeration module."""
+
+import asyncio
+import shutil
+from typing import Any, Dict, List
+
+
+async def run_sherlock(inputs: Dict[str, Any], timeout_seconds: int) -> List[Dict]:
+    """Run sherlock subprocess for username lookup.
+
+    Returns list of found social profiles across platforms.
+    Skips gracefully if sherlock is not installed.
+    """
+    username = inputs.get("username")
+    if not username:
+        return []
+
+    if not shutil.which("sherlock"):
+        return []
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "sherlock", username, "--print-found", "--timeout", "10",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout_seconds + 15)
+
+        findings = []
+        for line in stdout.decode(errors="replace").splitlines():
+            line = line.strip()
+            if line.startswith("[+]"):
+                # Format: [+] Platform: https://...
+                parts = line[3:].strip().split(":", 1)
+                platform = parts[0].strip() if parts else "unknown"
+                url = parts[1].strip() if len(parts) > 1 else None
+                findings.append({
+                    "source": "sherlock",
+                    "type": "social_profile",
+                    "data": {"platform": platform, "username": username},
+                    "url": url,
+                })
+        return findings
+
+    except asyncio.TimeoutError:
+        return []
+    except Exception:
+        return []

package/osintkit/modules/social.py

@@ -0,0 +1,58 @@
+"""Social profile enumeration via Maigret."""
+
+import asyncio
+import shutil
+import json
+from typing import Any, Dict, List
+
+from osintkit.modules import ModuleError
+
+
+async def run_social_profiles(inputs: Dict[str, Any], timeout_seconds: int) -> List[Dict]:
+    """Run social profile enumeration using Maigret."""
+    username = inputs.get("username")
+    if not username:
+        return []
+
+    if not shutil.which("maigret"):
+        raise ModuleError("Maigret not installed. Install with: pip install maigret")
+
+    try:
+        import tempfile, shutil as _shutil
+        tmpdir = tempfile.mkdtemp(prefix="osintkit_maigret_")
+        proc = await asyncio.create_subprocess_exec(
+            "maigret", username,
+            "-J", "simple",
+            "--folderoutput", tmpdir,
+            "--timeout", str(timeout_seconds),
+            "--no-progressbar",
+            stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE,
+        )
+        try:
+            _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_seconds + 30)
+        except asyncio.TimeoutError:
+            proc.kill()
+            await proc.wait()
+            raise ModuleError("Maigret timed out")
+
+        findings = []
+        import os, pathlib
+        report_path = pathlib.Path(tmpdir) / f"report_{username}_simple.json"
+        if report_path.exists():
+            try:
+                raw = json.loads(report_path.read_text())
+                for site, data in raw.items():
+                    if isinstance(data, dict) and data.get("status", {}).get("id") in ("claimed", "exists"):
+                        findings.append({
+                            "source": "maigret", "type": "social_profile",
+                            "data": {"platform": site, "username": username},
+                            "confidence": 0.9, "url": data.get("url_user")
+                        })
+            except (json.JSONDecodeError, Exception):
+                pass
+        _shutil.rmtree(tmpdir, ignore_errors=True)
+        return findings
+    except asyncio.TimeoutError:
+        raise ModuleError(f"Maigret timed out")
+    except json.JSONDecodeError as e:
+        raise ModuleError(f"Parse error: {e}")

package/osintkit/modules/stage2/__init__.py

@@ -0,0 +1 @@
+"""Stage 2 OSINT modules — require API keys."""

package/osintkit/modules/stage2/github_api.py

@@ -0,0 +1,65 @@
+"""GitHub user lookup via GitHub API (Stage 2 — requires API token)."""
+
+from typing import Dict, List
+
+import httpx
+
+
+async def run(inputs: dict, api_key: str) -> List[Dict]:
+    """Look up a GitHub user profile via the GitHub REST API.
+
+    Args:
+        inputs: dict with at least 'username' key (handle)
+        api_key: GitHub personal access token
+
+    Returns:
+        List with a single GitHub profile finding or [] if not found.
+
+    Raises:
+        Exception: on rate limiting (429) or invalid key (401/403).
+    """
+    username = inputs.get("username")
+    if not username:
+        return []
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                f"https://api.github.com/users/{username}",
+                headers={
+                    "Authorization": f"token {api_key}",
+                    "Accept": "application/vnd.github.v3+json",
+                },
+            )
+
+        if response.status_code == 429:
+            raise Exception("429 rate limited")
+        if response.status_code in (401, 403):
+            raise Exception("401 invalid key")
+        if response.status_code == 404:
+            return []
+
+        data = response.json()
+
+        return [{
+            "source": "github_api",
+            "type": "social_profile",
+            "data": {
+                "login": data.get("login"),
+                "name": data.get("name"),
+                "bio": data.get("bio"),
+                "company": data.get("company"),
+                "location": data.get("location"),
+                "email": data.get("email"),
+                "public_repos": data.get("public_repos"),
+                "followers": data.get("followers"),
+                "following": data.get("following"),
+                "created_at": data.get("created_at"),
+            },
+            "url": data.get("html_url"),
+        }]
+
+    except Exception as e:
+        if "429" in str(e) or "401" in str(e):
+            raise
+        return []

package/osintkit/modules/stage2/hunter.py

@@ -0,0 +1,64 @@
+"""Hunter.io email verifier (Stage 2 — requires API key)."""
+
+from typing import Dict, List
+
+import httpx
+
+
+async def run(inputs: dict, api_key: str) -> List[Dict]:
+    """Verify an email address via Hunter.io email-verifier endpoint.
+
+    Args:
+        inputs: dict with at least 'email' key
+        api_key: Hunter.io API key
+
+    Returns:
+        List with a single email verification finding or [] on no result.
+
+    Raises:
+        Exception: on rate limiting (429) or invalid key (401/403).
+    """
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                "https://api.hunter.io/v2/email-verifier",
+                params={"email": email, "api_key": api_key},
+            )
+
+        if response.status_code == 429:
+            raise Exception("429 rate limited")
+        if response.status_code in (401, 403):
+            raise Exception("401 invalid key")
+
+        data = response.json()
+        result = data.get("data", {})
+        if not result:
+            return []
+
+        return [{
+            "source": "hunter",
+            "type": "email_verification",
+            "data": {
+                "status": result.get("status"),
+                "score": result.get("score"),
+                "regexp": result.get("regexp"),
+                "gibberish": result.get("gibberish"),
+                "disposable": result.get("disposable"),
+                "webmail": result.get("webmail"),
+                "mx_records": result.get("mx_records"),
+                "smtp_server": result.get("smtp_server"),
+                "smtp_check": result.get("smtp_check"),
+                "accept_all": result.get("accept_all"),
+                "email": email,
+            },
+            "url": None,
+        }]
+
+    except Exception as e:
+        if "429" in str(e) or "401" in str(e):
+            raise
+        return []

package/osintkit/modules/stage2/leakcheck.py

@@ -0,0 +1,58 @@
+"""LeakCheck.io email breach lookup (Stage 2 — requires API key)."""
+
+from typing import Dict, List
+
+import httpx
+
+
+async def run(inputs: dict, api_key: str) -> List[Dict]:
+    """Query LeakCheck.io for email breach records.
+
+    Args:
+        inputs: dict with at least 'email' key
+        api_key: LeakCheck API key
+
+    Returns:
+        List of breach finding dicts or [] on no results.
+
+    Raises:
+        Exception: on rate limiting (429) or invalid key (401/403).
+    """
+    email = inputs.get("email")
+    if not email:
+        return []
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                "https://leakcheck.io/api/public",
+                params={"key": api_key, "check": email},
+            )
+
+        if response.status_code == 429:
+            raise Exception("429 rate limited")
+        if response.status_code in (401, 403):
+            raise Exception("401 invalid key")
+
+        data = response.json()
+        if not data.get("success") or not data.get("result"):
+            return []
+
+        findings = []
+        for record in data["result"]:
+            findings.append({
+                "source": "leakcheck",
+                "type": "breach_record",
+                "data": {
+                    "source_name": record.get("source", {}).get("name", "unknown"),
+                    "fields": record.get("fields", []),
+                    "email": email,
+                },
+                "url": None,
+            })
+        return findings
+
+    except Exception as e:
+        if "429" in str(e) or "401" in str(e):
+            raise
+        return []

package/osintkit/modules/stage2/numverify.py

@@ -0,0 +1,62 @@
+"""NumVerify phone validation (Stage 2 — requires API key)."""
+
+from typing import Dict, List
+
+import httpx
+
+
+async def run(inputs: dict, api_key: str) -> List[Dict]:
+    """Validate a phone number via the NumVerify/apilayer API.
+
+    Args:
+        inputs: dict with at least 'phone' key
+        api_key: NumVerify/apilayer access key
+
+    Returns:
+        List with a single phone validation finding or [] on no result.
+
+    Raises:
+        Exception: on rate limiting (429) or invalid key (401/403).
+    """
+    phone = inputs.get("phone")
+    if not phone:
+        return []
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                "https://apilayer.net/api/validate",
+                params={"access_key": api_key, "number": phone},
+            )
+
+        if response.status_code == 429:
+            raise Exception("429 rate limited")
+        if response.status_code in (401, 403):
+            raise Exception("401 invalid key")
+
+        data = response.json()
+        if not data.get("valid") and data.get("error"):
+            return []
+
+        return [{
+            "source": "numverify",
+            "type": "phone_validation",
+            "data": {
+                "valid": data.get("valid"),
+                "number": data.get("number"),
+                "local_format": data.get("local_format"),
+                "international_format": data.get("international_format"),
+                "country_prefix": data.get("country_prefix"),
+                "country_code": data.get("country_code"),
+                "country_name": data.get("country_name"),
+                "location": data.get("location"),
+                "carrier": data.get("carrier"),
+                "line_type": data.get("line_type"),
+            },
+            "url": None,
+        }]
+
+    except Exception as e:
+        if "429" in str(e) or "401" in str(e):
+            raise
+        return []

package/osintkit/modules/stage2/securitytrails.py

@@ -0,0 +1,65 @@
+"""SecurityTrails subdomain enumeration (Stage 2 — requires API key)."""
+
+from typing import Dict, List
+
+import httpx
+
+
+async def run(inputs: dict, api_key: str) -> List[Dict]:
+    """Enumerate subdomains for the target's email domain via SecurityTrails API.
+
+    Args:
+        inputs: dict with at least 'email' key (domain extracted) or 'domain' key
+        api_key: SecurityTrails API key
+
+    Returns:
+        List of subdomain findings or [] on no results.
+
+    Raises:
+        Exception: on rate limiting (429) or invalid key (401/403).
+    """
+    domain = inputs.get("domain")
+    if not domain:
+        email = inputs.get("email")
+        if email and "@" in email:
+            domain = email.split("@", 1)[1].strip()
+
+    if not domain:
+        return []
+
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            response = await client.get(
+                f"https://api.securitytrails.com/v1/domain/{domain}/subdomains",
+                headers={"APIKEY": api_key},
+            )
+
+        if response.status_code == 429:
+            raise Exception("429 rate limited")
+        if response.status_code in (401, 403):
+            raise Exception("401 invalid key")
+
+        data = response.json()
+        subdomains = data.get("subdomains", [])
+        if not subdomains:
+            return []
+
+        findings = []
+        for sub in subdomains:
+            full = f"{sub}.{domain}"
+            findings.append({
+                "source": "securitytrails",
+                "type": "subdomain",
+                "data": {
+                    "subdomain": sub,
+                    "domain": domain,
+                    "fqdn": full,
+                },
+                "url": f"https://{full}",
+            })
+        return findings
+
+    except Exception as e:
+        if "429" in str(e) or "401" in str(e):
+            raise
+        return []

package/osintkit/modules/wayback.py

@@ -0,0 +1,70 @@
+"""Wayback Machine CDX API lookup for email domain and username."""
+
+from typing import Any, Dict, List
+
+import httpx
+
+
+async def run_wayback(inputs: Dict[str, Any]) -> List[Dict]:
+    """Query the Wayback CDX API for archived URLs related to target inputs.
+
+    Checks the email domain and any handles/usernames found in inputs.
+    Returns a list of archived URL findings.
+    """
+    targets = []
+
+    email = inputs.get("email")
+    if email and "@" in email:
+        domain = email.split("@", 1)[1].strip()
+        if domain:
+            targets.append(domain)
+
+    username = inputs.get("username")
+    if username:
+        targets.append(username)
+
+    if not targets:
+        return []
+
+    findings = []
+
+    async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+        for target in targets:
+            try:
+                params = {
+                    "url": f"*.{target}",
+                    "output": "json",
+                    "limit": "5",
+                    "fl": "original,timestamp",
+                }
+                response = await client.get(
+                    "http://web.archive.org/cdx/search/cdx",
+                    params=params,
+                )
+
+                if response.status_code != 200:
+                    continue
+
+                rows = response.json()
+                # First row is header when output=json
+                if not rows or len(rows) < 2:
+                    continue
+
+                for row in rows[1:]:
+                    if len(row) >= 2:
+                        archived_url, timestamp = row[0], row[1]
+                        findings.append({
+                            "source": "wayback",
+                            "type": "web_archive",
+                            "data": {
+                                "url": archived_url,
+                                "timestamp": timestamp,
+                                "target": target,
+                            },
+                            "url": f"https://web.archive.org/web/{timestamp}/{archived_url}",
+                        })
+
+            except Exception:
+                continue
+
+    return findings

package/osintkit/output/__init__.py

@@ -0,0 +1 @@
+"""osintkit output writers."""

package/osintkit/output/html_writer.py

@@ -0,0 +1,36 @@
+"""HTML report writer using Jinja2."""
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+from jinja2 import Environment, FileSystemLoader
+
+
+def _scrub_keys(content: str, api_keys: Optional[Dict[str, str]] = None) -> str:
+    """Replace any API key values in content with [REDACTED]."""
+    if not api_keys:
+        return content
+    for _name, value in api_keys.items():
+        if value and len(value) > 6:
+            content = content.replace(value, "[REDACTED]")
+    return content
+
+
+def write_html(
+    findings: Dict[str, Any],
+    output_dir: Path,
+    api_keys: Optional[Dict[str, str]] = None,
+) -> Path:
+    """Render HTML report via Jinja2 template.
+
+    API key values are scrubbed from the rendered output before writing.
+    """
+    templates_dir = Path(__file__).parent / "templates"
+    env = Environment(loader=FileSystemLoader(templates_dir))
+    template = env.get_template("report.html")
+
+    html_content = template.render(**findings)
+    html_content = _scrub_keys(html_content, api_keys)
+
+    output_file = output_dir / "report.html"
+    output_file.write_text(html_content, encoding="utf-8")
+    return output_file

package/osintkit/output/json_writer.py

@@ -0,0 +1,31 @@
+"""JSON output writer."""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+def _scrub_keys(content: str, api_keys: Optional[Dict[str, str]] = None) -> str:
+    """Replace any API key values in content with [REDACTED]."""
+    if not api_keys:
+        return content
+    for _name, value in api_keys.items():
+        if value and len(value) > 6:
+            content = content.replace(value, "[REDACTED]")
+    return content
+
+
+def write_json(
+    findings: Dict[str, Any],
+    output_dir: Path,
+    api_keys: Optional[Dict[str, str]] = None,
+) -> Path:
+    """Write findings to JSON file with 2-space indent.
+
+    API key values are scrubbed from the output before writing.
+    """
+    output_file = output_dir / "findings.json"
+    content = json.dumps(findings, indent=2, default=str)
+    content = _scrub_keys(content, api_keys)
+    output_file.write_text(content, encoding="utf-8")
+    return output_file