chameleon-cv 0.1.0

package/scripts/job_scanner/base.py

@@ -0,0 +1,197 @@
+"""Base interface for job platform parsers."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field, asdict
+import hashlib
+import re
+import time
+
+
+@dataclass
+class JobListing:
+    """Unified job listing from any platform."""
+    title: str
+    company: str
+    url: str
+    location: str = ""
+    remote: bool = True
+    tags: list[str] = field(default_factory=list)
+    salary: str = ""
+    posted_at: str = ""
+    description: str = ""
+    source: str = ""
+    job_id: str = ""
+
+    def __post_init__(self):
+        if not self.job_id:
+            raw = f"{self.url}{self.title}{self.company}".encode()
+            try:
+                self.job_id = hashlib.md5(raw, usedforsecurity=False).hexdigest()[:12]
+            except TypeError:
+                self.job_id = hashlib.md5(raw).hexdigest()[:12]
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+class BaseParser(ABC):
+    """Base class all platform parsers inherit from."""
+
+    platform_name: str = "unknown"
+    _last_request_time: float = 0.0
+    _min_request_interval: float = 0.5
+
+    @abstractmethod
+    def fetch_jobs(
+        self,
+        query: str = "",
+        tags: list[str] | None = None,
+        limit: int = 25,
+        **kwargs,
+    ) -> list[JobListing]:
+        """Fetch jobs from the platform. Must be implemented by subclasses."""
+        ...
+
+    def _rate_limit(self) -> None:
+        """Simple rate limiting between requests."""
+        now = time.time()
+        elapsed = now - self._last_request_time
+        if elapsed < self._min_request_interval:
+            time.sleep(self._min_request_interval - elapsed)
+        self._last_request_time = time.time()
+
+    def _http_get(self, url: str, params: dict | None = None, headers: dict | None = None,
+                   timeout: int = 15, retries: int = 2) -> object | None:
+        """HTTP GET with retries and rate limiting. Returns httpx.Response or None."""
+        self._rate_limit()
+        default_headers = {
+            "User-Agent": "Mozilla/5.0 (compatible; ChameleonBot/1.0)"
+        }
+        if headers:
+            default_headers.update(headers)
+
+        for attempt in range(retries + 1):
+            try:
+                import httpx
+                resp = httpx.get(url, params=params, headers=default_headers,
+                                 timeout=timeout, follow_redirects=True)
+                if resp.status_code in (429, 403):
+                    if attempt < retries:
+                        time.sleep(2 ** attempt)
+                        continue
+                    return None
+                resp.raise_for_status()
+                return resp
+            except ImportError:
+                # Fallback to urllib if httpx not available
+                import urllib.request
+                import urllib.error
+                import json as _json
+                try:
+                    if params:
+                        from urllib.parse import urlencode
+                        url = f"{url}?{urlencode(params)}"
+                    req = urllib.request.Request(url, headers=default_headers)
+                    with urllib.request.urlopen(req, timeout=timeout) as resp:
+                        data = resp.read().decode("utf-8", errors="replace")
+                        # Return a minimal object mimicking httpx.Response
+                        class _FakeResp:
+                            def __init__(self, text, status_code=200):
+                                self.text = text
+                                self.status_code = status_code
+                            def json(self):
+                                return _json.loads(self.text)
+                            def raise_for_status(self):
+                                if self.status_code >= 400:
+                                    raise Exception(f"HTTP {self.status_code}")
+                        return _FakeResp(data)
+                except Exception:
+                    if attempt < retries:
+                        time.sleep(1)
+                        continue
+                    return None
+            except Exception:
+                if attempt < retries:
+                    time.sleep(1)
+                    continue
+                return None
+        return None
+
+    def _clean_html(self, text: str) -> str:
+        """Strip HTML tags and decode entities from text."""
+        import html
+        text = html.unescape(text)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+        return text
+
+    def _truncate(self, text: str, max_len: int = 500) -> str:
+        """Truncate text to max_len chars."""
+        if len(text) <= max_len:
+            return text
+        return text[:max_len].rsplit(" ", 1)[0] + "..."
+
+    def _extract_next_data(self, html: str) -> dict | None:
+        """Extract __NEXT_DATA__ JSON from a Next.js page."""
+        import json
+        m = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', html, re.DOTALL)
+        if m:
+            try:
+                return json.loads(m.group(1))
+            except json.JSONDecodeError:
+                return None
+        # Try data-inline-stat json
+        m2 = re.search(r'data-inline-stat=["\']({.*?})["\']', html, re.DOTALL)
+        if m2:
+            try:
+                return json.loads(m2.group(1))
+            except json.JSONDecodeError:
+                pass
+        return None
+
+    def _extract_json_ld(self, html: str) -> list[dict]:
+        """Extract all JSON-LD structured data blobs from HTML."""
+        import json
+        results = []
+        for m in re.finditer(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', html, re.DOTALL):
+            try:
+                data = json.loads(m.group(1))
+                if isinstance(data, list):
+                    results.extend(data)
+                else:
+                    results.append(data)
+            except json.JSONDecodeError:
+                pass
+        return results
+
+    def _browser_get(self, url: str, timeout: int = 15, **kwargs) -> object | None:
+        """Fetch a page using Playwright browser automation (anti-bot bypass)."""
+        from .browser import browser_get
+        return browser_get(url, timeout=timeout, **kwargs)
+
+    def _browser_available(self) -> bool:
+        """Check if Playwright browser automation is available."""
+        from .browser import is_playwright_available
+        return is_playwright_available()
+
+    @staticmethod
+    def _is_login_wall(text: str) -> bool:
+        """Detect if the page is behind a login/authentication wall.
+
+        Returns True if the page shows a sign-in wall (not just a form on
+        a legitimate content page).
+        """
+        low = text.lower()
+        markers = [
+            "sign in to view", "sign in to see", "sign in to continue",
+            "please sign in", "please log in", "please login",
+            "create an account to view", "create your account to view",
+            "join to view", "join to see", "subscribe to view",
+            "login to view", "login to see",
+            "auth-wall", "authentication-wall",
+        ]
+        # Require at least 2 markers to reduce false positives
+        count = sum(1 for m in markers if m in low)
+        return count >= 2

package/scripts/job_scanner/browser.py

@@ -0,0 +1,244 @@
+"""Browser automation module for anti-bot / JS-dependent job sites.
+
+Supports Playwright (auto-installed) with a graceful fallback
+to plain HTTP when the browser is not available.
+"""
+from __future__ import annotations
+
+import os
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+class _FakeResp:
+    """Mimics httpx.Response so callers don't need branching logic."""
+    def __init__(self, text: str = "", status_code: int = 200):
+        self.text = text
+        self.status_code = status_code
+
+    def json(self) -> Any:
+        import json
+        return json.loads(self.text)
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise Exception(f"HTTP {self.status_code}")
+
+
+# Lazy singleton for the browser manager
+_BROWSER_INSTANCE: dict[str, Any] = {}
+
+
+def _get_playwright_page(headless: bool = True, timeout_ms: int = 30000) -> Any:
+    """Lazy-init a Playwright browser page. Returns None on failure."""
+    global _BROWSER_INSTANCE
+    if "page" in _BROWSER_INSTANCE:
+        try:
+            _BROWSER_INSTANCE["page"].evaluate("1")
+            return _BROWSER_INSTANCE["page"]
+        except Exception:
+            pass
+
+    try:
+        from playwright.sync_api import sync_playwright as _sp
+    except ImportError:
+        return None
+
+    try:
+        p = _sp().start()
+        browser = p.chromium.launch(
+            headless=headless,
+            args=[
+                "--no-sandbox",
+                "--disable-blink-features=AutomationControlled",
+                "--disable-dev-shm-usage",
+            ],
+        )
+        ctx = browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/128.0.0.0 Safari/537.36"
+            ),
+            viewport={"width": 1920, "height": 1080},
+            locale="en-US",
+        )
+        page = ctx.new_page()
+        page.set_default_timeout(timeout_ms)
+        _BROWSER_INSTANCE["playwright"] = p
+        _BROWSER_INSTANCE["browser"] = browser
+        _BROWSER_INSTANCE["context"] = ctx
+        _BROWSER_INSTANCE["page"] = page
+        return page
+    except Exception:
+        return None
+
+
+def is_playwright_available() -> bool:
+    """Check if Playwright + chromium are installed."""
+    try:
+        import playwright  # noqa: F401
+    except ImportError:
+        return False
+    try:
+        from playwright.sync_api import sync_playwright as _sp  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def browser_get(
+    url: str,
+    timeout: int = 15,
+    wait_for: str | None = None,
+    wait_selector: str | None = None,
+    scroll: bool = False,
+    headless: bool = True,
+) -> _FakeResp | None:
+    """Fetch a page using Playwright, falling back to None.
+
+    Parameters
+    ----------
+    url : str
+        The URL to fetch.
+    timeout : int
+        Page load timeout in seconds.
+    wait_for : str, optional
+        A CSS selector or XPath to wait for before returning.
+    wait_selector : str, optional
+        Alias for wait_for.
+    scroll : bool
+        If True, scroll to bottom to trigger lazy loading.
+    headless : bool
+        Run browser in headless mode.
+    """
+    page = _get_playwright_page(headless=headless, timeout_ms=timeout * 1000)
+    if page is None:
+        return None
+
+    try:
+        page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+        # Wait for network idle (up to half the timeout)
+        try:
+            page.wait_for_load_state("networkidle", timeout=min(timeout * 500, 10000))
+        except Exception:
+            pass
+        # Wait for a specific selector if requested
+        selector = wait_for or wait_selector
+        if selector:
+            try:
+                page.wait_for_selector(selector, timeout=5000)
+            except Exception:
+                pass
+        # Scroll to trigger lazy content
+        if scroll:
+            try:
+                page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+                time.sleep(1)
+            except Exception:
+                pass
+        html = page.content()
+        return _FakeResp(text=html, status_code=200)
+    except Exception:
+        return None
+
+
+def browser_extract_json(
+    url: str,
+    timeout: int = 15,
+    script_pattern: str | None = None,
+    headless: bool = True,
+) -> list[dict] | dict | None:
+    """Fetch a page via browser and extract embedded JSON data.
+
+    Parameters
+    ----------
+    url : str
+        The URL to fetch.
+    timeout : int
+        Page load timeout in seconds.
+    script_pattern : str, optional
+        Regex pattern to find JSON inside <script> tags.
+        If None, returns full page as text.
+    headless : bool
+        Run browser in headless mode.
+    """
+    page = _get_playwright_page(headless=headless, timeout_ms=timeout * 1000)
+    if page is None:
+        return None
+
+    try:
+        page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+        try:
+            page.wait_for_load_state("networkidle", timeout=min(timeout * 500, 10000))
+        except Exception:
+            pass
+
+        if script_pattern:
+            content = page.content()
+            import json
+            for m in re.finditer(script_pattern, content, re.DOTALL):
+                try:
+                    return json.loads(m.group(1))
+                except json.JSONDecodeError:
+                    continue
+            return None
+        else:
+            return page.content()
+    except Exception:
+        return None
+
+
+def close_browser() -> None:
+    """Clean up browser resources."""
+    global _BROWSER_INSTANCE
+    try:
+        if "page" in _BROWSER_INSTANCE:
+            _BROWSER_INSTANCE["page"].close()
+        if "context" in _BROWSER_INSTANCE:
+            _BROWSER_INSTANCE["context"].close()
+        if "browser" in _BROWSER_INSTANCE:
+            _BROWSER_INSTANCE["browser"].close()
+        if "playwright" in _BROWSER_INSTANCE:
+            _BROWSER_INSTANCE["playwright"].stop()
+    except Exception:
+        pass
+    _BROWSER_INSTANCE = {}
+
+
+def install_playwright(progress_callback=None) -> bool:
+    """Install Playwright and chromium browser."""
+    try:
+        import subprocess
+        if progress_callback:
+            progress_callback("Installing Playwright Python package...")
+        r = subprocess.run(
+            [sys.executable, "-m", "pip", "install", "--quiet", "playwright"],
+            capture_output=True, text=True,
+        )
+        if r.returncode != 0:
+            if progress_callback:
+                progress_callback(f"pip install failed: {r.stderr}")
+            return False
+
+        if progress_callback:
+            progress_callback("Installing Chromium browser (~150MB)...")
+        r = subprocess.run(
+            [sys.executable, "-m", "playwright", "install", "chromium"],
+            capture_output=True, text=True,
+        )
+        if r.returncode != 0:
+            if progress_callback:
+                progress_callback(f"chromium install failed: {r.stderr}")
+            return False
+
+        if progress_callback:
+            progress_callback("Playwright + Chromium installed successfully!")
+        return True
+    except Exception as e:
+        if progress_callback:
+            progress_callback(f"Install error: {e}")
+        return False

package/scripts/job_scanner/cli.py

@@ -0,0 +1,6 @@
+"""CLI entry point for job scanner: python3 -m scripts.job_scanner.scanner"""
+
+from .scanner import main
+
+if __name__ == "__main__":
+    main()

package/scripts/job_scanner/parsers/__init__.py

@@ -0,0 +1,24 @@
+"""Parsers package — auto-imports all platform parsers to trigger registration."""
+
+from . import (  # noqa: F401
+    himalayas,
+    remoteok,
+    remotive,
+    weworkremotely,
+    hn_hiring,
+    jobicy,
+    adzuna,
+    jooble,
+    arc,
+    crypto_careers,
+    dailyremote,
+    relocate,
+    web3_career,
+    ycomb,
+    linkedin,
+    indeed,
+    wellfound,
+    flexjobs,
+    opentoworkremote,
+    remote_rocketship,
+)

package/scripts/job_scanner/parsers/adzuna.py

@@ -0,0 +1,81 @@
+"""Adzuna — structured job API with salary data. Tier 1 (free tier).
+
+Requires ADZUNA_APP_ID and ADZUNA_API_KEY env vars. Falls back gracefully.
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.parse
+import urllib.request
+import urllib.error
+from typing import Any
+
+from ..base import BaseParser, JobListing
+from ..registry import register_parser
+
+
+@register_parser
+class AdzunaParser(BaseParser):
+    platform_name = "adzuna"
+
+    def fetch_jobs(self, query: str = "", **kwargs: Any) -> list[JobListing]:
+        app_id = os.environ.get("ADZUNA_APP_ID")
+        api_key = os.environ.get("ADZUNA_API_KEY")
+        if not app_id or not api_key:
+            return []
+
+        country = kwargs.get("country", "gb")
+        params = {
+            "app_id": app_id,
+            "app_key": api_key,
+            "results_per_page": str(min(kwargs.get("limit", 25), 50)),
+            "content_type": "application/json",
+        }
+        if query:
+            params["what"] = query
+        params["where"] = kwargs.get("location", "")
+
+        url = f"https://api.adzuna.com/v1/api/jobs/{country}/search/1?" + urllib.parse.urlencode(params)
+
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "Chameleon/1.0"})
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                data = json.loads(resp.read().decode())
+        except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, OSError):
+            return []
+
+        jobs: list[JobListing] = []
+        for j in data.get("results", [])[:kwargs.get("limit", 25)]:
+            title = (j.get("title") or "").strip()
+            company = (j.get("company", {}) or {}).get("display_name") or "Unknown"
+            url = j.get("redirect_url") or j.get("url") or ""
+            desc = (j.get("description") or "")[:2000]
+            location = j.get("location", {}).get("display_name") or j.get("location") or ""
+            salary_min = j.get("salary_min") or j.get("salary_minimum")
+            salary_max = j.get("salary_max") or j.get("salary_maximum")
+            salary = ""
+            if salary_min and salary_max:
+                salary = f"${int(salary_min):,} - ${int(salary_max):,}"
+            elif salary_min:
+                salary = f"${int(salary_min):,}+"
+            remote = False
+            if desc and ("remote" in desc.lower() or "anywhere" in desc.lower()):
+                remote = True
+            if location and ("remote" in location.lower()):
+                remote = True
+
+            job_id = j.get("id") or f"adzuna-{hash(title + company)}"
+            jobs.append(JobListing(
+                job_id=f"adz-{job_id}",
+                title=title,
+                company=company,
+                url=url,
+                location=location,
+                description=desc,
+                salary=salary,
+                source=self.platform_name,
+                remote=remote,
+                tags=[],
+            ))
+        return jobs