@music-league-eras/local-runner 0.1.0

package/vendor/python/app/browser/session.py

@@ -0,0 +1,267 @@
+import asyncio
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import urljoin, urlparse
+
+from playwright.async_api import async_playwright
+
+from ..services.viewer import extract_viewer_avatar_url, extract_viewer_user_id
+
+
+@dataclass(frozen=True)
+class CaptureResult:
+    storage_state: dict[str, Any]
+    storage_state_path: str | None
+    screenshot_path: str | None
+    viewer_user_id: str | None
+    viewer_avatar_url: str | None
+
+
+def _ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def _build_target(base_url: str) -> str:
+    return urljoin(base_url, "/completed/")
+
+
+def _cookie_domain_matches(domain: str, host: str | None) -> bool:
+    if not domain or not host:
+        return False
+    normalized = domain.lstrip(".")
+    return host == normalized or host.endswith(f".{normalized}")
+
+
+def _is_music_league_host(host: str | None, base_url: str) -> bool:
+    if not host:
+        return False
+    base_host = urlparse(base_url).hostname
+    if base_host and _cookie_domain_matches(base_host, host):
+        return True
+    return host.endswith(".musicleague.com") or host == "musicleague.com"
+
+
+def _has_base_cookie(storage_state: dict[str, Any], base_url: str) -> bool:
+    host = urlparse(base_url).hostname
+    if not host:
+        return False
+    for cookie in storage_state.get("cookies", []):
+        domain = str(cookie.get("domain") or "")
+        if _cookie_domain_matches(domain, host):
+            return True
+    return False
+
+
+def _is_missing_playwright_browser_error(exc: BaseException) -> bool:
+    message = str(exc)
+    return (
+        "Executable doesn't exist" in message
+        or "Looks like Playwright was just installed or updated" in message
+        or "playwright install" in message
+    )
+
+
+def _missing_playwright_browser_help() -> str:
+    return (
+        "Playwright browser binaries are not installed for this environment.\n\n"
+        "Fix (repo venv):\n"
+        "  cd services/scraper\n"
+        "  .venv/bin/python -m playwright install chromium\n"
+        "\n"
+        "If you prefer, this also works:\n"
+        "  cd services/scraper\n"
+        "  .venv/bin/playwright install chromium\n"
+    )
+
+
+async def _is_login_prompt_visible(page) -> bool:
+    login_ctas = [
+        "Log in",
+        "Login",
+        "Sign in",
+        "Sign up",
+        "Create account",
+        "Continue with Spotify",
+        "Log in with Spotify",
+        "Sign in with Spotify",
+    ]
+    spotify_hints = [
+        "spotify",
+        "log in",
+        "login",
+        "sign in",
+        "sign up",
+        "continue",
+        "create account",
+    ]
+    try:
+        parsed = urlparse(page.url)
+        if "spotify.com" in (parsed.hostname or ""):
+            return True
+        if parsed.path in {"/login", "/sign-in", "/signin", "/auth"}:
+            return True
+    except Exception:
+        pass
+    for label in login_ctas:
+        try:
+            if await page.get_by_role("button", name=label).count():
+                return True
+        except Exception:
+            pass
+        try:
+            if await page.get_by_role("link", name=label).count():
+                return True
+        except Exception:
+            pass
+    try:
+        body_text = await page.evaluate("document.body ? document.body.innerText : ''")
+        if isinstance(body_text, str):
+            lowered = body_text.lower()
+            if "spotify" in lowered and any(hint in lowered for hint in spotify_hints):
+                return True
+    except Exception:
+        pass
+    return False
+
+
+async def _is_capture_ready(page, base_url: str) -> bool:
+    try:
+        host = urlparse(page.url).hostname
+    except Exception:
+        host = None
+    if not _is_music_league_host(host, base_url):
+        return False
+    if await _is_login_prompt_visible(page):
+        return False
+    return True
+
+
+async def _has_logout_link(page) -> bool:
+    logout_labels = ["Sign Out", "Log out", "Logout"]
+    for label in logout_labels:
+        try:
+            if await page.get_by_role("link", name=label).count():
+                return True
+        except Exception:
+            pass
+    try:
+        if await page.locator("a[href*=\"/logout\"]").count():
+            return True
+    except Exception:
+        pass
+    return False
+
+
+async def _advance_login_flow(page) -> bool:
+    try:
+        buttons = [
+            re.compile(r"^import$", re.I),
+            re.compile(r"import", re.I),
+            re.compile(r"log in with spotify", re.I),
+            re.compile(r"continue with spotify", re.I),
+            re.compile(r"sign in with spotify", re.I),
+            re.compile(r"log in", re.I),
+            re.compile(r"sign in", re.I),
+        ]
+        for pattern in buttons:
+            locator = page.get_by_role("button", name=pattern)
+            if await locator.count():
+                await locator.first.click()
+                try:
+                    await page.wait_for_load_state("domcontentloaded", timeout=3000)
+                except Exception:
+                    pass
+                return True
+        for pattern in buttons:
+            locator = page.get_by_role("link", name=pattern)
+            if await locator.count():
+                await locator.first.click()
+                try:
+                    await page.wait_for_load_state("domcontentloaded", timeout=3000)
+                except Exception:
+                    pass
+                return True
+    except Exception:
+        pass
+    return False
+
+
+async def capture_storage_state(
+    *,
+    base_url: str,
+    headless: bool,
+    storage_state_path: str | None,
+    screenshot_path: str | None,
+) -> CaptureResult:
+    target = _build_target(base_url)
+    async with async_playwright() as p:
+        try:
+            browser = await p.chromium.launch(headless=headless)
+        except Exception as exc:
+            if _is_missing_playwright_browser_error(exc):
+                raise RuntimeError(_missing_playwright_browser_help()) from exc
+            raise
+        context = await browser.new_context()
+        page = await context.new_page()
+        last_advance = 0.0
+        try:
+            await page.goto(target, wait_until="domcontentloaded")
+
+            while True:
+                if page.url.startswith(base_url) and await _is_login_prompt_visible(page):
+                    now = asyncio.get_running_loop().time()
+                    if now - last_advance > 1.0:
+                        if await _advance_login_flow(page):
+                            last_advance = now
+                # Capture as soon as we have valid Music League cookies, even if the user was
+                # already logged in and never hit the Spotify auth domain in this run.
+                if await _is_capture_ready(page, base_url):
+                    storage_state = cast(dict[str, Any], await context.storage_state())
+                    if _has_base_cookie(storage_state, base_url):
+                        if storage_state_path:
+                            path = Path(storage_state_path)
+                            _ensure_parent(path)
+                            path.write_text(json.dumps(storage_state, indent=2), encoding="utf-8")
+                        if screenshot_path:
+                            shot_path = Path(screenshot_path)
+                            _ensure_parent(shot_path)
+                            await page.screenshot(path=str(shot_path), full_page=True)
+                        viewer_user_id = None
+                        viewer_avatar_url = None
+                        try:
+                            html = await page.content()
+                            viewer_user_id = extract_viewer_user_id(html)
+                            viewer_avatar_url = extract_viewer_avatar_url(html)
+                        except Exception:
+                            viewer_user_id = None
+                            viewer_avatar_url = None
+                        return CaptureResult(
+                            storage_state=storage_state,
+                            storage_state_path=storage_state_path,
+                            screenshot_path=screenshot_path,
+                            viewer_user_id=viewer_user_id,
+                            viewer_avatar_url=viewer_avatar_url,
+                        )
+                await asyncio.sleep(0.5)
+        finally:
+            await context.close()
+            await browser.close()
+
+    raise RuntimeError("Login capture aborted before storage state was saved.")
+
+
+def build_storage_state_path(base_dir: str, session_id: str) -> str:
+    filename = f"storage_state_{session_id}.json"
+    dated_dir = datetime.now().strftime("%Y-%m-%d")
+    return str(Path(base_dir) / dated_dir / filename)
+
+
+def build_screenshot_path(base_dir: str, session_id: str) -> str:
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"login_{session_id}_{ts}.png"
+    dated_dir = datetime.now().strftime("%Y-%m-%d")
+    return str(Path(base_dir) / dated_dir / "screenshots" / filename)

package/vendor/python/app/local_runner_cli.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .local_sync_runner import run_local_sync
+
+
+def _env_bool(key: str, default: bool) -> bool:
+    raw = os.getenv(key)
+    if raw is None:
+        return default
+    return raw.strip().lower() == "true"
+
+
+def _default_music_league_base_url() -> str:
+    return os.getenv("MUSIC_LEAGUE_BASE_URL", "https://app.musicleague.com")
+
+
+def _default_capture_headless() -> bool:
+    # Keep parity with `services/scraper/app/core/config.py`.
+    return _env_bool("SCRAPER_CAPTURE_HEADLESS", _env_bool("SCRAPER_HEADLESS", False))
+
+
+def _default_scrape_headless() -> bool:
+    # Keep parity with `services/scraper/app/core/config.py`.
+    return _env_bool("SCRAPER_SCRAPE_HEADLESS", _env_bool("SCRAPER_HEADLESS", True))
+
+
+def _default_runs_dir() -> str:
+    return os.getenv("SCRAPER_RUNS_DIR", "artifacts/scraper_runs")
+
+
+def _default_runner_out_dir(sync_session_id: str) -> Path:
+    run_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
+    return Path(_default_runs_dir()) / run_date / sync_session_id
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Music League Eras local runner (vendored entrypoint).")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    local_sync_parser = subparsers.add_parser(
+        "local-sync",
+        help="Run local login + scrape, then upload manifest to the API using a sync token.",
+    )
+    local_sync_parser.add_argument("--api-base-url", required=True, help="API base URL (e.g. http://localhost:8000)")
+    local_sync_parser.add_argument("--sync-session-id", required=True, help="Local sync session id")
+    local_sync_parser.add_argument("--sync-token", required=True, help="Local sync token (secret)")
+    local_sync_parser.add_argument(
+        "--music-league-base-url",
+        default=_default_music_league_base_url(),
+        help="Music League base URL",
+    )
+    local_sync_parser.add_argument(
+        "--capture-headless",
+        default=_default_capture_headless(),
+        action=argparse.BooleanOptionalAction,
+        help="Run the login capture headless",
+    )
+    local_sync_parser.add_argument(
+        "--scrape-headless",
+        default=_default_scrape_headless(),
+        action=argparse.BooleanOptionalAction,
+        help="Run the scrape headless",
+    )
+    local_sync_parser.add_argument(
+        "--out-dir",
+        type=Path,
+        default=None,
+        help="Output directory for artifacts (defaults under SCRAPER_RUNS_DIR)",
+    )
+    local_sync_parser.add_argument(
+        "--timeout-s",
+        type=int,
+        default=60,
+        help="HTTP timeout seconds for upload (default: 60)",
+    )
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command != "local-sync":
+        raise SystemExit(f"Unknown command: {args.command}")
+
+    out_dir = args.out_dir or _default_runner_out_dir(args.sync_session_id)
+    result = asyncio.run(
+        run_local_sync(
+            api_base_url=args.api_base_url,
+            sync_session_id=args.sync_session_id,
+            sync_token=args.sync_token,
+            music_league_base_url=args.music_league_base_url,
+            capture_headless=args.capture_headless,
+            scrape_headless=args.scrape_headless,
+            out_dir=out_dir,
+            timeout_s=args.timeout_s,
+        )
+    )
+    print(f"Manifest: {result.manifest_path}")
+    print(f"Upload: {result.upload_response}")
+
+
+if __name__ == "__main__":
+    main()
+

package/vendor/python/app/local_sync_runner.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .browser.session import capture_storage_state
+from .services.scrape_manifest import build_manifest
+
+
+@dataclass(frozen=True)
+class LocalSyncRunnerResult:
+    manifest_path: str
+    upload_response: dict[str, Any]
+
+
+def _now_iso() -> str:
+    return datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
+
+
+def _post_json(*, url: str, headers: dict[str, str], payload: dict[str, Any], timeout_s: int) -> dict[str, Any]:
+    data = json.dumps(payload).encode("utf-8")
+    request = urllib.request.Request(url, method="POST", data=data)
+    request.add_header("content-type", "application/json")
+    request.add_header("accept", "application/json")
+    for key, value in headers.items():
+        request.add_header(key, value)
+    try:
+        with urllib.request.urlopen(request, timeout=timeout_s) as response:  # noqa: S310 - URL is user-provided
+            body = response.read().decode("utf-8", errors="replace")
+            if not body:
+                return {}
+            parsed = json.loads(body)
+            if isinstance(parsed, dict):
+                return parsed
+            return {"_": parsed}
+    except urllib.error.HTTPError as exc:
+        try:
+            body = exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            body = ""
+        message = body or (exc.reason if isinstance(exc.reason, str) else "HTTP error")
+        raise RuntimeError(f"Upload failed ({exc.code}): {message}") from exc
+    except urllib.error.URLError as exc:
+        raise RuntimeError(f"Upload failed (network): {exc.reason}") from exc
+
+
+async def run_local_sync(
+    *,
+    api_base_url: str,
+    sync_session_id: str,
+    sync_token: str,
+    music_league_base_url: str,
+    capture_headless: bool,
+    scrape_headless: bool,
+    out_dir: Path,
+    timeout_s: int,
+) -> LocalSyncRunnerResult:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    storage_state_path = str(out_dir / f"storage_state_{_now_iso()}.json")
+    screenshot_path = str(out_dir / f"login_{_now_iso()}.png")
+
+    print("Local sync: opening a browser window to capture login (close it once the command finishes).")
+    capture = await capture_storage_state(
+        base_url=music_league_base_url,
+        headless=capture_headless,
+        storage_state_path=storage_state_path,
+        screenshot_path=screenshot_path,
+    )
+
+    print("Local sync: login captured, scraping your Music League history…")
+    manifest, manifest_path = await build_manifest(
+        storage_state=capture.storage_state,
+        storage_state_path=None,
+        base_url=music_league_base_url,
+        output_dir=str(out_dir),
+        headless=scrape_headless,
+    )
+
+    print("Local sync: uploading manifest to the API…")
+    upload_url = (
+        api_base_url.rstrip("/")
+        + f"/api/profile/music-league/local-sync/sessions/{sync_session_id}/upload"
+    )
+    upload_payload = {"manifest": manifest}
+
+    upload_response = await asyncio.to_thread(
+        _post_json,
+        url=upload_url,
+        headers={"x-sync-token": sync_token},
+        payload=upload_payload,
+        timeout_s=timeout_s,
+    )
+    return LocalSyncRunnerResult(manifest_path=manifest_path, upload_response=upload_response)