@music-league-eras/local-runner 0.1.0

package/vendor/python/app/services/scrape_manifest.py

@@ -0,0 +1,774 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import re
+import time
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+from playwright.async_api import async_playwright
+
+from .viewer import extract_viewer_user_id
+
+COMPLETED_LEAGUES_XHR = "/completed/-/completedLeagues"
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/123.0.0.0 Safari/537.36"
+)
+
+LEAGUE_HREF_RE = re.compile(r"^/l/([0-9a-f]{32})/?$", re.I)
+ROUND_NO_RE = re.compile(r"ROUND\s+(\d+)", re.IGNORECASE)
+ISO_TS_RE = re.compile(r"dateFns\.parse\('([^']+)'\)")
+USER_HREF_RE = re.compile(r"^/user/([0-9a-f]{32})/?$", re.I)
+USER_ASSET_RE = re.compile(r"/users/([0-9a-f]{32})/", re.I)
+ORDINAL_RE = re.compile(r"^\d+(st|nd|rd|th)$", re.IGNORECASE)
+
+
+def _abs_url(base_url: str, href: str) -> str:
+    return urljoin(base_url, href)
+
+
+def _ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def _write_text(path: Path, content: str) -> None:
+    _ensure_parent(path)
+    path.write_text(content, encoding="utf-8")
+
+
+def _parse_user_id_from_href(href: Optional[str]) -> Optional[str]:
+    if not href:
+        return None
+    match = USER_HREF_RE.match(href.strip())
+    return match.group(1) if match else None
+
+
+def _parse_user_id_from_img_src(src: Optional[str]) -> Optional[str]:
+    if not src:
+        return None
+    match = USER_ASSET_RE.search(src)
+    return match.group(1) if match else None
+
+
+def _parse_int(value: Optional[str]) -> Optional[int]:
+    if value is None:
+        return None
+    cleaned = re.sub(r"[^\d\-]", "", str(value).strip())
+    if cleaned in ("", "-"):
+        return None
+    try:
+        return int(cleaned)
+    except ValueError:
+        return None
+
+
+def _iso_to_date_only(ts: Optional[str]) -> Optional[str]:
+    return ts[:10] if ts and len(ts) >= 10 else None
+
+
+
+
+def _extract_completed_leagues(fragment_html: str, base_url: str) -> list[dict[str, Any]]:
+    soup = BeautifulSoup(fragment_html, "html.parser")
+    leagues: dict[str, dict[str, Any]] = {}
+
+    for tile in soup.select(".league-tile"):
+        link = tile.select_one('a[href^="/l/"]')
+        if not link:
+            continue
+        href = (link.get("href") or "").strip()
+        match = LEAGUE_HREF_RE.match(href)
+        if not match:
+            continue
+
+        league_id = match.group(1)
+        title = (link.get_text(" ", strip=True) or "").strip()
+
+        img = tile.select_one('img[alt="League image"]')
+        src = (img.get("src") or "").strip() if img else ""
+        league_image_url = None
+        if src:
+            league_image_url = _abs_url(base_url, src) if src.startswith("/") else src
+
+        leagues[league_id] = {
+            "league_id": league_id,
+            "league_href": href if href.endswith("/") else f"{href}/",
+            "league_url": _abs_url(base_url, href),
+            "league_title": title,
+            "league_image_url": league_image_url,
+        }
+
+    if leagues:
+        return list(leagues.values())
+
+    # Fallback: older tiles without `.league-tile` wrapper.
+    for link in soup.select('a[href^="/l/"]'):
+        href = link.get("href", "")
+        match = LEAGUE_HREF_RE.match(href)
+        if not match:
+            continue
+        league_id = match.group(1)
+        title = (link.get_text(" ", strip=True) or "").strip()
+        leagues[league_id] = {
+            "league_id": league_id,
+            "league_href": href if href.endswith("/") else f"{href}/",
+            "league_url": _abs_url(base_url, href),
+            "league_title": title,
+            "league_image_url": None,
+        }
+
+    return list(leagues.values())
+
+
+def _extract_completed_rounds(league_id: str, rounds_html: str, base_url: str) -> list[dict[str, Any]]:
+    soup = BeautifulSoup(rounds_html, "html.parser")
+
+    header = soup.find(
+        lambda tag: tag.name in ("h3", "h4", "h5")
+        and tag.get_text(strip=True).lower() == "completed rounds"
+    )
+    container = header.find_parent() if header else soup
+
+    rounds: list[dict[str, Any]] = []
+
+    for item in container.select(".league-round-item"):
+        round_id = (item.get("id") or "").strip()
+        if not round_id:
+            continue
+
+        round_no = None
+        span = item.find("span", string=lambda s: bool(s and "ROUND" in s.upper()))
+        if span:
+            match = ROUND_NO_RE.search(span.get_text(" ", strip=True))
+            if match:
+                round_no = int(match.group(1))
+
+        title_tag = item.select_one("h5.card-title") or item.find("h5")
+        round_title = title_tag.get_text(" ", strip=True) if title_tag else ""
+
+        desc_tag = item.select_one("p[data-description]")
+        round_description = desc_tag.get("data-description") if desc_tag else None
+
+        ts = None
+        match = ISO_TS_RE.search(str(item))
+        if match:
+            ts = match.group(1)
+        completed_date_utc = _iso_to_date_only(ts)
+
+        playlist_tag = item.select_one('a[href^="https://open.spotify.com/playlist/"]')
+        playlist_url = playlist_tag.get("href") if playlist_tag else None
+
+        results_href = None
+        for link in item.select('a[href^="/l/"]'):
+            if "RESULTS" in (link.get_text(" ", strip=True) or "").upper():
+                results_href = link.get("href")
+                break
+        results_url = _abs_url(base_url, results_href) if results_href else None
+        results_xhr_url = _abs_url(base_url, f"/l/{league_id}/{round_id}/-/results")
+
+        rounds.append(
+            {
+                "round_id": round_id,
+                "round_no": round_no,
+                "round_title": round_title,
+                "round_description": round_description,
+                "completed_date_utc": completed_date_utc,
+                "playlist_url": playlist_url,
+                "results_url": results_url,
+                "results_xhr_url": results_xhr_url,
+            }
+        )
+
+    rounds.sort(key=lambda entry: (entry["round_no"] is None, entry["round_no"] or 0))
+    return rounds
+
+
+def _parse_round_results(results_html: str) -> list[dict[str, Any]]:
+    soup = BeautifulSoup(results_html, "html.parser")
+    body = soup.body or soup
+
+    containers: list[Any] = []
+    for div in body.find_all("div", recursive=False):
+        if div.select_one('a[href^="https://open.spotify.com/track/"]'):
+            containers.append(div)
+
+    if not containers:
+        seen = set()
+        for link in soup.select('a[href^="https://open.spotify.com/track/"]'):
+            node = link
+            while node and getattr(node, "name", None) not in ("body", None):
+                if node.name == "div" and node.select_one('img[alt="Album art"]'):
+                    if id(node) not in seen:
+                        seen.add(id(node))
+                        containers.append(node)
+                    break
+                node = node.parent
+
+    submissions: list[dict[str, Any]] = []
+
+    for cont in containers:
+        album_art_img = cont.select_one('img[alt="Album art"]')
+        album_art_url = album_art_img.get("src") if album_art_img else None
+
+        album_art_link = cont.select_one('a[title="Album art"][href^="/user/"]')
+        submitter_user_id = _parse_user_id_from_href(
+            album_art_link.get("href") if album_art_link else None
+        )
+
+        track_link = cont.select_one('a[href^="https://open.spotify.com/track/"]')
+        track_url = track_link.get("href") if track_link else None
+        song_title = track_link.get_text(" ", strip=True) if track_link else None
+
+        artist = None
+        album = None
+        if track_link:
+            h6 = track_link.find_parent("h6")
+            block = h6.find_parent() if h6 else track_link.find_parent()
+            ps = block.find_all("p") if block else []
+            if len(ps) > 0:
+                artist = ps[0].get_text(" ", strip=True)
+            if len(ps) > 1:
+                album = ps[1].get_text(" ", strip=True)
+
+        score_tag = cont.find("h3")
+        score_overridden = False
+        score_override_expected = None
+        score_effective = None
+
+        if score_tag:
+            strike_tag = score_tag.find("s", class_=lambda c: bool(c and "text-danger" in c))
+            if strike_tag:
+                score_overridden = True
+                score_override_expected = _parse_int(strike_tag.get_text(" ", strip=True))
+                strike_tag.extract()
+                score_effective = _parse_int(score_tag.get_text(" ", strip=True))
+            else:
+                score_effective = _parse_int(score_tag.get_text(" ", strip=True))
+
+        voters_count = None
+        for p_tag in cont.find_all("p"):
+            if "voter" in (p_tag.get_text(" ", strip=True) or "").lower():
+                voters_count = _parse_int(p_tag.get_text(" ", strip=True))
+                break
+
+        rank_tag = cont.find("h6", string=lambda s: bool(s and ORDINAL_RE.match(s.strip())))
+        rank = rank_tag.get_text(" ", strip=True) if rank_tag else None
+
+        submitter_name = None
+        for h6 in cont.find_all("h6"):
+            if h6.find("a"):
+                continue
+            text = h6.get_text(" ", strip=True)
+            if text and not ORDINAL_RE.match(text):
+                submitter_name = text
+                break
+
+        comment_span = cont.select_one("span.text-break.ws-pre-wrap")
+        submitter_comment = comment_span.get_text(" ", strip=True) if comment_span else None
+        if submitter_comment == "":
+            submitter_comment = None
+
+        submitter_avatar_url = None
+        for img in cont.select('img[src*="musicleague-user-assets"]'):
+            src = img.get("src", "")
+            if submitter_user_id and submitter_user_id in src:
+                submitter_avatar_url = src
+                break
+
+        votes: list[dict[str, Any]] = []
+        for bold in cont.find_all("b"):
+            voter_name = bold.get_text(" ", strip=True)
+
+            node = bold
+            row = None
+            for _ in range(6):
+                node = node.parent
+                if not node:
+                    break
+                if (
+                    getattr(node, "name", None) == "div"
+                    and node.select_one('img[src*="musicleague-user-assets"]')
+                ):
+                    row = node
+                    break
+            if not row:
+                continue
+
+            voter_avatar_img = row.select_one('img[src*="musicleague-user-assets"]')
+            voter_avatar_url = voter_avatar_img.get("src") if voter_avatar_img else None
+            voter_user_id = _parse_user_id_from_img_src(voter_avatar_url)
+
+            span = row.find("span")
+            comment = span.get_text(" ", strip=True) if span else None
+            if comment == "":
+                comment = None
+
+            vote_val = None
+            for h6 in reversed(row.find_all("h6")):
+                parsed = _parse_int(h6.get_text(" ", strip=True))
+                if parsed is not None:
+                    vote_val = parsed
+                    break
+
+            if score_overridden:
+                vote_effective = None
+                is_effective = False
+            else:
+                vote_effective = vote_val
+                is_effective = True
+
+            votes.append(
+                {
+                    "voter_user_id": voter_user_id,
+                    "voter_name": voter_name,
+                    "vote": vote_val,
+                    "vote_raw": vote_val,
+                    "vote_effective": vote_effective,
+                    "is_effective": is_effective,
+                    "comment": comment,
+                    "voter_avatar_url": voter_avatar_url,
+                }
+            )
+
+        submissions.append(
+            {
+                "track_url": track_url,
+                "song_title": song_title,
+                "artist": artist,
+                "album": album,
+                "album_art_url": album_art_url,
+                "score": score_effective,
+                "score_effective": score_effective,
+                "score_overridden": score_overridden,
+                "score_override_expected": score_override_expected,
+                "voters_count": voters_count,
+                "rank": rank,
+                "submitter": {
+                    "user_id": submitter_user_id,
+                    "name": submitter_name,
+                    "avatar_url": submitter_avatar_url,
+                    "comment": submitter_comment,
+                },
+                "votes": votes,
+            }
+        )
+
+    return submissions
+
+
+async def _fetch_html(page, url: str) -> str:
+    await page.goto(url, wait_until="domcontentloaded")
+    return await page.content()
+
+
+def _cookie_matches(domain: str, host: str | None) -> bool:
+    if not domain or not host:
+        return False
+    normalized = domain.lstrip(".")
+    return host == normalized or host.endswith(f".{normalized}")
+
+
+def _has_host_cookie(storage_state: dict[str, Any] | None, host: str | None) -> bool:
+    if not storage_state or not host:
+        return False
+    for cookie in storage_state.get("cookies", []):
+        domain = str(cookie.get("domain") or "")
+        if _cookie_matches(domain, host):
+            return True
+    return False
+
+
+def _cookie_header_for_host(storage_state: dict[str, Any] | None, host: str | None) -> str | None:
+    if not storage_state or not host:
+        return None
+    cookies = []
+    for cookie in storage_state.get("cookies", []):
+        domain = str(cookie.get("domain") or "")
+        if _cookie_matches(domain, host):
+            name = cookie.get("name")
+            value = cookie.get("value")
+            if name and value is not None:
+                cookies.append(f"{name}={value}")
+    return "; ".join(cookies) if cookies else None
+
+
+def _fingerprint_storage_state(storage_state: dict[str, Any]) -> str:
+    # Stable hash for debugging (do not log/store cookies themselves in artifacts/DB).
+    payload = json.dumps(storage_state, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()[:12]
+
+
+async def _cookie_header_from_context(context, base_url: str) -> str | None:
+    try:
+        cookies = await context.cookies(base_url)
+    except Exception:
+        return None
+    if not cookies:
+        return None
+    parts: list[str] = []
+    for cookie in cookies:
+        name = cookie.get("name")
+        value = cookie.get("value")
+        if name and value is not None:
+            parts.append(f"{name}={value}")
+    return "; ".join(parts) if parts else None
+
+
+def _build_xhr_headers(referer: str | None, cookie_header: str | None) -> dict[str, str]:
+    headers = {
+        "x-requested-with": "XMLHttpRequest",
+        "HX-Request": "true",
+        "accept": "text/html, */*; q=0.01",
+        "user-agent": DEFAULT_USER_AGENT,
+    }
+    if referer:
+        headers["referer"] = referer
+        headers["HX-Current-URL"] = referer
+        headers["HX-Trigger"] = "load"
+    if cookie_header:
+        headers["cookie"] = cookie_header
+    return headers
+
+
+def _fetch_xhr_html_via_http(
+    url: str,
+    *,
+    headers: dict[str, str],
+    expected_marker: str | None = None,
+    retries: int = 2,
+    return_last_on_marker_miss: bool = False,
+) -> str:
+    last_text = ""
+    for attempt in range(retries + 1):
+        req = urllib.request.Request(url, headers=headers, method="GET")
+        try:
+            with urllib.request.urlopen(req, timeout=20) as resp:
+                text = resp.read().decode("utf-8", errors="replace")
+        except Exception:
+            text = ""
+        last_text = text or ""
+        if last_text.strip() and (expected_marker is None or expected_marker in last_text):
+            return last_text
+        if attempt < retries:
+            time.sleep(0.4 * (attempt + 1))
+    if expected_marker and expected_marker not in last_text and not return_last_on_marker_miss:
+        return ""
+    return last_text
+
+
+async def _fetch_xhr_html_in_page(
+    page,
+    url: str,
+    *,
+    referer: str | None = None,
+    expected_marker: str | None = None,
+    retries: int = 2,
+) -> str:
+    headers = {"x-requested-with": "XMLHttpRequest", "HX-Request": "true"}
+    last_text = ""
+    for attempt in range(retries + 1):
+        text = await page.evaluate(
+            """async ({ url, headers, referer }) => {
+                const res = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers,
+                    referrer: referer || undefined
+                });
+                return await res.text();
+            }""",
+            {"url": url, "headers": headers, "referer": referer},
+        )
+        last_text = text or ""
+        if last_text.strip() and (expected_marker is None or expected_marker in last_text):
+            return last_text
+        if attempt < retries:
+            await asyncio.sleep(0.4 * (attempt + 1))
+    return last_text
+
+
+async def _fetch_xhr_html(
+    request,
+    url: str,
+    *,
+    referer: str | None = None,
+    expected_marker: str | None = None,
+    retries: int = 2,
+    cookie_header: str | None = None,
+    return_last_on_marker_miss: bool = False,
+) -> str:
+    headers = _build_xhr_headers(referer, cookie_header)
+    last_text = ""
+    for attempt in range(retries + 1):
+        response = await request.get(url, headers=headers)
+        text = await response.text()
+        last_text = text or ""
+        if last_text.strip() and (expected_marker is None or expected_marker in last_text):
+            return last_text
+        if attempt < retries:
+            await asyncio.sleep(0.4 * (attempt + 1))
+    if expected_marker and expected_marker not in last_text and not return_last_on_marker_miss:
+        return ""
+    return last_text
+
+
+async def build_manifest(
+    *,
+    storage_state: dict[str, Any] | None,
+    storage_state_path: str | None,
+    base_url: str,
+    output_dir: str,
+    headless: bool,
+) -> tuple[dict[str, Any], str]:
+    out_dir = Path(output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest: dict[str, Any] = {
+        "schema_version": 2,
+        "generated_at_local": datetime.now().isoformat(timespec="seconds"),
+        "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        "base_url": base_url,
+        "out_dir": str(out_dir.resolve()),
+        "viewer": None,
+        "leagues": [],
+        "errors": [],
+    }
+
+    storage_state_arg: dict[str, Any] | str | None = None
+    storage_state_payload: dict[str, Any] | None = None
+    storage_state_source: str | None = None
+    storage_state_path_missing: str | None = None
+    if storage_state_path and Path(storage_state_path).exists():
+        storage_state_arg = storage_state_path
+        storage_state_payload = json.loads(Path(storage_state_path).read_text())
+        storage_state_source = "path"
+    elif storage_state_path:
+        storage_state_path_missing = storage_state_path
+    elif storage_state:
+        storage_state_arg = storage_state
+        storage_state_payload = storage_state
+        storage_state_source = "inline"
+
+    if not storage_state_arg:
+        raise RuntimeError("Missing storage state for session scrape.")
+
+    host = urlparse(base_url).hostname
+    if not _has_host_cookie(storage_state_payload, host):
+        raise RuntimeError(
+            "Storage state missing Music League cookies. Complete login before scraping."
+        )
+    initial_cookie_header = _cookie_header_for_host(storage_state_payload, host)
+    storage_state_ref = {
+        "source": storage_state_source,
+        "path_used": storage_state_path if storage_state_source == "path" else None,
+        "path_missing": storage_state_path_missing,
+        "fingerprint": _fingerprint_storage_state(storage_state_payload),
+        "cookies_total": len(storage_state_payload.get("cookies", [])) if storage_state_payload else 0,
+        "host_cookie_present": True,
+    }
+    manifest["storage_state_ref"] = storage_state_ref
+
+    async with async_playwright() as playwright:
+        browser = await playwright.chromium.launch(headless=headless)
+        context = await browser.new_context(storage_state=storage_state_arg)
+        page = await context.new_page()
+        try:
+            completed_page_url = _abs_url(base_url, "/completed/")
+            response = await page.goto(completed_page_url, wait_until="domcontentloaded")
+            completed_final_url = page.url
+            completed_page_html = await page.content()
+            _write_text(out_dir / "completed.page.html", completed_page_html)
+            manifest["debug"] = {
+                "completed_url": completed_page_url,
+                "completed_final_url": completed_final_url,
+                "completed_status": response.status if response else None,
+            }
+
+            cookie_header = await _cookie_header_from_context(context, base_url) or initial_cookie_header
+
+            viewer_id = extract_viewer_user_id(completed_page_html)
+            manifest["viewer"] = {"user_id": viewer_id} if viewer_id else None
+
+            completed_url = _abs_url(base_url, COMPLETED_LEAGUES_XHR)
+            completed_debug: dict[str, Any] = {
+                "url": completed_url,
+                "expected_marker": "league-tile",
+                "attempts": [],
+            }
+
+            completed_html = await asyncio.to_thread(
+                _fetch_xhr_html_via_http,
+                completed_url,
+                headers=_build_xhr_headers(_abs_url(base_url, "/completed/"), cookie_header),
+                expected_marker="league-tile",
+                return_last_on_marker_miss=True,
+            )
+            completed_marker = bool(completed_html.strip()) and ("league-tile" in completed_html)
+            completed_debug["attempts"].append(
+                {"method": "http", "bytes": len(completed_html), "marker_found": completed_marker}
+            )
+
+            if not completed_marker:
+                resp2 = await context.request.get(
+                    completed_url,
+                    headers=_build_xhr_headers(_abs_url(base_url, "/completed/"), cookie_header),
+                )
+                completed_html = await resp2.text()
+                completed_marker = bool(completed_html.strip()) and ("league-tile" in completed_html)
+                completed_debug["attempts"].append(
+                    {
+                        "method": "context_request",
+                        "status": resp2.status,
+                        "final_url": resp2.url,
+                        "bytes": len(completed_html),
+                        "marker_found": completed_marker,
+                    }
+                )
+
+            if not completed_marker:
+                completed_html = await _fetch_xhr_html_in_page(
+                    page,
+                    completed_url,
+                    referer=_abs_url(base_url, "/completed/"),
+                )
+                completed_marker = bool(completed_html.strip()) and ("league-tile" in completed_html)
+                completed_debug["attempts"].append(
+                    {"method": "in_page", "bytes": len(completed_html), "marker_found": completed_marker}
+                )
+
+            _write_text(out_dir / "completedLeagues.html", completed_html)
+            _write_text(out_dir / "scrape.debug.json", json.dumps(
+                {
+                    "storage_state_ref": storage_state_ref,
+                    "cookie_header_source": "context" if cookie_header != initial_cookie_header else "storage_state",
+                    "completed": manifest.get("debug"),
+                    "completedLeagues": completed_debug,
+                },
+                indent=2,
+            ))
+            leagues = _extract_completed_leagues(completed_html, base_url)
+            if not leagues:
+                leagues = _extract_completed_leagues(completed_page_html, base_url)
+
+            for league_meta in leagues:
+                league_id = league_meta["league_id"]
+                league_dir = out_dir / "leagues" / league_id
+                league_dir.mkdir(parents=True, exist_ok=True)
+
+                try:
+                    rounds_xhr_url = _abs_url(base_url, f"/l/{league_id}/-/rounds")
+                    rounds_html = await asyncio.to_thread(
+                        _fetch_xhr_html_via_http,
+                        rounds_xhr_url,
+                        headers=_build_xhr_headers(league_meta["league_url"], cookie_header),
+                        expected_marker="league-round-item",
+                        return_last_on_marker_miss=True,
+                    )
+                    rounds_marker = bool(rounds_html.strip()) and ("league-round-item" in rounds_html)
+                    if not rounds_marker:
+                        rounds_html = await _fetch_xhr_html(
+                            context.request,
+                            rounds_xhr_url,
+                            referer=league_meta["league_url"],
+                            expected_marker="league-round-item",
+                            cookie_header=cookie_header,
+                            return_last_on_marker_miss=True,
+                        )
+                        rounds_marker = bool(rounds_html.strip()) and ("league-round-item" in rounds_html)
+                    if not rounds_marker:
+                        rounds_html = await _fetch_xhr_html_in_page(
+                            page,
+                            rounds_xhr_url,
+                            referer=league_meta["league_url"],
+                        )
+                    _write_text(league_dir / "rounds.html", rounds_html)
+
+                    rounds = _extract_completed_rounds(league_id, rounds_html, base_url)
+
+                    for rd in rounds:
+                        round_id = rd["round_id"]
+                        round_dir = league_dir / "rounds" / round_id
+                        round_dir.mkdir(parents=True, exist_ok=True)
+
+                        results_html = await asyncio.to_thread(
+                            _fetch_xhr_html_via_http,
+                            rd["results_xhr_url"],
+                            headers=_build_xhr_headers(league_meta["league_url"], cookie_header),
+                            expected_marker="open.spotify.com/track/",
+                            return_last_on_marker_miss=True,
+                        )
+                        results_marker = bool(results_html.strip()) and (
+                            "open.spotify.com/track/" in results_html
+                        )
+                        if not results_marker:
+                            results_html = await _fetch_xhr_html(
+                                context.request,
+                                rd["results_xhr_url"],
+                                referer=league_meta["league_url"],
+                                expected_marker="open.spotify.com/track/",
+                                cookie_header=cookie_header,
+                                return_last_on_marker_miss=True,
+                            )
+                            results_marker = bool(results_html.strip()) and (
+                                "open.spotify.com/track/" in results_html
+                            )
+                        if not results_marker:
+                            results_html = await _fetch_xhr_html_in_page(
+                                page,
+                                rd["results_xhr_url"],
+                                referer=league_meta["league_url"],
+                            )
+                        _write_text(round_dir / "results.html", results_html)
+
+                        submissions = _parse_round_results(results_html)
+                        rd["submissions_total"] = len(submissions)
+                        rd["votes_total"] = sum(len(s.get("votes", [])) for s in submissions)
+                        rd["submissions"] = submissions
+
+                        _write_text(
+                            round_dir / "results.parsed.json",
+                            json.dumps({"round_id": round_id, "submissions": submissions}, indent=2),
+                        )
+
+                    dates = [
+                        rd.get("completed_date_utc")
+                        for rd in rounds
+                        if rd.get("completed_date_utc")
+                    ]
+                    league_completed_date_utc = max(dates) if dates else None
+
+                    league_obj = {
+                        **league_meta,
+                        "league_completed_date_utc": league_completed_date_utc,
+                        "rounds_total": len(rounds),
+                        "rounds": rounds,
+                    }
+
+                    _write_text(
+                        league_dir / "league.summary.json", json.dumps(league_obj, indent=2)
+                    )
+                    manifest["leagues"].append(league_obj)
+                except Exception as exc:
+                    manifest["errors"].append(
+                        {
+                            "type": type(exc).__name__,
+                            "message": str(exc),
+                            "league_id": league_id,
+                        }
+                    )
+
+        finally:
+            await context.close()
+            await browser.close()
+
+    manifest_path = out_dir / "manifest.json"
+    _write_text(manifest_path, json.dumps(manifest, indent=2))
+    return manifest, str(manifest_path.resolve())