luckiest-co 1.0.0 → 1.0.3

package/skills/luckiest-trends/scripts/lib/reddit_public.py

@@ -0,0 +1,275 @@
+"""Reddit public ``.json`` search module (demoted to keyless Tier 0).
+
+Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts
+(shreddit anti-bot), so this is no longer the primary free path. The keyless
+pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot
+Tier 0 attempt — a residential machine may occasionally get a 200 — before
+falling through to RSS discovery (reddit_rss.py) and shreddit comment
+enrichment (reddit_shreddit.py).
+
+``search_reddit_public`` is retained as a compatibility shim that delegates to
+the keyless pipeline, so existing callers (pipeline.py) need no change.
+
+Endpoints (Tier 0):
+- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
+- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
+
+Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
+network timeouts, and missing subreddits.
+"""
+
+import gzip
+import json
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+
+# Depth-aware limits for thread counts
+DEPTH_LIMITS = {
+    "quick": 10,
+    "default": 25,
+    "deep": 50,
+}
+
+MAX_RETRIES = 3
+BASE_BACKOFF = 2.0  # seconds
+
+
+def _log(msg: str):
+    """Log to stderr."""
+    sys.stderr.write(f"[RedditPublic] {msg}\n")
+    sys.stderr.flush()
+
+
+def _url_encode(text: str) -> str:
+    """URL-encode a query string."""
+    return urllib.parse.quote_plus(text)
+
+
+def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
+    """Fetch JSON from a URL with retry on 429 and error handling.
+
+    Returns parsed JSON dict, or None on unrecoverable failure.
+    """
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept-Encoding": "gzip, deflate",
+        "Connection": "keep-alive",
+    }
+    req = urllib.request.Request(url, headers=headers)
+
+    for attempt in range(MAX_RETRIES):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                content_type = resp.headers.get("Content-Type", "")
+                if "json" not in content_type and "text/html" in content_type:
+                    _log(f"Anti-bot HTML response (Content-Type: {content_type})")
+                    return None
+
+                raw = resp.read()
+                if resp.headers.get("Content-Encoding", "").lower() == "gzip":
+                    raw = gzip.decompress(raw)
+                body = raw.decode("utf-8")
+                return json.loads(body)
+
+        except urllib.error.HTTPError as e:
+            if e.code == 429:
+                delay = BASE_BACKOFF * (2 ** attempt)
+                retry_after = None
+                if hasattr(e, "headers"):
+                    retry_after = e.headers.get("Retry-After")
+                if retry_after:
+                    try:
+                        delay = float(retry_after)
+                    except ValueError:
+                        pass
+                _log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s")
+                if attempt < MAX_RETRIES - 1:
+                    time.sleep(delay)
+                    continue
+                # Last attempt exhausted
+                _log("429 retries exhausted")
+                return None
+            elif e.code == 404:
+                _log(f"404 not found: {url}")
+                return None
+            elif e.code == 403:
+                _log(f"403 forbidden: {url}")
+                return None
+            else:
+                _log(f"HTTP {e.code}: {e.reason}")
+                return None
+
+        except (urllib.error.URLError, OSError, TimeoutError) as e:
+            _log(f"Network error: {e}")
+            return None
+
+        except json.JSONDecodeError as e:
+            _log(f"JSON decode error: {e}")
+            return None
+
+    return None
+
+
+def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Parse Reddit listing JSON into normalized post dicts."""
+    if not data:
+        return []
+
+    children = data.get("data", {}).get("children", [])
+    posts = []
+
+    for child in children:
+        if child.get("kind") != "t3":
+            continue
+        post = child.get("data", {})
+        permalink = str(post.get("permalink", "")).strip()
+        if not permalink or "/comments/" not in permalink:
+            continue
+
+        score = int(post.get("score", 0) or 0)
+        num_comments = int(post.get("num_comments", 0) or 0)
+        selftext = str(post.get("selftext", ""))
+        author = str(post.get("author", "[deleted]"))
+        created_utc = post.get("created_utc")
+
+        # Parse date
+        date_str = None
+        if created_utc:
+            try:
+                from datetime import datetime, timezone
+                dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc)
+                date_str = dt.strftime("%Y-%m-%d")
+            except (ValueError, TypeError, OSError):
+                pass
+
+        posts.append({
+            "id": "",  # Will be assigned after dedup
+            "title": str(post.get("title", "")).strip(),
+            "url": f"https://www.reddit.com{permalink}",
+            "score": score,
+            "num_comments": num_comments,
+            "subreddit": str(post.get("subreddit", "")).strip(),
+            "created_utc": float(created_utc) if created_utc else None,
+            "author": author if author not in ("[deleted]", "[removed]") else "[deleted]",
+            "selftext": selftext[:500] if selftext else "",
+            # Normalized fields matching ScrapeCreators output
+            "date": date_str,
+            "engagement": {
+                "score": score,
+                "num_comments": num_comments,
+                "upvote_ratio": post.get("upvote_ratio"),
+            },
+            "relevance": _compute_relevance(score, num_comments),
+            "why_relevant": "Reddit public search",
+            "metadata": {},
+        })
+
+    return posts
+
+
+def _compute_relevance(score: int, num_comments: int) -> float:
+    """Estimate relevance from engagement signals."""
+    score_component = min(1.0, max(0.0, score / 500.0))
+    comments_component = min(1.0, max(0.0, num_comments / 200.0))
+    return round((score_component * 0.6) + (comments_component * 0.4), 3)
+
+
+def search(
+    query: str,
+    depth: str = "default",
+    subreddit: Optional[str] = None,
+    timeout: int = 15,
+) -> List[Dict[str, Any]]:
+    """Search Reddit via the public JSON endpoint.
+
+    Args:
+        query: Search query string
+        depth: 'quick', 'default', or 'deep' — controls result limit
+        subreddit: Optional subreddit name (without r/) for scoped search
+        timeout: HTTP timeout in seconds
+
+    Returns:
+        List of normalized post dicts. Empty list on any failure.
+    """
+    limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
+    encoded_query = _url_encode(query)
+
+    if subreddit:
+        sub = subreddit.removeprefix("r/").strip()
+        url = (
+            f"https://www.reddit.com/r/{sub}/search.json"
+            f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
+        )
+    else:
+        url = (
+            f"https://www.reddit.com/search.json"
+            f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1"
+        )
+
+    data = _fetch_json(url, timeout=timeout)
+    posts = _parse_posts(data)
+
+    # Dedupe by URL and assign IDs
+    seen_urls = set()
+    unique = []
+    for post in posts:
+        if post["url"] not in seen_urls:
+            seen_urls.add(post["url"])
+            unique.append(post)
+
+    for i, post in enumerate(unique):
+        post["id"] = f"R{i + 1}"
+
+    return unique[:limit]
+
+
+def search_reddit_public(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    subreddits: Optional[List[str]] = None,
+    dedicated_subreddits: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
+    """High-level free Reddit search + enrichment (keyless).
+
+    Thin compatibility shim over the keyless pipeline: the legacy ``.json``
+    search/enrichment endpoints now return HTTP 403, so this delegates to
+    ``reddit_keyless.search_and_enrich`` (dedicated-sub listings + RSS discovery
+    → shreddit comment enrichment; no ``.json`` search). The name and signature
+    are preserved so ``pipeline.py`` and other callers need no change and the
+    ScrapeCreators backup still engages when this returns empty.
+
+    The module-level ``search`` / ``_parse_posts`` helpers remain as a
+    standalone ``.json`` search utility (own test coverage), no longer wired
+    into the keyless production path.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+        subreddits: Optional list of subreddit names (without r/) for targeted search
+
+    Returns:
+        List of normalized item dicts matching ScrapeCreators output format.
+        Empty list on total failure (so SC backup can engage).
+    """
+    from . import reddit_keyless
+    return reddit_keyless.search_and_enrich(
+        topic, from_date, to_date, depth=depth, subreddits=subreddits,
+        dedicated_subreddits=dedicated_subreddits,
+    )

package/skills/luckiest-trends/scripts/lib/reddit_rss.py

@@ -0,0 +1,224 @@
+"""Keyless Reddit discovery via public RSS/Atom feeds.
+
+Reddit's ``.json`` search endpoints now return HTTP 403 (shreddit anti-bot).
+RSS feeds still serve HTTP 200 with no API key, so this module uses them for
+post discovery, replacing ``reddit_public.search`` as the free search path.
+
+Two feed families are combined and deduped:
+- search:  /search.rss?q=... and /r/{sub}/search.rss?q=...&restrict_sr=on
+- listing: /r/{sub}/{top,hot}.rss?t=month
+
+RSS entries carry no engagement score, so ``score``/``num_comments`` start at 0
+and are backfilled during shreddit enrichment (see reddit_shreddit.py). Output
+dicts match the normalized shape emitted by ``reddit_public._parse_posts`` so
+downstream code (pipeline, renderer) is unaffected.
+"""
+
+import sys
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+from urllib.parse import quote_plus
+
+from . import http
+from .relevance import token_overlap_relevance
+
+ATOM = "{http://www.w3.org/2005/Atom}"
+
+# Mirror reddit_public depth-aware limits so the two free paths behave alike.
+DEPTH_LIMITS = {
+    "quick": 10,
+    "default": 25,
+    "deep": 50,
+}
+
+# Listing sorts pulled per subreddit (in addition to search), for volume.
+LISTING_SORTS = {
+    "quick": ["top"],
+    "default": ["top", "hot"],
+    "deep": ["top", "hot", "new"],
+}
+
+MAX_WORKERS = 4
+FEED_TIMEOUT = 15
+
+
+def _log(msg: str) -> None:
+    sys.stderr.write(f"[RedditRSS] {msg}\n")
+    sys.stderr.flush()
+
+
+def _iso_to_date(value: Optional[str]) -> Optional[str]:
+    """Parse an ISO-8601 timestamp (e.g. 2026-05-20T18:48:31+00:00) to YYYY-MM-DD."""
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value.strip())
+        return dt.date().isoformat()
+    except (ValueError, TypeError):
+        return None
+
+
+def _iso_to_epoch(value: Optional[str]) -> Optional[float]:
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value.strip())
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt.timestamp()
+    except (ValueError, TypeError):
+        return None
+
+
+def _subreddit_from(category: str, url: str) -> str:
+    """Derive subreddit name from the entry category or, failing that, the URL."""
+    if category:
+        return category
+    # URL form: https://www.reddit.com/r/{sub}/comments/{id}/...
+    parts = url.split("/r/", 1)
+    if len(parts) == 2:
+        return parts[1].split("/", 1)[0]
+    return ""
+
+
+def _parse_feed(xml_text: str, query: str = "") -> List[Dict[str, Any]]:
+    """Parse an Atom feed string into normalized post dicts. Never raises."""
+    if not xml_text:
+        return []
+    try:
+        root = ET.fromstring(xml_text)
+    except ET.ParseError as e:
+        _log(f"feed parse error: {e}")
+        return []
+
+    posts: List[Dict[str, Any]] = []
+    for entry in root.iter(f"{ATOM}entry"):
+        link_el = entry.find(f"{ATOM}link")
+        url = link_el.get("href", "").strip() if link_el is not None else ""
+        if not url or "/comments/" not in url:
+            continue
+
+        title_el = entry.find(f"{ATOM}title")
+        title = (title_el.text or "").strip() if title_el is not None else ""
+
+        author = ""
+        author_el = entry.find(f"{ATOM}author/{ATOM}name")
+        if author_el is not None and author_el.text:
+            author = author_el.text.strip().removeprefix("/u/").removeprefix("u/")
+        if author in ("[deleted]", "[removed]", ""):
+            author = "[deleted]"
+
+        cat_el = entry.find(f"{ATOM}category")
+        category = cat_el.get("term", "").strip() if cat_el is not None else ""
+        subreddit = _subreddit_from(category, url)
+
+        updated_el = entry.find(f"{ATOM}updated")
+        updated = (updated_el.text or "").strip() if updated_el is not None else ""
+
+        content_el = entry.find(f"{ATOM}content")
+        selftext = ""
+        if content_el is not None and content_el.text:
+            # Strip the simplest HTML; renderer only needs an excerpt.
+            import re as _re
+            selftext = _re.sub(r"<[^>]+>", " ", content_el.text)
+            selftext = _re.sub(r"\s+", " ", selftext).strip()[:500]
+
+        relevance = round(token_overlap_relevance(query, title), 3) if query else 0.0
+
+        posts.append({
+            "id": "",  # assigned after dedup
+            "title": title,
+            "url": url,
+            "score": 0,            # backfilled by shreddit enrichment
+            "num_comments": 0,     # backfilled by shreddit enrichment
+            "subreddit": subreddit,
+            "created_utc": _iso_to_epoch(updated),
+            "author": author,
+            "selftext": selftext,
+            "date": _iso_to_date(updated),
+            "engagement": {
+                "score": 0,
+                "num_comments": 0,
+                "upvote_ratio": None,
+            },
+            "relevance": relevance,
+            "why_relevant": "Reddit RSS",
+            "metadata": {},
+        })
+
+    return posts
+
+
+def _build_urls(query: str, depth: str, subreddits: Optional[List[str]]) -> List[str]:
+    """Build the keyless RSS feed URLs to fan out across."""
+    q = quote_plus(query)
+    urls: List[str] = [
+        f"https://www.reddit.com/search.rss?q={q}&sort=relevance&t=month"
+    ]
+    for raw_sub in (subreddits or []):
+        sub = raw_sub.removeprefix("r/").strip()
+        if not sub:
+            continue
+        urls.append(
+            f"https://www.reddit.com/r/{sub}/search.rss"
+            f"?q={q}&restrict_sr=on&sort=relevance&t=month"
+        )
+        for sort in LISTING_SORTS.get(depth, LISTING_SORTS["default"]):
+            urls.append(f"https://www.reddit.com/r/{sub}/{sort}.rss?t=month")
+    return urls
+
+
+def _fetch_feed(url: str, query: str) -> List[Dict[str, Any]]:
+    """Fetch and parse one feed. Never raises."""
+    try:
+        text = http.reddit_keyless_get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")
+        return _parse_feed(text, query) if text else []
+    except Exception as e:  # defensive: a single bad feed must not sink the run
+        _log(f"feed fetch failed for {url}: {e}")
+        return []
+
+
+def search_rss(
+    query: str,
+    depth: str = "default",
+    subreddits: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
+    """Discover Reddit posts for a query via keyless RSS feeds.
+
+    Args:
+        query: Search query string
+        depth: 'quick', 'default', or 'deep' — controls result limit and feeds
+        subreddits: Optional pre-resolved subreddit names (without r/) to target
+
+    Returns:
+        List of normalized post dicts (deduped by URL, capped by depth),
+        with placeholder scores to be backfilled during enrichment.
+        Empty list on any failure.
+    """
+    limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
+    urls = _build_urls(query, depth, subreddits)
+
+    all_posts: List[Dict[str, Any]] = []
+    workers = min(MAX_WORKERS, len(urls)) or 1
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {executor.submit(_fetch_feed, url, query): url for url in urls}
+        for future in futures:
+            try:
+                all_posts.extend(future.result(timeout=FEED_TIMEOUT + 5))
+            except (Exception, FuturesTimeoutError) as e:
+                _log(f"feed future failed: {e}")
+
+    # Dedupe by URL (first occurrence wins).
+    seen: set = set()
+    unique: List[Dict[str, Any]] = []
+    for post in all_posts:
+        if post["url"] not in seen:
+            seen.add(post["url"])
+            unique.append(post)
+
+    for i, post in enumerate(unique):
+        post["id"] = f"R{i + 1}"
+
+    return unique[:limit]

package/skills/luckiest-trends/scripts/lib/reddit_shreddit.py

@@ -0,0 +1,184 @@
+"""Keyless Reddit comment enrichment via shreddit /svc endpoints.
+
+Reddit's ``{thread}.json`` endpoint now returns HTTP 403. The shreddit partial
+endpoint ``/svc/shreddit/comments/r/{sub}/t3_{id}`` still serves HTTP 200 HTML
+with no API key, embedding each comment as a ``<shreddit-comment>`` custom
+element whose start-tag attributes carry ``score`` / ``author`` / ``created`` /
+``permalink``, and whose body lives in a ``<div id="{thingId}-post-rtjson-content">``
+block. This module parses that markup into top comments, matching the
+``top_comments`` / ``comment_insights`` shape produced by ``reddit_enrich`` so
+the renderer is unaffected.
+
+Limitation: the comments endpoint carries the real comment count
+(``total-comments``) but not the post's upvote score, so post-level ``score``
+cannot be recovered keylessly here (ScrapeCreators backup still provides it).
+"""
+
+import html as _html
+import re
+import sys
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from . import http
+from . import reddit_enrich
+
+# Up to N posts enriched per run, by depth (mirrors reddit_public.ENRICH_LIMITS).
+ENRICH_LIMITS = {
+    "quick": 3,
+    "default": 5,
+    "deep": 8,
+}
+
+# Max comments returned per post (independent of how many posts get enriched).
+MAX_COMMENTS = 10
+
+SVC_TIMEOUT = 12
+
+# Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree>
+# or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>').
+_COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>")
+_TOTAL_COMMENTS = re.compile(r'total-comments="(\d+)"')
+_PARA = re.compile(r"<p[^>]*>(.*?)</p>", re.S)
+_TAG = re.compile(r"<[^>]+>")
+_WS = re.compile(r"\s+")
+_NEXT_RTJSON = re.compile(r'id="t1_[A-Za-z0-9]+-(?:comment|post)-rtjson-content"')
+
+
+def _log(msg: str) -> None:
+    sys.stderr.write(f"[RedditShreddit] {msg}\n")
+    sys.stderr.flush()
+
+
+def extract_post_ref(url: str) -> Optional[tuple]:
+    """Return (subreddit, post_id) from a Reddit thread URL, or None."""
+    m = re.search(r"/r/([^/]+)/comments/([A-Za-z0-9]+)", url or "")
+    if not m:
+        return None
+    return m.group(1), m.group(2)
+
+
+def _svc_url(subreddit: str, post_id: str) -> str:
+    # sort=top guarantees Reddit front-loads the highest-scored comments on the
+    # first page, so the true top comments are captured even on huge threads
+    # (we still re-sort by score locally as a backstop).
+    return (
+        f"https://www.reddit.com/svc/shreddit/comments/r/{subreddit}/t3_{post_id}"
+        f"?sort=top"
+    )
+
+
+def _attr(tag: str, name: str) -> str:
+    m = re.search(rf'\b{name}="([^"]*)"', tag)
+    return _html.unescape(m.group(1)) if m else ""
+
+
+def _iso_to_date(value: str) -> Optional[str]:
+    if not value:
+        return None
+    try:
+        return datetime.fromisoformat(value.strip()).date().isoformat()
+    except (ValueError, TypeError):
+        return None
+
+
+def _body_for(html_text: str, thing_id: str) -> str:
+    """Extract a comment's text body, anchored on its unique thingId.
+
+    The body div id embeds the comment's thingId, so this assigns body→comment
+    correctly even for nested replies. The slice is bounded by the next
+    comment's rtjson anchor to avoid swallowing child-comment text.
+    """
+    if not thing_id:
+        return ""
+    anchor = f'id="{thing_id}-post-rtjson-content"'
+    idx = html_text.find(anchor)
+    if idx == -1:
+        return ""
+    window = html_text[idx + len(anchor): idx + len(anchor) + 8000]
+    nxt = _NEXT_RTJSON.search(window)
+    if nxt:
+        window = window[: nxt.start()]
+    paras = _PARA.findall(window)
+    if not paras:
+        return ""
+    text = " ".join(_TAG.sub("", p) for p in paras)
+    return _WS.sub(" ", _html.unescape(text)).strip()
+
+
+def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]:
+    """Parse <shreddit-comment> elements into scored comment dicts (sorted desc)."""
+    comments: List[Dict[str, Any]] = []
+    for m in _COMMENT_START.finditer(html_text or ""):
+        tag = m.group(0)
+        author = _attr(tag, "author") or "[deleted]"
+        if author in ("[deleted]", "[removed]"):
+            continue
+        thing_id = _attr(tag, "thingId")
+        body = _body_for(html_text, thing_id)
+        if not body or body in ("[deleted]", "[removed]"):
+            continue
+        try:
+            score = int(_attr(tag, "score") or 0)
+        except ValueError:
+            score = 0
+        permalink = _attr(tag, "permalink")
+        comments.append({
+            "score": score,
+            "author": author,
+            "body": body[:300],
+            "excerpt": body[:200],
+            "permalink": permalink,
+            "date": _iso_to_date(_attr(tag, "created")),
+            "url": f"https://reddit.com{permalink}" if permalink else "",
+        })
+
+    comments.sort(key=lambda c: c.get("score", 0), reverse=True)
+    return comments[:limit]
+
+
+def _total_comments(html_text: str) -> Optional[int]:
+    m = _TOTAL_COMMENTS.search(html_text or "")
+    return int(m.group(1)) if m else None
+
+
+def fetch_comments(
+    post_url: str,
+    timeout: int = SVC_TIMEOUT,
+) -> Dict[str, Any]:
+    """Fetch and parse top comments for a Reddit post via the shreddit endpoint.
+
+    Args:
+        post_url: Reddit thread URL (…/r/{sub}/comments/{id}/…)
+        timeout: HTTP timeout in seconds
+
+    Returns:
+        Dict with 'top_comments' (list, reddit_enrich shape), 'comment_insights'
+        (list[str]), and 'num_comments' (int or None). Empty/None on any
+        failure — never raises, so the caller can fall through to SC backup.
+    """
+    ref = extract_post_ref(post_url)
+    if not ref:
+        return {"top_comments": [], "comment_insights": [], "num_comments": None}
+    sub, post_id = ref
+
+    html_text = http.reddit_keyless_get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")
+    if not html_text:
+        return {"top_comments": [], "comment_insights": [], "num_comments": None}
+
+    comments = parse_comments(html_text, limit=MAX_COMMENTS)
+    insights = reddit_enrich.extract_comment_insights(comments)
+    return {
+        "top_comments": [
+            {
+                "score": c["score"],
+                "date": c["date"],
+                "author": c["author"],
+                "excerpt": c["excerpt"],
+                "url": c["url"],
+            }
+            for c in comments
+        ],
+        "comment_insights": insights,
+        "num_comments": _total_comments(html_text),
+    }