luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/reddit.py

@@ -7,12 +7,16 @@ Requires SCRAPECREATORS_API_KEY in config (same key as TikTok + Instagram).
 API docs: https://scrapecreators.com/docs
 """
 
+import copy
 import math
 import re
 import sys
+import threading
 import time
+from contextlib import ExitStack
 from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_wait
+from datetime import date, datetime, timezone
 from typing import Any, Dict, List, Optional, Set
 
 def _first_of(*values, default=None):
@@ -22,7 +26,7 @@ def _first_of(*values, default=None):
             return v
     return default
 
-from . import dates, http, log
+from . import dates, health, http, log
 
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/reddit"
 
@@ -39,19 +43,16 @@ DEPTH_CONFIG = {
         "global_searches": 1,
         "subreddit_searches": 2,
         "comment_enrichments": 3,
-        "timeframe": "week",
     },
     "default": {
         "global_searches": 2,
         "subreddit_searches": 3,
         "comment_enrichments": 5,
-        "timeframe": "month",
     },
     "deep": {
         "global_searches": 3,
         "subreddit_searches": 5,
         "comment_enrichments": 8,
-        "timeframe": "month",
     },
 }
 
@@ -78,6 +79,16 @@ def _log(msg: str):
     log.source_log("Reddit", msg, tty_only=False)
 
 
+def classify_run_failure(detail: str) -> str:
+    """Map Reddit auth and anti-bot responses that do not carry HTTP status."""
+    text = detail.lower()
+    if any(marker in text for marker in ("interstitial", "blocked by reddit", "too many requests")):
+        return health.RATE_LIMITED
+    if any(marker in text for marker in ("login required", "invalid token", "expired token")):
+        return health.AUTH_FAILED
+    return http.classify_failure(message=detail)
+
+
 def _extract_core_subject(topic: str) -> str:
     """Extract core subject from verbose query.
 
@@ -253,7 +264,7 @@ def _relevance_rank_key(item: Dict[str, Any]) -> float:
     ~0) above an on-topic one (relevance >= RELEVANCE_FLOOR).
     """
     rel = item.get("relevance") or 0.0
-    eng_bonus = min(0.25, math.log10(_total_engagement(item) + 1) / 20.0)
+    eng_bonus = min(0.25, math.log10(max(0, _total_engagement(item)) + 1) / 20.0)
     return rel + eng_bonus
 
 
@@ -394,6 +405,9 @@ def _subreddit_search(
 def fetch_post_comments(
     url: str,
     token: str,
+    *,
+    deadline_monotonic: float | None = None,
+    cancel: threading.Event | None = None,
 ) -> List[Dict[str, Any]]:
     """Fetch comments for a Reddit post via ScrapeCreators.
 
@@ -411,6 +425,9 @@ def fetch_post_comments(
             params={"url": url},
             timeout=30,
             retries=2,
+            deadline_monotonic=deadline_monotonic,
+            cancel=cancel,
+            owned_get=deadline_monotonic is not None,
         )
         return data.get("comments", data.get("data", []))
     except http.HTTPError as e:
@@ -443,6 +460,51 @@ def _dedupe_posts(posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     return unique
 
 
+def _days_to_reddit_bucket(days: float) -> str:
+    """Map a day count onto the smallest Reddit rolling bucket that covers it.
+
+    Adds one day of slack so calendar windows that cross a day boundary still
+    fit inside Reddit's rolling ``t=`` buckets (a yesterday→today request needs
+    ``week``, not ``day``).
+    """
+    covered = days + 1
+    if covered <= 1:
+        return "day"
+    if covered <= 7:
+        return "week"
+    if covered <= 31:
+        return "month"
+    if covered <= 366:
+        return "year"
+    return "all"
+
+
+def _window_to_time_filter(from_date: str, to_date: str) -> str:
+    """Map a requested YYYY-MM-DD window onto Reddit's coarse `t` param.
+
+    Reddit's ``t=day|week|month`` buckets are rolling windows ending *now*, not
+    calendar spans and not anchored to ``to_date``. Coverage therefore needs:
+
+    1. Span — a yesterday→today request needs more than rolling ``t=day``.
+    2. Historical reach — a one-day request ending two weeks ago still needs a
+       bucket that reaches ``from_date``; span-alone would pick ``week`` and
+       the API would omit the entire requested range.
+
+    Take the wider of the two, independently of retrieval depth.
+    Phase 5 still trims to ``from_date``/``to_date``. Falls back to ``month``
+    if the dates don't parse.
+    """
+    try:
+        start = date.fromisoformat(from_date)
+        end = date.fromisoformat(to_date)
+    except (ValueError, TypeError):
+        return "month"
+    span_days = max(0, (end - start).days)
+    # Age of from_date relative to "today" — Reddit always anchors to now.
+    age_days = max(0, (datetime.now(timezone.utc).date() - start).days)
+    return _days_to_reddit_bucket(max(span_days, age_days))
+
+
 def search_reddit(
     topic: str,
     from_date: str,
@@ -470,7 +532,7 @@ def search_reddit(
         return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
-    timeframe = config["timeframe"]
+    timeframe = _window_to_time_filter(from_date, to_date)
     intent = infer_query_intent(topic)
 
     # === Phase 1: Query Expansion ===
@@ -487,7 +549,9 @@ def search_reddit(
         with ThreadPoolExecutor(max_workers=min(5, len(subreddits))) as executor:
             futures = {}
             for sub in subreddits:
-                futures[executor.submit(_subreddit_search, sub, core, token, "relevance", timeframe)] = sub
+                futures[http.submit_with_context(
+                    executor, _subreddit_search, sub, core, token, "relevance", timeframe,
+                )] = sub
             for future in as_completed(futures):
                 sub = futures[future]
                 sub_posts = future.result()
@@ -506,7 +570,9 @@ def search_reddit(
             # from relevant communities instead of keyword-matched noise.
             sort = "top" if intent in ("product", "comparison") else ("relevance" if i == 0 else "top")
             _log(f"Global search {i+1}/{max_global}: '{query}' (sort={sort})")
-            futures[executor.submit(_global_search, query, token, sort, timeframe)] = query
+            futures[http.submit_with_context(
+                executor, _global_search, query, token, sort, timeframe,
+            )] = query
         for future in as_completed(futures):
             query = futures[future]
             posts = future.result()
@@ -529,7 +595,9 @@ def search_reddit(
             futures = {}
             for sub in discovered_subs[:subreddit_limit]:
                 _log(f"Subreddit search: r/{sub} for '{core}'")
-                futures[executor.submit(_subreddit_search, sub, core, token, "relevance", timeframe)] = sub
+                futures[http.submit_with_context(
+                    executor, _subreddit_search, sub, core, token, "relevance", timeframe,
+                )] = sub
             for future in as_completed(futures):
                 sub = futures[future]
                 sub_posts = future.result()
@@ -608,7 +676,7 @@ def enrich_with_comments(
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     max_comments = config["comment_enrichments"]
 
-    if not items or not token or max_comments <= 0:
+    if not items or not token or max_comments <= 0 or budget_seconds <= 0:
         return items
 
     # Select the top threads by total engagement (upvotes + comment count),
@@ -619,10 +687,24 @@ def enrich_with_comments(
     _log(f"Enriching comments for {len(top_items)} posts (by total engagement)")
 
     start = time.monotonic()
+    deadline = start + budget_seconds
+    cancel = threading.Event()
+
+    def fetch(item):
+        if cancel.is_set() or time.monotonic() >= deadline:
+            return None
+        comments = fetch_post_comments(
+            item.get("url", ""), token,
+            deadline_monotonic=deadline, cancel=cancel,
+        )
+        return comments if time.monotonic() < deadline else None
 
-    with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
+    with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor, ExitStack() as cleanup:
+        cleanup.callback(cancel.set)
         futures = {
-            executor.submit(fetch_post_comments, item.get("url", ""), token): item
+            http.submit_with_context(
+                executor, fetch, item,
+            ): item
             for item in top_items
             if item.get("url")
         }
@@ -630,6 +712,8 @@ def enrich_with_comments(
         # Wait with budget instead of unbounded as_completed
         remaining = max(0, budget_seconds - (time.monotonic() - start))
         done, not_done = futures_wait(futures, timeout=remaining)
+        if not_done:
+            cancel.set()
 
         enriched_count = 0
         for future in done:
@@ -723,6 +807,87 @@ def search_and_enrich(
     return result
 
 
+# Run-scoped memo for the paid ScrapeCreators Reddit call (R9). All subquery
+# streams share the raw topic, and the thin-source retry repeats it, so without
+# this one run could pay for the same query five times. Concurrent callers for
+# one key wait on the first call; results AND failures are kept until the
+# per-command reset, so a failed backfill is never retried within the run.
+_SC_MEMO: Dict[tuple, tuple] = {}
+_SC_INFLIGHT: Dict[tuple, threading.Event] = {}
+_SC_MEMO_LOCK = threading.Lock()
+
+
+def reset_scrapecreators_memo() -> None:
+    """Forget memoized ScrapeCreators Reddit calls. Called once per command, and by tests."""
+    with _SC_MEMO_LOCK:
+        _SC_MEMO.clear()
+        _SC_INFLIGHT.clear()
+
+
+def _sc_memo_outcome(entry: tuple, replay_failures: bool = True) -> Dict[str, Any]:
+    ok, value, failures = entry
+    if replay_failures:
+        # search_and_enrich swallows ScrapeCreators HTTP errors into the
+        # caller's failure sink; a later caller must see them too, or an
+        # empty cached result reads as a clean no-results.
+        for failure in failures:
+            http._record_failure(failure)
+    if not ok:
+        raise value
+    # Each caller gets its own copy so one stream cannot mutate another's items.
+    return copy.deepcopy(value)
+
+
+def search_and_enrich_memo(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = None,
+    subreddits: List[str] | None = None,
+) -> Dict[str, Any]:
+    """:func:`search_and_enrich`, at most once per key per command.
+
+    Keyed by query, date window, depth, and sorted subreddits.
+    """
+    key = (topic, from_date, to_date, depth, tuple(sorted(subreddits or ())))
+    while True:
+        with _SC_MEMO_LOCK:
+            entry = _SC_MEMO.get(key)
+            if entry is None:
+                gate = _SC_INFLIGHT.get(key)
+                owner = gate is None
+                if owner:
+                    gate = threading.Event()
+                    _SC_INFLIGHT[key] = gate
+        if entry is not None:
+            return _sc_memo_outcome(entry)
+        if owner:
+            break
+        gate.wait()
+        # Loop: read the owner's cached outcome, or re-elect if it was
+        # interrupted before caching one (e.g. KeyboardInterrupt).
+    try:
+        with http.tee_failures() as recorded:
+            try:
+                result = search_and_enrich(
+                    topic, from_date, to_date, depth=depth, token=token,
+                    subreddits=subreddits,
+                )
+                outcome = (True, result)
+            except Exception as exc:
+                outcome = (False, exc)
+        entry = (*outcome, list(recorded))
+        with _SC_MEMO_LOCK:
+            _SC_MEMO[key] = entry
+    finally:
+        with _SC_MEMO_LOCK:
+            _SC_INFLIGHT.pop(key, None)
+        gate.set()
+    # The owner's failures already reached its own sink through tee_failures.
+    return _sc_memo_outcome(entry, replay_failures=False)
+
+
 def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
     """Parse ScrapeCreators response to item list.
 

package/skills/trends/scripts/lib/reddit_arctic.py

@@ -1,9 +1,9 @@
 """Arctic-shift score resolver — post upvote counts by id, keyless and free.
 
-``search.json`` and ``/comments/{id}.json`` are 403 keyless, and ``search.rss``
-(used for discovery) carries titles but NO score; the shreddit listing partials
-score only posts that appear in a pulled listing. For a thread found only via
-global RSS search in a broad sub, the free score comes from arctic-shift
+``search.json`` and ``/comments/{id}.json`` are 403 keyless. Site search and the
+shreddit listing partials usually carry a score, but some posts still arrive
+unscored. For a thread that reaches the merge with score 0, the free score
+comes from arctic-shift
 (https://arctic-shift.photon-reddit.com), a public Reddit archive whose
 ``/api/posts/ids`` returns the post object (score, num_comments, title) for a
 batch of base36 post ids. Scores are point-in-time snapshots — slightly stale vs
@@ -16,16 +16,32 @@ count rather than failing the Reddit source.
 
 import sys
 import time
-from typing import Dict, List
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
 
 from . import http
 
 API = "https://arctic-shift.photon-reddit.com/api/posts/ids"
+SEARCH_API = "https://arctic-shift.photon-reddit.com/api/posts/search"
 BATCH = 50          # ids per request
 TIMEOUT = 15
 MAX_BATCHES = 3     # cap total requests per run (bounds latency + rate-limit risk)
 PACE_SECONDS = 0.4  # gap between batches; arctic-shift answers 422 "slow down"
 CACHE_MAX = 4096    # hard size bound so the in-run memo can never grow unbounded
+# Listing-lane knobs. Base limits mirror reddit_listing's DEPTH_LIMITS so callers
+# get the same per-depth volume. The supplement multiplier is applied when the
+# caller requested multiple sorts (top/hot/new) — arctic-shift has no sort lanes,
+# so we fetch more posts to increase the chance of covering what the failed
+# shreddit lanes would have returned.
+#
+# KNOWN LIMITATION: Arctic-shift is recency-only (sort=desc). It has no top/hot/
+# new/rising lanes — failed shreddit sort lanes are supplemented with recent
+# posts, not lane-specific results. This is a fundamental backend constraint.
+_LISTING_DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50}
+_LISTING_SUPPLEMENT_MULTIPLIER = 2  # fetch 2x posts when supplementing multi-sort requests
+# Total deadline for listing fetches to prevent unbounded stalls when many
+# subreddits are requested and arctic is slow/unreachable.
+_LISTING_DEADLINE_SECONDS = 45  # ~3 subs at 15s timeout each
 # In-run memo: base36 id -> {score, num_comments}. Module-level so repeated
 # fetch_scores calls within one `/last30days` run (e.g. across subqueries) reuse
 # results, but capped at CACHE_MAX entries (never reached in a normal CLI run).
@@ -90,3 +106,128 @@ def fetch_scores(post_ids: List[str]) -> Dict[str, Dict[str, int]]:
                 _cache[rid] = entry
             out[rid] = entry
     return out
+
+
+def _epoch_to_date(value: Any) -> Optional[str]:
+    """Epoch seconds -> YYYY-MM-DD (UTC), or None on garbage."""
+    try:
+        return datetime.fromtimestamp(int(value), tz=timezone.utc).date().isoformat()
+    except (TypeError, ValueError, OSError):
+        return None
+
+
+def _normalize_listing_row(row: Dict[str, Any], query: str = "") -> Dict[str, Any]:
+    """Normalize an arctic-shift post row to reddit_listing.parse_cards shape.
+
+    Mirrors the shreddit card schema (title/url/score/num_comments/subreddit/
+    created_utc/author/selftext/date/engagement/relevance/metadata.post_id) so
+    reddit_keyless can consume either backend interchangeably.
+    """
+    from .relevance import token_overlap_relevance
+
+    pid = str(row.get("id") or "").removeprefix("t3_")
+    permalink = row.get("permalink") or ""
+    title = row.get("title") or ""
+    try:
+        score = int(row.get("score") or 0)
+    except (TypeError, ValueError):
+        score = 0
+    try:
+        num_comments = int(row.get("num_comments") or 0)
+    except (TypeError, ValueError):
+        num_comments = 0
+    author = row.get("author") or "[deleted]"
+    if author in ("[deleted]", "[removed]"):
+        author = "[deleted]"
+    url = f"https://www.reddit.com{permalink}" if permalink.startswith("/") else (permalink or "")
+    return {
+        "id": "",
+        "title": title,
+        "url": url,
+        "score": score,
+        "num_comments": num_comments,
+        "subreddit": row.get("subreddit") or "",
+        "created_utc": row.get("created_utc"),
+        "author": author,
+        "selftext": row.get("selftext") or "",
+        "date": _epoch_to_date(row.get("created_utc")),
+        "engagement": {"score": score, "num_comments": num_comments, "upvote_ratio": None},
+        "relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0,
+        "why_relevant": "Reddit listing (arctic-shift)",
+        "metadata": {"post_id": pid},
+    }
+
+
+def fetch_listings(
+    subreddits: List[str],
+    depth: str = "default",
+    query: str = "",
+    sorts: Optional[List[str]] = None,
+    timeframe: str = "month",
+    limit: Optional[int] = None,
+) -> List[Dict[str, Any]]:
+    """Scored subreddit listings from the arctic-shift archive, keyless.
+
+    Drop-in fallback/supplement for ``reddit_listing.fetch_listings`` (shreddit
+    partials), which datacenter IPs get HTTP 403 on. Arctic-shift serves recent
+    posts with real score/num_comments from any IP.
+
+    Arctic-shift has no top/hot/new lanes, only recency. When ``sorts`` contains
+    multiple entries (e.g., dedicated lanes requesting top+hot+new), we fetch
+    more posts per subreddit to partially compensate for the missing lane
+    coverage — the caller's engagement ranking does the final sorting.
+
+    Best-effort, never raises: returns ``[]`` on any failure.
+    """
+    if not subreddits:
+        return []
+    base = limit or _LISTING_DEPTH_LIMITS.get(depth, _LISTING_DEPTH_LIMITS["default"])
+    # When multiple sorts were requested, fetch more posts to compensate for
+    # arctic-shift's lack of sort lanes.
+    n = base * _LISTING_SUPPLEMENT_MULTIPLIER if sorts and len(sorts) > 1 else base
+    out: List[Dict[str, Any]] = []
+    # Process all requested subreddits with pacing and a total deadline to
+    # prevent unbounded stalls when arctic is slow or unreachable.
+    deadline = time.time() + _LISTING_DEADLINE_SECONDS
+    fetched_count = 0
+    for sub in subreddits:
+        if time.time() >= deadline:
+            _log(f"listing deadline reached after {fetched_count} subs; skipping remaining")
+            break
+        sub = sub.removeprefix("r/").strip()
+        if not sub or sub.lower() == "all":
+            continue
+        if fetched_count:
+            time.sleep(PACE_SECONDS)
+        fetched_count += 1
+        try:
+            # Use retries=1 (single attempt) so retries don't exceed our deadline.
+            # The deadline handles overall timing; per-request retries would
+            # multiply the delay unpredictably.
+            data = http.get(
+                f"{SEARCH_API}?subreddit={sub}&limit={n}&sort=desc",
+                headers={"User-Agent": http.BROWSER_USER_AGENT},
+                timeout=TIMEOUT,
+                retries=1,
+            )
+        except Exception as e:  # network error / non-200 — degrade, never raise
+            _log(f"listing search failed r/{sub}: {e}")
+            continue
+        rows = (data or {}).get("data")
+        if not isinstance(rows, list):
+            _log(f"unexpected listing response for r/{sub}: {str(data)[:80]}")
+            continue
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            post = _normalize_listing_row(row, query)
+            if post["url"]:
+                out.append(post)
+
+    seen: set = set()
+    unique: List[Dict[str, Any]] = []
+    for p in out:
+        if p["url"] not in seen:
+            seen.add(p["url"])
+            unique.append(p)
+    return unique