luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/reddit_keyless.py

@@ -6,9 +6,11 @@ on whatever was discovered:
 
   Dedicated lane  entity-home subreddits (e.g. r/Kanye) pulled in full via the
                   shreddit listing partials (top+hot+new, real scores), kept
-                  whole — floor-exempt — because the sub IS the topic.
-  RSS lane        reddit_rss breadth (incl. global keyword search) + broad-sub
-                  listing partials for real upvote scores. Relevance-floored.
+                  whole (floor-exempt) because the sub IS the topic.
+  Search lane     Reddit's site search fragment (reddit_search): global keyword
+                  search plus per-sub search for targeted subreddits. Results
+                  arrive dated and scored. Targeted runs also merge the
+                  subreddits' listing partials. Relevance-floored.
   Enrichment      shreddit comment + count enrichment (reddit_shreddit) for the
                   top-ranked posts (author + score + text + permalink).
 
@@ -22,19 +24,17 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, Dict, List, Optional
 
-from collections import Counter
-
-from . import reddit_rss, reddit_shreddit, reddit_listing, reddit_arctic
-# Scores are backfilled from popular derived subreddits, so an engagement-first
-# final sort buries on-topic RSS hits under viral off-topic posts. A relevance
-# floor + relevance-first final ranking keeps the section on-topic. Thresholds
-# are shared with the keyed path (reddit.py) via relevance.py.
+from . import http
+from . import reddit_search, reddit_shreddit, reddit_listing, reddit_arctic
+# High-upvote posts from targeted-sub listings can bury on-topic search hits
+# under an engagement-first final sort. A relevance floor + relevance-first
+# final ranking keeps the section on-topic. Thresholds are shared with the
+# keyed path (reddit.py) via relevance.py.
 from .relevance import RELEVANCE_FLOOR, MIN_ON_TOPIC
 
 ENRICH_LIMITS = reddit_shreddit.ENRICH_LIMITS
 ENRICH_BUDGET = 45  # seconds total across all enrichment threads
 MAX_ENRICH_WORKERS = 4
-MAX_DERIVED_SUBS = 5  # subreddits derived from RSS results for score backfill
 # Dedicated subreddits (the entity's home, e.g. r/Kanye for "Kanye West") are
 # wholly on-topic, so pull top+hot+new — the top-of-month listing alone misses
 # fresh threads — and keep every item (floor-exempt).
@@ -50,7 +50,7 @@ def _relevance_rank_key(post: Dict[str, Any]) -> float:
     """
     eng = post.get("engagement", {})
     total = (eng.get("score", 0) or 0) + (eng.get("num_comments", 0) or 0)
-    return (post.get("relevance") or 0.0) + min(0.25, math.log10(total + 1) / 20.0)
+    return (post.get("relevance") or 0.0) + min(0.25, math.log10(max(0, total) + 1) / 20.0)
 
 
 def _log(msg: str) -> None:
@@ -58,12 +58,6 @@ def _log(msg: str) -> None:
     sys.stderr.flush()
 
 
-def _top_subreddits(posts: List[Dict[str, Any]], limit: int = MAX_DERIVED_SUBS) -> List[str]:
-    """Most frequent subreddits across discovered posts (for score backfill)."""
-    counts = Counter(p.get("subreddit", "") for p in posts if p.get("subreddit"))
-    return [sub for sub, _ in counts.most_common(limit)]
-
-
 def _apply_scores(post: Dict[str, Any], scored: Dict[str, int]) -> None:
     post["score"] = scored["score"]
     post["num_comments"] = scored["num_comments"]
@@ -71,66 +65,107 @@ def _apply_scores(post: Dict[str, Any], scored: Dict[str, int]) -> None:
     post["engagement"]["num_comments"] = scored["num_comments"]
 
 
+def _scored_listings(
+    subreddits: List[str],
+    depth: str = "default",
+    query: str = "",
+    sorts: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
+    """Scored subreddit listings: shreddit partials, arctic-shift supplement.
+
+    The shreddit ``community-more-posts`` partials 403 from datacenter IPs
+    (and any host Reddit decides to block). Shreddit is tried first; arctic-
+    shift supplements with any posts shreddit missed. Individual sort lanes
+    can fail silently (shreddit's ``fetch_listings`` flattens results without
+    exposing per-sort status), so arctic is called for all requested subreddits
+    and merged via deduplication. This ensures fresh posts sought through
+    ``hot`` or ``new`` are recovered even when only ``top`` succeeded. Never
+    raises.
+    """
+    posts = reddit_listing.fetch_listings(subreddits, depth=depth, query=query, sorts=sorts)
+
+    # Supplement with arctic for all requested subreddits. Shreddit's per-sort
+    # success/failure is opaque, so arctic provides coverage for any failed
+    # sort lanes (e.g., hot/new failing while top succeeded). Deduplication
+    # ensures no redundant posts when shreddit fully succeeded.
+    if subreddits:
+        try:
+            arctic_posts = reddit_arctic.fetch_listings(
+                subreddits, depth=depth, query=query, sorts=sorts
+            )
+        except Exception as exc:  # the fallback must never break the pipeline
+            _log(f"arctic-shift listing supplement failed: {exc}")
+            arctic_posts = []
+        if arctic_posts:
+            # Merge and dedupe by URL — shreddit posts take priority.
+            seen = {p["url"] for p in posts}
+            added = 0
+            for p in arctic_posts:
+                if p["url"] not in seen:
+                    seen.add(p["url"])
+                    posts.append(p)
+                    added += 1
+            if added:
+                _log(f"arctic-shift supplement: {added} new posts from {len(arctic_posts)} arctic results")
+    return posts
+
+
 def _discover(
     topic: str,
     depth: str,
     subreddits: Optional[List[str]],
     dedicated_subreddits: Optional[List[str]] = None,
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
 ) -> List[Dict[str, Any]]:
     # Dedicated lane: the entity's home subs are wholly on-topic. Pull
     # top+hot+new (real scores from the listing) and mark them floor-exempt so
     # an on-topic post whose title lacks the entity name is never dropped.
     dedicated_posts: List[Dict[str, Any]] = []
     if dedicated_subreddits:
-        dedicated_posts = reddit_listing.fetch_listings(
+        dedicated_posts = _scored_listings(
             dedicated_subreddits, depth=depth, query=topic, sorts=DEDICATED_SORTS
         )
         for p in dedicated_posts:
             p["dedicated"] = True
         _log(f"Dedicated lane: {len(dedicated_posts)} posts from {dedicated_subreddits}")
 
-    # search.json is permanently 403/429 keyless (no Tier 0). Discovery is RSS
-    # breadth (incl. global keyword search) + broad-sub listing partials for
-    # real upvote scores.
-    rss_posts = reddit_rss.search_rss(topic, depth=depth, subreddits=subreddits)
+    # search.json is permanently 403/429 keyless (no Tier 0). Discovery is
+    # Reddit's site search: global, plus per-sub for targeted subreddits. The
+    # dedicated subs keep their full-listing lane above and are not searched.
+    search_posts = reddit_search.search(
+        topic, depth=depth, subreddits=subreddits, from_date=from_date, to_date=to_date
+    )
 
-    if subreddits:
-        # Targeted run: the caller chose these subreddits, so their listing cards
-        # are on-topic — include them as scored discovery AND as a score source.
-        listing_posts = reddit_listing.fetch_listings(subreddits, depth=depth, query=topic)
-        score_source = listing_posts
-    else:
-        # Bare global run: subreddits derived from noisy RSS results are NOT
-        # reliably on-topic, so their listings are used ONLY to backfill scores
-        # onto the keyword-matched RSS posts — never merged as discovery, which
-        # would flood results with high-upvote but irrelevant posts.
-        listing_posts = []
-        derived = _top_subreddits(rss_posts)
-        score_source = reddit_listing.fetch_listings(derived, depth=depth, query=topic)
+    # Targeted run: the caller chose these subreddits, so their listing cards
+    # are on-topic; include them as scored discovery. A bare run fetches no
+    # listings, so high-upvote off-topic posts cannot flood the results.
+    listing_posts = (
+        _scored_listings(subreddits, depth=depth, query=topic) if subreddits else []
+    )
     _log(
-        f"Tier 1 (RSS) {len(rss_posts)} posts; "
-        f"{'listing discovery ' + str(len(listing_posts)) if subreddits else 'score-only'}; "
-        f"{len(score_source)} scored cards"
+        f"Tier 1 (site search) {len(search_posts)} posts; "
+        f"{'listing discovery ' + str(len(listing_posts)) if subreddits else 'no listings'}"
     )
 
     # Score lookup by post id, from the scored listing cards.
     score_map: Dict[str, Dict[str, int]] = {}
-    for p in score_source:
+    for p in listing_posts:
         pid = p.get("metadata", {}).get("post_id", "")
         if pid:
             score_map[pid] = {"score": p["score"], "num_comments": p["num_comments"]}
 
-    # Merge: dedicated-sub posts first (floor-exempt), then scored broad listing
-    # posts (targeted only), then RSS breadth backfilled with real scores where
-    # the post appears in a listing. First writer wins the dedupe, so a thread
-    # in both the dedicated lane and a listing keeps its floor-exempt status.
+    # Merge: dedicated-sub posts first (floor-exempt), then scored listing posts
+    # (targeted only), then search results, taking a listing's live score where
+    # the post also appears there. First writer wins the dedupe, so a thread in
+    # both the dedicated lane and a listing keeps its floor-exempt status.
     merged: List[Dict[str, Any]] = []
     seen: set = set()
     for p in dedicated_posts + listing_posts:
         if p["url"] not in seen:
             seen.add(p["url"])
             merged.append(p)
-    for p in rss_posts:
+    for p in search_posts:
         if p["url"] in seen:
             continue
         pid = reddit_listing._post_id(p["url"])
@@ -139,9 +174,9 @@ def _discover(
         seen.add(p["url"])
         merged.append(p)
 
-    # Backfill scores for RSS-only posts (no listing card scored them) from the
-    # free arctic-shift archive. Posts already scored by a listing keep that
-    # live score; arctic only fills the gap, and is best-effort (never raises).
+    # Backfill posts still at score 0 from the free arctic-shift archive. Posts
+    # already scored by search or a listing keep that live score; arctic only
+    # fills the gap, and is best-effort (never raises).
     need = [pid for p in merged
             if not (p.get("engagement", {}).get("score"))
             for pid in [reddit_listing._post_id(p["url"])] if pid]
@@ -189,10 +224,15 @@ def _enrich(posts: List[Dict[str, Any]], depth: str) -> List[Dict[str, Any]]:
     try:
         with ThreadPoolExecutor(max_workers=min(limit, MAX_ENRICH_WORKERS)) as executor:
             futures = {
-                executor.submit(_enrich_one, post): i
+                http.submit_with_context(executor, _enrich_one, post): i
                 for i, post in enumerate(to_enrich)
             }
-            done, not_done = concurrent.futures.wait(futures, timeout=ENRICH_BUDGET)
+            # The budget covers the fetches; the allowance covers the shared
+            # bucket's queue (other lanes, other entities in compare mode).
+            done, not_done = concurrent.futures.wait(
+                futures,
+                timeout=ENRICH_BUDGET + http.reddit_keyless_wait_allowance(len(to_enrich)),
+            )
             for future in done:
                 idx = futures[future]
                 try:
@@ -210,6 +250,21 @@ def _enrich(posts: List[Dict[str, Any]], depth: str) -> List[Dict[str, Any]]:
     return enriched + rest
 
 
+def _by_comments(posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Stable-sort posts by comment count descending for enrichment slots.
+
+    Ties (equal counts, including unknown counts treated as 0) preserve the
+    incoming order, which search_and_enrich's provisional score-first sort
+    establishes. Mirrors _relevance_rank_key's `or 0` guard so a present-but-
+    None count is treated as 0 rather than raising.
+    """
+    def _comment_count(post: Dict[str, Any]) -> int:
+        eng = post.get("engagement") or {}
+        return eng.get("num_comments") or post.get("num_comments") or 0
+
+    return sorted(posts, key=_comment_count, reverse=True)
+
+
 def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     """Order posts for enrichment slots: entity-matching posts first.
 
@@ -222,9 +277,11 @@ def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, An
     post text) so slots go to posts likely to survive final ranking — keying
     on the same head token keeps the two paths from diverging. Falls back to
     token-overlap relevance when the topic yields no usable primary entity.
-    Within each tier the incoming
-    (score-first) order is preserved. Never raises; on any failure the
-    incoming order is returned unchanged.
+    Within each tier posts are ordered by comment count descending (stable:
+    equal or unknown counts preserve the incoming score-first order), so the
+    scarce slots go to the threads with the most discussion rather than to
+    near-empty threads that merely ranked higher by score. Never raises; on
+    any failure the incoming order is returned unchanged.
     """
     try:
         from . import relevance, rerank
@@ -246,7 +303,7 @@ def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, An
         misses: List[Dict[str, Any]] = []
         for post in posts:
             (matches if _matches(post) else misses).append(post)
-        return matches + misses
+        return _by_comments(matches) + _by_comments(misses)
     except Exception:
         return posts
 
@@ -275,7 +332,9 @@ def search_and_enrich(
         with top_comments/comment_insights attached on enriched posts.
         Empty list when all keyless tiers fail (so SC backup can engage).
     """
-    posts = _discover(topic, depth, subreddits, dedicated_subreddits)
+    posts = _discover(
+        topic, depth, subreddits, dedicated_subreddits, from_date=from_date, to_date=to_date
+    )
     if not posts:
         return []
 
@@ -287,7 +346,7 @@ def search_and_enrich(
 
     # Relevance floor: strip zero-overlap posts (relevance exactly 0 = no
     # title/body token match at all) when anything relevant remains, so
-    # backfilled high-upvote posts from popular subs can't bury on-topic RSS
+    # high-upvote listing posts can't bury on-topic search
     # hits. Keep all only when nothing scored above zero.
     before = len(posts)
     # Dedicated-sub posts are floor-exempt: their whole subreddit is the topic,
@@ -303,7 +362,8 @@ def search_and_enrich(
         _log(f"Relevance floor dropped {before - len(posts)} off-topic posts")
 
     # Provisional score-first order so enrichment-slot selection has a stable
-    # within-tier order to preserve.
+    # within-tier tiebreak order to preserve: within each entity tier, slots go
+    # to the most-commented threads first, and equal counts keep score order.
     posts.sort(
         key=lambda p: (
             p.get("engagement", {}).get("score", 0) or 0,
@@ -313,9 +373,10 @@ def search_and_enrich(
         reverse=True,
     )
 
-    # Enrichment slot selection is relevance-aware: entity-matching posts
-    # claim the scarce comment slots first (score order preserved within
-    # each tier).
+    # Enrichment slot selection is comment-aware within entity tiers:
+    # entity-matching posts claim the scarce comment slots first, and within
+    # each tier the most-commented threads get slots first (score order is the
+    # stable tiebreak for equal counts).
     posts = _enrich(_slot_priority(topic, posts), depth)
 
     # Final display order ranks relevance-first with a bounded engagement bonus,

package/skills/trends/scripts/lib/reddit_listing.py

@@ -3,14 +3,14 @@
 The subreddit listing partial
 ``/svc/shreddit/community-more-posts/{sort}/?name={sub}[&t={range}]`` serves
 HTTP 200 with no API key and **server-renders each post's upvote score**, which
-neither RSS nor the comments endpoint provides. Each post is a
+the comments endpoint does not provide. Each post is a
 ``<shreddit-post>`` element whose start-tag attributes carry ``score``,
 ``comment-count``, ``post-title``, ``permalink``, ``author``, ``subreddit-name``
 and ``created-timestamp``.
 
 This is the keyless source of post-level upvotes. It works for normal users on
 ordinary connections (verified), so reddit_keyless uses it both as a scored
-discovery source and to backfill scores onto RSS-discovered posts.
+discovery source and to backfill scores onto posts discovered elsewhere.
 """
 
 import html as _html
@@ -18,10 +18,16 @@ import re
 import sys
 from datetime import datetime, timezone
 from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Set
 
 from . import http
-from .relevance import token_overlap_relevance
+from .relevance import token_overlap_relevance, tokenize
+
+# Generic domain terms that are excluded from the keyword gate — matches
+# pipeline._DISCOVERY_GENERIC_DOMAIN_TERMS (duplicated to avoid circular import).
+_DISCOVERY_GENERIC_DOMAIN_TERMS: Set[str] = {
+    "ai", "artificial", "intelligence", "tech", "technology", "trending", "trend",
+}
 
 # Listing sorts pulled per subreddit, by depth.
 LISTING_SORTS = {
@@ -35,6 +41,19 @@ MAX_WORKERS = 4
 LISTING_TIMEOUT = 15
 
 _POST_CARD = re.compile(r"<shreddit-post(?=[\s>])[^>]*>")
+_BLOCK_MARKERS = (
+    "verify you are human",
+    "confirm you are a human",
+    "just a moment",
+    "attention required",
+    "cf-browser-verification",
+    "challenge-platform",
+    "/svc/shreddit/js-challenge",
+    "please wait for verification",
+    "enable javascript and reload",
+    "you've been blocked",
+    "whoa there",
+)
 
 
 def _log(msg: str) -> None:
@@ -42,6 +61,26 @@ def _log(msg: str) -> None:
     sys.stderr.flush()
 
 
+def _matches_discovery_domain(domain: str, text: str) -> bool:
+    """Require a distinctive domain term, not a generic token such as ``AI``.
+
+    Duplicated from pipeline._matches_discovery_domain to avoid circular imports.
+    The rule must stay in sync: pipeline.py owns the authoritative version and
+    test_reddit_listing.py verifies parity.
+    """
+    def terms(value: str) -> Set[str]:
+        words: Set[str] = set()
+        for word in tokenize(value):
+            words.add(word)
+            if len(word) > 4 and word.endswith("s") and not word.endswith("ss"):
+                words.add(word[:-1])
+        return words
+
+    domain_terms = terms(domain)
+    anchors = domain_terms - _DISCOVERY_GENERIC_DOMAIN_TERMS
+    return bool((anchors or domain_terms) & terms(text))
+
+
 def _attr(tag: str, name: str) -> Optional[str]:
     m = re.search(rf'\b{name}="([^"]*)"', tag)
     return _html.unescape(m.group(1)) if m else None
@@ -73,6 +112,21 @@ def _post_id(permalink: str) -> str:
     return m.group(1) if m else ""
 
 
+_ERROR_PATTERN = re.compile(r"^r/(\S+)\s+(\S+):", re.IGNORECASE)
+
+
+def _shreddit_error_recovered(error: str, successes: Set[tuple[str, str]]) -> bool:
+    """Return True if the error's (sub, sort) pair is in the successes set.
+
+    Error format: "r/{sub} {sort}: {message}".
+    """
+    m = _ERROR_PATTERN.match(error)
+    if not m:
+        return False
+    sub, sort = m.group(1).lower(), m.group(2).lower()
+    return (sub, sort) in successes
+
+
 def parse_cards(html_text: str, query: str = "") -> List[Dict[str, Any]]:
     """Parse <shreddit-post> cards into normalized post dicts with real scores."""
     posts: List[Dict[str, Any]] = []
@@ -118,22 +172,69 @@ def parse_cards(html_text: str, query: str = "") -> List[Dict[str, Any]]:
     return posts
 
 
-def _listing_url(subreddit: str, sort: str) -> str:
+def _listing_url(subreddit: str, sort: str, timeframe: str = TIMEFRAME) -> str:
     sub = subreddit.removeprefix("r/").strip()
+    if sub.lower() == "all":
+        url = f"https://www.reddit.com/r/all/{sort}/"
+        if sort == "top":
+            url += f"?t={timeframe}"
+        return url
     url = f"https://www.reddit.com/svc/shreddit/community-more-posts/{sort}/?name={sub}"
     if sort == "top":
-        url += f"&t={TIMEFRAME}"
+        url += f"&t={timeframe}"
     return url
 
 
-def _fetch_one(subreddit: str, sort: str, query: str) -> List[Dict[str, Any]]:
+def _listing_body_problem(subreddit: str, body: str) -> Optional[str]:
+    lower = body.lower()
+    is_blocked = any(marker in lower for marker in _BLOCK_MARKERS)
+    is_all = subreddit.removeprefix("r/").strip().lower() == "all"
+    if not is_all and not is_blocked:
+        return None
+    if parse_cards(body):
+        return None
+    if is_blocked:
+        return "Reddit listing interstitial: HTTP 2xx with no post cards"
+    return "Reddit r/all listing schema drift: HTTP 2xx with no post cards"
+
+
+def _fetch_one(
+    subreddit: str,
+    sort: str,
+    query: str,
+    timeframe: str = TIMEFRAME,
+) -> List[Dict[str, Any]]:
+    items, _ = _fetch_one_with_status(subreddit, sort, query, timeframe)
+    return items
+
+
+def _fetch_one_with_status(
+    subreddit: str,
+    sort: str,
+    query: str,
+    timeframe: str = TIMEFRAME,
+) -> tuple[List[Dict[str, Any]], Optional[str]]:
     try:
-        text = http.reddit_keyless_get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,
-                                            accept="text/html")
-        return parse_cards(text, query) if text else []
+        # retry_429 records a terminal miss into the pipeline sink (issue #899)
+        # and retries a 429 once through the limiter (issue #985). Empty
+        # dedicated-subreddit fragments can be valid; r/all cannot be empty.
+        text, error = http.reddit_keyless_get_text_retry_429(
+            _listing_url(subreddit, sort, timeframe),
+            timeout=LISTING_TIMEOUT,
+            accept="text/html",
+            validate=lambda body: _listing_body_problem(subreddit, body),
+        )
+        if text is None:
+            return [], (error or "no response")
+        return parse_cards(text, query), None
     except Exception as e:
         _log(f"listing fetch failed r/{subreddit} {sort}: {e}")
-        return []
+        return [], str(e)
+
+
+def _result_timeout(batch_size: int) -> float:
+    """Per-future wait: the fetch's own timeout plus the bucket's queue depth."""
+    return LISTING_TIMEOUT + 5 + http.reddit_keyless_wait_allowance(batch_size)
 
 
 def fetch_listings(
@@ -141,6 +242,7 @@ def fetch_listings(
     depth: str = "default",
     query: str = "",
     sorts: Optional[List[str]] = None,
+    timeframe: str = TIMEFRAME,
 ) -> List[Dict[str, Any]]:
     """Fetch scored post cards across subreddits × sorts.
 
@@ -157,11 +259,13 @@ def fetch_listings(
     jobs = [(sub, sort) for sub in subreddits for sort in sorts]
     all_posts: List[Dict[str, Any]] = []
     with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor:
-        futures = {executor.submit(_fetch_one, sub, sort, query): (sub, sort)
+        # submit_with_context, not executor.submit — see the note in
+        # fetch_discovery_listings below (issue #899).
+        futures = {http.submit_with_context(executor, _fetch_one, sub, sort, query, timeframe): (sub, sort)
                    for sub, sort in jobs}
         for future in futures:
             try:
-                all_posts.extend(future.result(timeout=LISTING_TIMEOUT + 5))
+                all_posts.extend(future.result(timeout=_result_timeout(len(jobs))))
             except (Exception, FuturesTimeoutError) as e:
                 _log(f"listing future failed: {e}")
 
@@ -174,10 +278,107 @@ def fetch_listings(
     return unique
 
 
+def fetch_discovery_listings(
+    subreddits: List[str],
+    *,
+    query: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Fetch rising/top-week listings while preserving per-feed failures.
+
+    When shreddit fails and arctic-shift recovers, errors are cleared only for
+    subreddits whose posts survive the keyword gate. If query is empty (global
+    ``--discover`` with no domain), the gate is skipped and any arctic result
+    counts as recovery.
+    """
+    if not subreddits:
+        return {"items": [], "errors": []}
+    jobs = [(subreddit, sort) for subreddit in subreddits for sort in ("rising", "top")]
+    items: List[Dict[str, Any]] = []
+    errors: List[str] = []
+    # Track which (sub, sort) pairs shreddit successfully delivered posts for.
+    # Used to decide which errors to clear — Arctic can supplement but cannot
+    # "recover" a failed hot/top/new/rising lane (it's recency-only).
+    shreddit_successes: Set[tuple[str, str]] = set()
+    with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor:
+        # submit_with_context, not executor.submit: a plain submit starts the
+        # worker with an empty context, dropping the pipeline's
+        # capture_failures() sink so a listing's 429/403 is silently discarded
+        # and the source reports a clean no-results (issue #899).
+        futures = {
+            http.submit_with_context(
+                executor, _fetch_one_with_status, subreddit, sort, query, "week"
+            ): (subreddit, sort)
+            for subreddit, sort in jobs
+        }
+        for future, (subreddit, sort) in futures.items():
+            try:
+                fetched, error = future.result(timeout=_result_timeout(len(jobs)))
+            except (Exception, FuturesTimeoutError) as exc:
+                errors.append(f"r/{subreddit} {sort}: {exc}")
+                continue
+            items.extend(fetched)
+            if error:
+                errors.append(f"r/{subreddit} {sort}: {error}")
+            elif fetched:
+                # Shreddit succeeded for this (sub, sort) lane.
+                shreddit_successes.add((subreddit.lower(), sort.lower()))
+
+    seen: set[str] = set()
+    unique = []
+    for item in items:
+        if item["url"] in seen:
+            continue
+        seen.add(item["url"])
+        unique.append(item)
+
+    # Supplement with arctic-shift for all requested subreddits. Shreddit's
+    # per-sort success/failure is opaque (individual rising/top lanes can fail
+    # while others succeed), so arctic provides coverage for any failed lanes.
+    # Deduplication ensures no redundant posts when shreddit fully succeeded.
+    from . import reddit_arctic
+    arctic_items = reddit_arctic.fetch_listings(
+        subreddits, depth=depth, query=query, sorts=("rising", "top")
+    )
+    if arctic_items:
+        _log(f"discovery arctic supplement: {len(arctic_items)} posts")
+        # Apply the same keyword gate that pipeline._fetch_discovery_source
+        # uses downstream. When query is empty (global --discover), skip the
+        # gate — there's no keyword to match, and the river feed IS the signal.
+        if query:
+            arctic_items = [
+                item for item in arctic_items
+                if _matches_discovery_domain(
+                    query,
+                    f"{item.get('title') or ''} {item.get('selftext') or ''}",
+                )
+            ]
+        # Merge arctic items into unique list, deduping by URL.
+        added = 0
+        for item in arctic_items:
+            if item["url"] not in seen:
+                seen.add(item["url"])
+                unique.append(item)
+                added += 1
+        if added:
+            _log(f"discovery arctic supplement added {added} new posts")
+
+    # Clear errors only for (sub, sort) pairs where shreddit succeeded.
+    # Arctic supplements recency posts but cannot "recover" a failed hot/top/
+    # rising lane — it has no sort lanes. Errors for failed shreddit lanes are
+    # preserved even when another sort for the same subreddit succeeded.
+    if errors and shreddit_successes:
+        errors = [
+            e for e in errors
+            if not _shreddit_error_recovered(e, shreddit_successes)
+        ]
+    return {"items": unique, "errors": errors}
+
+
 def score_index(subreddits: List[str], depth: str = "default") -> Dict[str, Dict[str, int]]:
     """Build a {post_id: {score, num_comments}} map from subreddit listings.
 
-    Used to backfill real scores onto posts discovered via RSS, which carries
+    Used to backfill real scores onto posts discovered by a lane that carries
     no engagement numbers.
     """
     index: Dict[str, Dict[str, int]] = {}