luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/reddit_public.py

@@ -4,8 +4,8 @@ Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts
 (shreddit anti-bot), so this is no longer the primary free path. The keyless
 pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot
 Tier 0 attempt — a residential machine may occasionally get a 200 — before
-falling through to RSS discovery (reddit_rss.py) and shreddit comment
-enrichment (reddit_shreddit.py).
+falling through to site search discovery (reddit_search.py) and shreddit
+comment enrichment (reddit_shreddit.py).
 
 ``search_reddit_public`` is retained as a compatibility shim that delegates to
 the keyless pipeline, so existing callers (pipeline.py) need no change.
@@ -27,6 +27,8 @@ import urllib.parse
 import urllib.request
 from typing import Any, Dict, List, Optional
 
+from lib import http
+
 
 USER_AGENT = (
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
@@ -86,15 +88,12 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
 
         except urllib.error.HTTPError as e:
             if e.code == 429:
-                delay = BASE_BACKOFF * (2 ** attempt)
-                retry_after = None
-                if hasattr(e, "headers"):
-                    retry_after = e.headers.get("Retry-After")
-                if retry_after:
-                    try:
-                        delay = float(retry_after)
-                    except ValueError:
-                        pass
+                # Reddit answers an anonymous 429 with x-ratelimit-reset and no
+                # Retry-After; honour either. See http.retry_delay_from_headers.
+                delay = http.retry_delay_from_headers(
+                    getattr(e, "headers", None),
+                    BASE_BACKOFF * (2 ** attempt),
+                )
                 _log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s")
                 if attempt < MAX_RETRIES - 1:
                     time.sleep(delay)
@@ -248,8 +247,8 @@ def search_reddit_public(
 
     Thin compatibility shim over the keyless pipeline: the legacy ``.json``
     search/enrichment endpoints now return HTTP 403, so this delegates to
-    ``reddit_keyless.search_and_enrich`` (dedicated-sub listings + RSS discovery
-    → shreddit comment enrichment; no ``.json`` search). The name and signature
+    ``reddit_keyless.search_and_enrich`` (dedicated-sub listings + site search
+    discovery → shreddit comment enrichment; no ``.json`` search). The name and signature
     are preserved so ``pipeline.py`` and other callers need no change and the
     ScrapeCreators backup still engages when this returns empty.
 

package/skills/trends/scripts/lib/reddit_search.py

@@ -0,0 +1,304 @@
+"""Keyless Reddit discovery via Reddit's own site search fragment.
+
+``/svc/shreddit/search/?q=...&type=posts`` is the server-rendered HTML the
+reddit.com search page loads. It serves HTTP 200 with no API key, and
+``/svc/shreddit/r/{sub}/search/`` restricts the same search to one subreddit.
+Each post result unit carries:
+
+- a ``<search-telemetry-tracker>`` whose JSON tracking context has
+  ``action_info.type == "post"``, the post id, title, and subreddit,
+- the post-title link (the canonical permalink),
+- a ``<faceplate-timeago ts=...>`` post date,
+- a counter row of ``<faceplate-number>`` votes and comments.
+
+So every discovered post arrives dated and scored, with no extra listing fetch.
+The next page is a lazy ``<faceplate-partial>`` whose ``src`` carries a
+``cursor``. Output dicts match ``reddit_listing.parse_cards`` so downstream
+code is unaffected.
+"""
+
+import html as _html
+import json
+import re
+import sys
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import parse_qs, quote, urlencode, urlsplit
+
+from . import http, reddit_listing
+from .relevance import token_overlap_relevance
+
+BASE = "https://www.reddit.com"
+
+# Post caps per run by depth.
+DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50}
+# Global search pages per depth (about 7 posts per page).
+GLOBAL_PAGE_CAPS = {"quick": 2, "default": 4, "deep": 8}
+# Targeted subreddits get one page each.
+SUB_PAGE_CAP = 1
+MAX_WORKERS = 4
+SEARCH_TIMEOUT = 15
+
+# A recognizable page has post result units or Reddit's explicit no-results
+# unit. Anything else (challenge page, changed markup) is a lane failure.
+RESULTS_MARKER = 'view-events="search/view/post"'
+NO_RESULTS_MARKER = 'view-events="search/view/no_results"'
+
+_TRACKER = re.compile(
+    r'<search-telemetry-tracker\b[^>]*?\bdata-faceplate-tracking-context="([^"]*)"'
+)
+_TITLE_HREF = re.compile(
+    r'<a\b[^>]*?data-testid="post-title"[^>]*?\bhref="([^"]*)"'
+    r'|<a\b[^>]*?\bhref="([^"]*)"[^>]*?data-testid="post-title"'
+)
+_TIMEAGO = re.compile(r'<faceplate-timeago\b[^>]*?\bts="([^"]+)"')
+_COUNTER = re.compile(
+    r'<faceplate-number\b[^>]*?\bnumber="(\d+)"[^>]*>\s*</faceplate-number>\s*(vote|comment)'
+)
+_NEXT_PAGE = re.compile(
+    r'<faceplate-partial\b[^>]*?\bsrc="([^"]*/svc/shreddit/[^"]*search/[^"]*)"'
+)
+
+
+def _log(msg: str) -> None:
+    sys.stderr.write(f"[RedditSearch] {msg}\n")
+    sys.stderr.flush()
+
+
+def unrecognized_body(text: str) -> Optional[str]:
+    """Validator for the fetch hook: None for a search page, else a reason."""
+    if NO_RESULTS_MARKER in text:
+        return None
+    if RESULTS_MARKER in text:
+        # Marker kept but tracking context changed shape: a clean zero-result
+        # parse would hide the drift, so treat it as a lane failure.
+        if _post_units(text):
+            return None
+        return "Reddit search result units no longer parse"
+    return "not a Reddit search results page"
+
+
+def search_url(
+    query: str,
+    time_filter: str = "month",
+    subreddit: Optional[str] = None,
+    cursor: Optional[str] = None,
+) -> str:
+    """Build a deterministic search URL so concurrent streams share the memo."""
+    if subreddit:
+        sub = subreddit.strip().removeprefix("r/").strip("/")
+        path = f"/svc/shreddit/r/{quote(sub, safe='')}/search/"
+    else:
+        path = "/svc/shreddit/search/"
+    params = [("q", query), ("type", "posts"), ("t", time_filter), ("sort", "relevance")]
+    if cursor:
+        params.append(("cursor", cursor))
+    return f"{BASE}{path}?{urlencode(params)}"
+
+
+def _post_units(html_text: str) -> List[Tuple[int, Dict[str, Any]]]:
+    """(offset, tracking context) for each post-title tracker, in page order."""
+    units = []
+    for m in _TRACKER.finditer(html_text):
+        try:
+            ctx = json.loads(_html.unescape(m.group(1)))
+        except (ValueError, TypeError):
+            continue
+        if not isinstance(ctx, dict):
+            continue
+        action = ctx.get("action_info")
+        post = ctx.get("post")
+        # A field that is no longer an object is drift: skip the unit, and a
+        # page left with none is flagged by unrecognized_body.
+        if not isinstance(action, dict) or not isinstance(post, dict):
+            continue
+        if action.get("type") == "post" and post.get("id"):
+            units.append((m.start(), ctx))
+    return units
+
+
+def _next_cursor(html_text: str) -> Optional[str]:
+    for m in _NEXT_PAGE.finditer(html_text):
+        query = urlsplit(_html.unescape(m.group(1))).query
+        cursor = (parse_qs(query).get("cursor") or [None])[0]
+        if cursor:
+            return cursor
+    return None
+
+
+def parse_page(html_text: str, query: str = "") -> Tuple[List[Dict[str, Any]], Optional[str]]:
+    """Parse one search fragment into (normalized posts, next-page cursor)."""
+    html_text = html_text or ""
+    units = _post_units(html_text)
+    posts: List[Dict[str, Any]] = []
+    seen: set = set()
+    for i, (start, ctx) in enumerate(units):
+        post_id = str(ctx["post"]["id"]).removeprefix("t3_")
+        if post_id in seen:
+            continue
+        seen.add(post_id)
+        end = units[i + 1][0] if i + 1 < len(units) else len(html_text)
+        chunk = html_text[start:end]
+
+        sub_ctx = ctx.get("subreddit")
+        subreddit = str(sub_ctx.get("name") or "") if isinstance(sub_ctx, dict) else ""
+        title = str(ctx["post"].get("title") or "")
+        href_match = _TITLE_HREF.search(chunk)
+        href = _html.unescape((href_match.group(1) or href_match.group(2))) if href_match else ""
+        href = href.split("?", 1)[0]
+        if f"/comments/{post_id}/" in href and href.startswith("/r/"):
+            url = f"{BASE}{href}"
+        else:
+            url = f"{BASE}/r/{subreddit}/comments/{post_id}/"
+
+        ts_match = _TIMEAGO.search(chunk)
+        ts = ts_match.group(1) if ts_match else None
+        # First counter of each kind wins: this post's own counters precede any
+        # that belong to a skipped (malformed) neighbour sharing the chunk.
+        counts: Dict[str, int] = {}
+        for number, label in _COUNTER.findall(chunk):
+            counts.setdefault(label, int(number))
+        score, num_comments = counts.get("vote", 0), counts.get("comment", 0)
+
+        posts.append({
+            "id": "",
+            "title": title,
+            "url": url,
+            "score": score,
+            "num_comments": num_comments,
+            "subreddit": subreddit,
+            "created_utc": reddit_listing._to_epoch(ts),
+            "author": "",
+            "selftext": "",
+            "date": reddit_listing._to_date(ts),
+            "engagement": {
+                "score": score,
+                "num_comments": num_comments,
+                "upvote_ratio": None,
+            },
+            "relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0,
+            "why_relevant": "Reddit search",
+            "metadata": {"post_id": post_id},
+        })
+    return posts, _next_cursor(html_text)
+
+
+def _fetch_page(url: str) -> Tuple[Optional[str], Optional[str]]:
+    """(body, error). Shared limiter, run memo, one 429 retry, failure sink."""
+    try:
+        return http.reddit_keyless_get_text_retry_429(
+            url,
+            timeout=SEARCH_TIMEOUT,
+            accept="text/html",
+            validate=unrecognized_body,
+        )
+    except Exception as e:  # defensive: one bad page must not sink the run
+        _log(f"search fetch failed for {url}: {e}")
+        return None, str(e)
+
+
+def _search_stream(
+    query: str,
+    time_filter: str,
+    max_pages: int,
+    subreddit: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """Page one search sequentially. Keeps earlier pages when a later one fails."""
+    label = f"r/{subreddit}" if subreddit else "global"
+    posts: List[Dict[str, Any]] = []
+    seen_ids: set = set()
+    used_cursors: set = set()
+    cursor: Optional[str] = None
+    for page in range(1, max_pages + 1):
+        text, error = _fetch_page(search_url(query, time_filter, subreddit, cursor))
+        if text is None:
+            _log(f"{label} page {page} failed: {error or 'no response'}")
+            break
+        page_posts, next_cursor = parse_page(text, query)
+        new = [p for p in page_posts if p["metadata"]["post_id"] not in seen_ids]
+        for p in new:
+            seen_ids.add(p["metadata"]["post_id"])
+        posts.extend(new)
+        if not new or not next_cursor or next_cursor in used_cursors:
+            break
+        used_cursors.add(next_cursor)
+        cursor = next_cursor
+    return posts
+
+
+def _result_timeout(batch_size: int, pages: int = 1) -> float:
+    """Per-future wait: each sequential page's own timeout plus the bucket's queue."""
+    return pages * (SEARCH_TIMEOUT + 5) + http.reddit_keyless_wait_allowance(batch_size)
+
+
+def _time_filter(from_date: Optional[str], to_date: Optional[str]) -> str:
+    # Same mapping as the ScrapeCreators path; falls back to "month".
+    from .reddit import _window_to_time_filter
+    return _window_to_time_filter(from_date or "", to_date or "")
+
+
+def search(
+    query: str,
+    depth: str = "default",
+    subreddits: Optional[List[str]] = None,
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """Discover Reddit posts for a query via the site search fragment.
+
+    Runs a paged global search plus one page of per-subreddit search for each
+    targeted subreddit. Returns normalized, dated, scored posts, deduped by URL
+    and capped by depth. Streams are interleaved round-robin before the cap so
+    neither the targeted subreddits nor the global search crowds out the other. Returns ``[]`` on total failure and never raises; failures
+    land in the pipeline's failure sink.
+    """
+    try:
+        limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
+        global_pages = GLOBAL_PAGE_CAPS.get(depth, GLOBAL_PAGE_CAPS["default"])
+        time_filter = _time_filter(from_date, to_date)
+        subs = []
+        for raw in subreddits or []:
+            sub = (raw or "").strip().removeprefix("r/").strip("/")
+            if sub and sub.lower() not in {s.lower() for s in subs}:
+                subs.append(sub)
+
+        jobs = [(sub, SUB_PAGE_CAP) for sub in subs] + [(None, global_pages)]
+        batch = sum(pages for _sub, pages in jobs)
+        streams: List[List[Dict[str, Any]]] = []
+        with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs))) as executor:
+            # submit_with_context, not executor.submit: a plain submit starts the
+            # worker with an empty context, dropping the pipeline's
+            # capture_failures() sink so a 429/403 or a challenge page is
+            # silently discarded (issue #899).
+            futures = [
+                (http.submit_with_context(executor, _search_stream, query, time_filter, pages, sub), sub, pages)
+                for sub, pages in jobs
+            ]
+            for future, sub, pages in futures:
+                try:
+                    streams.append(future.result(timeout=_result_timeout(batch, pages)))
+                except (Exception, FuturesTimeoutError) as e:
+                    _log(f"{'r/' + sub if sub else 'global'} search future failed: {e}")
+
+        # Round-robin across streams, each in its own relevance order, so the
+        # depth cap keeps a share of every stream.
+        results = [
+            stream[i]
+            for i in range(max((len(s) for s in streams), default=0))
+            for stream in streams
+            if i < len(stream)
+        ]
+        seen: set = set()
+        unique: List[Dict[str, Any]] = []
+        for post in results:
+            if post["url"] not in seen:
+                seen.add(post["url"])
+                unique.append(post)
+        unique = unique[:limit]
+        for i, post in enumerate(unique):
+            post["id"] = f"R{i + 1}"
+        _log(f"{len(unique)} posts (t={time_filter}, {len(subs)} targeted subs)")
+        return unique
+    except Exception as e:  # defensive: discovery must never raise into the pipeline
+        _log(f"search failed: {e}")
+        return []

package/skills/trends/scripts/lib/reddit_shreddit.py

@@ -23,18 +23,47 @@ from typing import Any, Dict, List, Optional
 from . import http
 from . import reddit_enrich
 
-# Up to N posts enriched per run, by depth (mirrors reddit_public.ENRICH_LIMITS).
+# Up to N posts enriched per subquery, by depth. Raised from 3/5/8 once the
+# per-command memo (http.reddit_keyless_get_text) collapsed repeat shreddit
+# fetches across subqueries and the 1 req/s bucket stopped the 429s: eight
+# fetches fit inside ENRICH_BUDGET at four workers, and the comments are the
+# lane's headline value.
 ENRICH_LIMITS = {
-    "quick": 3,
-    "default": 5,
-    "deep": 8,
+    "quick": 4,
+    "default": 8,
+    "deep": 12,
 }
 
 # Max comments returned per post (independent of how many posts get enriched).
-MAX_COMMENTS = 10
+# Twelve so a thousand-comment thread feeds more than ten candidates into the
+# cross-platform Top Community Comments block.
+MAX_COMMENTS = 12
 
 SVC_TIMEOUT = 12
 
+# Known bots whose comments carry no community signal.
+BOT_AUTHORS = frozenset({
+    "automoderator",
+    "remindmebot",
+    "repostsleuthbot",
+    "sneakpeekbot",
+    "savevideo",
+    "videodownloadbot",
+    "totesmessenger",
+    "b0trank",
+    "amputatorbot",
+    "stabbot",
+    "gifreversingbot",
+    "haikubotinaction",
+    "imagesofnetwork",
+    "botdefense",
+})
+
+_BOT_SUFFIXES = ("-bot", "_bot")
+
+# CamelCase catches WikiTextBot without swallowing names such as Talbot.
+_CAMEL_BOT = re.compile(r"[a-z0-9]Bot\d*$")
+
 # Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree>
 # or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>').
 _COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>")
@@ -106,13 +135,28 @@ def _body_for(html_text: str, thing_id: str) -> str:
     return _WS.sub(" ", _html.unescape(text)).strip()
 
 
+def _is_bot_author(author: str) -> bool:
+    """Whether an author is a bot whose comments carry no community signal."""
+    raw = (author or "").strip()
+    if not raw:
+        return False
+    name = raw.lower()
+    return (name in BOT_AUTHORS
+            or name.endswith(_BOT_SUFFIXES)
+            or bool(_CAMEL_BOT.search(raw)))
+
+
 def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]:
-    """Parse <shreddit-comment> elements into scored comment dicts (sorted desc)."""
+    """Parse <shreddit-comment> elements into scored comment dicts (sorted desc).
+
+    Deleted, removed, and bot authors are dropped: they occupy top-comment slots
+    on high-traffic threads without saying anything about the topic.
+    """
     comments: List[Dict[str, Any]] = []
     for m in _COMMENT_START.finditer(html_text or ""):
         tag = m.group(0)
         author = _attr(tag, "author") or "[deleted]"
-        if author in ("[deleted]", "[removed]"):
+        if author in ("[deleted]", "[removed]") or _is_bot_author(author):
             continue
         thing_id = _attr(tag, "thingId")
         body = _body_for(html_text, thing_id)

package/skills/trends/scripts/lib/registers.py

@@ -0,0 +1,134 @@
+"""Named audience registers for standard research brief synthesis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Mapping
+
+
+SectionName = str
+
+
+@dataclass(frozen=True)
+class AudienceRegister:
+    """A bounded renderer/synthesis preset for one intended audience."""
+
+    name: str
+    section_order: tuple[SectionName, ...]
+    item_budgets: Mapping[SectionName, int]
+    emphasis_weights: Mapping[str, float]
+
+    def budget_for(self, section: SectionName, fallback: int) -> int:
+        return self.item_budgets.get(section, fallback)
+
+    def emphasis_for(self, source: str) -> float:
+        return self.emphasis_weights.get(source, 1.0)
+
+
+_DEFAULT_ORDER = (
+    "hiring_signals",
+    "clusters",
+    "stats",
+    "best_takes",
+    "top_comments",
+    "source_outcomes",
+    "source_coverage",
+)
+
+
+def _preset(
+    name: str,
+    *,
+    section_order: tuple[SectionName, ...] = _DEFAULT_ORDER,
+    item_budgets: Mapping[SectionName, int] | None = None,
+    emphasis_weights: Mapping[str, float] | None = None,
+) -> AudienceRegister:
+    return AudienceRegister(
+        name=name,
+        section_order=section_order,
+        item_budgets=MappingProxyType(dict(item_budgets or {})),
+        emphasis_weights=MappingProxyType(dict(emphasis_weights or {})),
+    )
+
+
+_REGISTERS = {
+    "default": _preset("default"),
+    "exec": _preset(
+        "exec",
+        section_order=(
+            "stats",
+            "clusters",
+            "hiring_signals",
+            "source_outcomes",
+            "source_coverage",
+            "best_takes",
+            "top_comments",
+        ),
+        item_budgets={"clusters": 5, "best_takes": 2, "top_comments": 3},
+        emphasis_weights={
+            "polymarket": 1.50,
+            "jobs": 1.30,
+            "github": 1.20,
+            "grounding": 1.10,
+        },
+    ),
+    "dev": _preset(
+        "dev",
+        section_order=(
+            "clusters",
+            "source_outcomes",
+            "source_coverage",
+            "hiring_signals",
+            "stats",
+            "top_comments",
+            "best_takes",
+        ),
+        item_budgets={"clusters": 10, "best_takes": 3, "top_comments": 4},
+        emphasis_weights={
+            "github": 1.60,
+            "hackernews": 1.35,
+            "arxiv": 1.30,
+            "devto": 1.25,
+            "grounding": 1.10,
+        },
+    ),
+    "creator": _preset(
+        "creator",
+        section_order=(
+            "best_takes",
+            "top_comments",
+            "stats",
+            "clusters",
+            "hiring_signals",
+            "source_outcomes",
+            "source_coverage",
+        ),
+        item_budgets={"clusters": 6, "best_takes": 5, "top_comments": 8},
+        emphasis_weights={
+            "tiktok": 1.60,
+            "instagram": 1.50,
+            "youtube": 1.40,
+            "x": 1.20,
+            "reddit": 1.10,
+        },
+    ),
+    # ELI5 historically changed only the agent's prose. Keep the renderer
+    # descriptor identical to default and express its voice in SKILL.md.
+    "eli5": _preset("eli5"),
+}
+
+REGISTER_NAMES = tuple(_REGISTERS)
+
+
+def get_register(name: str | None = None) -> AudienceRegister:
+    """Return a named register, rejecting unsupported/free-form templates."""
+
+    normalized = (name or "default").strip().lower()
+    try:
+        return _REGISTERS[normalized]
+    except KeyError as exc:
+        choices = ", ".join(REGISTER_NAMES)
+        raise ValueError(
+            f"unknown audience register {name!r}; choose one of: {choices}"
+        ) from exc

package/skills/trends/scripts/lib/relevance.py

@@ -59,6 +59,19 @@ SYNONYMS = {
 
 # Generic query words that should not carry relevance on their own.
 # They still help when paired with stronger entity/topic matches.
+#
+# The second group is scaffolding emitted by planner's ranking-query templates
+# ("What recent evidence from the last 30 days is most relevant to X?" and its
+# siblings). Those words are not the topic, but every one of them was being
+# counted as an informative query token, which capped achievable coverage at the
+# topic's share of the query and demoted on-topic posts. Kept here rather than
+# stripped in the planner so any caller building a similar natural-language
+# ranking query gets the same treatment.
+#
+# Domain nouns from those same templates (production, market, workflows,
+# experience, signals, ...) are deliberately absent: they can legitimately be a
+# user's topic, and demoting them globally would hurt every source.
+# tests/test_ranking_query_scaffolding.py pins that split.
 LOW_SIGNAL_QUERY_TOKENS = frozenset({
     'advice', 'animation', 'animations', 'best', 'chance', 'chances',
     'code', 'compare', 'comparison', 'differences', 'explain', 'guide',
@@ -67,6 +80,10 @@ LOW_SIGNAL_QUERY_TOKENS = frozenset({
     'prompting', 'prompts', 'rate', 'review', 'reviews', 'thoughts',
     'tip', 'tips', 'tutorial', 'tutorials', 'update', 'updates', 'use',
     'using', 'versus', 'vs', 'worth',
+    # planner ranking-query scaffolding
+    '30', 'current', 'days', 'describing', 'especially', 'evidence', 'exist',
+    'follow', 'hands', 'last', 'matter', 'most', 'new', 'people', 'real',
+    'recent', 'relevant', 'running', 'up', 'world',
 })