luckiest-co 1.0.52 → 1.0.53

package/skills/trends/scripts/lib/devto.py

@@ -4,16 +4,21 @@ Uses the public Forem API at dev.to (no key):
 
 - ``/api/articles?tag=<tag>&top=<days>`` returns the most-engaged articles for
   a tag in the window. Precise, but only as good as the tag.
-- ``/api/articles/search?q=<term>&top=<days>`` is free-text search sorted by
-  recency. Single distinctive terms match well; multi-word queries match
-  loosely, so every search hit must pass the engine's token-overlap relevance
-  check before it is kept.
+- ``/api/articles/search?q=<term>&top=<days>`` is free-text search sorted
+  roughly by recency. Single distinctive terms match well; multi-word queries
+  match loosely, so every search hit must pass the engine's token-overlap
+  relevance check before it is kept.
 - ``/api/comments?a_id=<id>`` returns the comment tree for an article.
 
 The source is off by default. Host-inferred tags (``--devto-tags``) turn it
 on for a run; ``INCLUDE_SOURCES=devto`` or ``--search devto`` turn it on
 without tags, in which case only the keyword search runs. An unknown tag
 returns ``[]`` with HTTP 200, so a wrong tag stays quiet instead of failing.
+
+Both endpoints count ``top`` back from now and accept no end date, sort, or
+date-range parameter. A window that ends before today therefore pages
+through everything published since its start and filters locally, within
+``HISTORICAL_MAX_PAGES`` and ``HISTORICAL_BUDGET_SECONDS``.
 """
 
 from __future__ import annotations
@@ -48,6 +53,11 @@ SEARCH_RELEVANCE_FLOOR = 0.25
 # Total wall-clock cap per request. The socket timeout alone bounds each
 # read, so a response trickling in small chunks could otherwise run on.
 REQUEST_DEADLINE_SECONDS = 20.0
+# Forem's per_page ceiling. A busy term ("kubernetes") has ~4,000 search hits
+# over 78 days, about 2 MB and 4-5 s per full page.
+HISTORICAL_PER_PAGE = 1000
+HISTORICAL_MAX_PAGES = 10
+HISTORICAL_BUDGET_SECONDS = 45.0
 ENRICH_WAIT_SECONDS = 25.0
 
 _TAG_RE = re.compile(r"[^a-z0-9]")
@@ -76,11 +86,22 @@ def resolve_tags(tags: list[str] | None, depth: str = "default") -> list[str]:
     return resolved[: TAG_LIMITS.get(depth, TAG_LIMITS["default"])]
 
 
+def _today() -> datetime.date:
+    return datetime.datetime.now(datetime.timezone.utc).date()
+
+
+def _is_historical(to_date: str | None) -> bool:
+    return bool(to_date) and str(to_date) < _today().isoformat()
+
+
 def _window_days(from_date: str | None, to_date: str | None) -> int:
     try:
         start = datetime.date.fromisoformat(from_date or "")
-        end = datetime.date.fromisoformat(to_date or "") if to_date else datetime.date.today()
-        return max(1, (end - start).days)
+        # Forem counts top=N back from now, including for historical windows,
+        # so the reach always runs to today and _in_window enforces to_date.
+        # The extra day keeps all of from_date: a now-anchored cutoff of
+        # exactly N days lands partway through it.
+        return max(1, (_today() - start).days + 1)
     except ValueError:
         return 30
 
@@ -98,7 +119,9 @@ def clear_cache() -> None:
         _URL_LOCKS.clear()
 
 
-def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str, Any]]:
+def _get_list(
+    url: str, headers: dict[str, str] | None = None, deadline: float | None = None,
+) -> list[dict[str, Any]]:
     # Concurrent subquery streams can request the same URL; the per-URL lock
     # makes the second caller wait for the first fetch instead of repeating it.
     with _LOCKS_GUARD:
@@ -107,7 +130,10 @@ def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str,
         if url not in _RESPONSE_CACHE:
             data = http.request(
                 "GET", url, headers=headers, timeout=15, retries=2,
-                deadline_monotonic=time.monotonic() + REQUEST_DEADLINE_SECONDS,
+                deadline_monotonic=min(
+                    time.monotonic() + REQUEST_DEADLINE_SECONDS,
+                    deadline if deadline is not None else float("inf"),
+                ),
                 owned_get=True,
             )
             if not isinstance(data, list):
@@ -118,8 +144,58 @@ def _get_list(url: str, headers: dict[str, str] | None = None) -> list[dict[str,
         return [dict(row) for row in _RESPONSE_CACHE[url]]
 
 
+def _published(article: dict[str, Any]) -> str:
+    return str(article.get("published_at") or "")[:10]
+
+
+def _fetch_rows(
+    base_url: str,
+    params: dict[str, Any],
+    to_date: str | None,
+    *,
+    label: str,
+    deadline: float,
+    headers: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
+    """Fetch one page for a current window, or page through a past one.
+
+    Every page after the first stops at ``deadline``, which the caller shares
+    across all endpoints so a deep run's tags cannot each spend the budget.
+    """
+    if not _is_historical(to_date):
+        return _get_list(f"{base_url}?{urllib.parse.urlencode(params)}", headers=headers)
+    rows: list[dict[str, Any]] = []
+    for page in range(1, HISTORICAL_MAX_PAGES + 1):
+        if page > 1 and time.monotonic() >= deadline:
+            _log(f"{label}: stopped after {page - 1} pages at the time budget; older in-window articles may be missing")
+            return rows
+        page_params = {**params, "per_page": HISTORICAL_PER_PAGE, "page": page}
+        url = f"{base_url}?{urllib.parse.urlencode(page_params)}"
+        try:
+            batch = _get_list(url, headers=headers, deadline=deadline if page > 1 else None)
+        except Exception as exc:
+            if page == 1:
+                raise
+            _log(f"{label}: page {page} failed ({exc}); keeping the {page - 1} pages fetched")
+            return rows
+        rows.extend(batch)
+        # top=N bounds the result set at from_date, so a short page is the end.
+        if len(batch) < HISTORICAL_PER_PAGE:
+            return rows
+    _log(f"{label}: stopped at {HISTORICAL_MAX_PAGES} pages; older in-window articles may be missing")
+    return rows
+
+
+def _top_engaged(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
+    return sorted(
+        rows,
+        key=lambda row: int(row.get("public_reactions_count") or row.get("positive_reactions_count") or 0),
+        reverse=True,
+    )[:limit]
+
+
 def _in_window(article: dict[str, Any], from_date: str | None, to_date: str | None) -> bool:
-    published = str(article.get("published_at") or "")[:10]
+    published = _published(article)
     if not published:
         return True
     if from_date and published < from_date:
@@ -151,24 +227,28 @@ def search_devto(
     endpoint is recorded in ``errors`` and the other endpoint still counts.
     """
     days = _window_days(from_date, to_date)
+    historical = _is_historical(to_date)
+    paging_deadline = time.monotonic() + HISTORICAL_BUDGET_SECONDS
     resolved_tags = resolve_tags(tags, depth)
     articles: dict[int, dict[str, Any]] = {}
     errors: list[str] = []
 
     for tag in resolved_tags:
-        url = f"{ARTICLES_URL}?{urllib.parse.urlencode({'tag': tag, 'top': days, 'per_page': TAG_PER_PAGE})}"
+        params = {"tag": tag, "top": days, "per_page": TAG_PER_PAGE}
         try:
-            rows = _get_list(url)
+            rows = _fetch_rows(ARTICLES_URL, params, to_date, label=f"tag '{tag}'", deadline=paging_deadline)
         except Exception as exc:
             errors.append(f"tag {tag}: {exc}")
             _log(f"tag feed failed for '{tag}': {exc}")
             continue
+        rows = [row for row in rows if row.get("id") is not None and _in_window(row, from_date, to_date)]
+        if historical:
+            rows = _top_engaged(rows, TAG_PER_PAGE)
         kept = 0
         for row in rows:
-            if row.get("id") is not None and _in_window(row, from_date, to_date):
-                row.setdefault("_devto_match", f"tag:{tag}")
-                articles.setdefault(row["id"], row)
-                kept += 1
+            row.setdefault("_devto_match", f"tag:{tag}")
+            articles.setdefault(row["id"], row)
+            kept += 1
         _log(f"tag '{tag}': {kept} articles in window")
 
     # Planner queries quote phrases for engines with phrase syntax; Forem
@@ -176,21 +256,31 @@ def search_devto(
     query = (search_query or topic or "").replace('"', " ").strip()
     query = " ".join(query.split())
     if query:
-        params = {"q": query, "top": days, "per_page": SEARCH_PER_PAGE.get(depth, SEARCH_PER_PAGE["default"])}
-        url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
+        per_page = SEARCH_PER_PAGE.get(depth, SEARCH_PER_PAGE["default"])
+        params = {"q": query, "top": days, "per_page": per_page}
         try:
-            rows = _get_list(url, headers=_SEARCH_HEADERS)
+            rows = _fetch_rows(
+                SEARCH_URL, params, to_date, label=f"search '{query}'",
+                deadline=paging_deadline, headers=_SEARCH_HEADERS,
+            )
         except Exception as exc:
             errors.append(f"search: {exc}")
             _log(f"search failed for '{query}': {exc}")
             rows = []
         kept = dropped = 0
+        relevant = []
         for row in rows:
             if row.get("id") is None or not _in_window(row, from_date, to_date):
                 continue
             if token_overlap_relevance(topic or query, _article_text(row)) < SEARCH_RELEVANCE_FLOOR:
                 dropped += 1
                 continue
+            relevant.append(row)
+        if historical:
+            # A current window keeps the newest page; a past one keeps the
+            # most-engaged in-window hits so the item count stays comparable.
+            relevant = _top_engaged(relevant, per_page)
+        for row in relevant:
             row.setdefault("_devto_match", "search")
             articles.setdefault(row["id"], row)
             kept += 1

package/skills/trends/scripts/lib/digg.py

@@ -19,9 +19,10 @@ top-ranked X posts to each cluster's ``posts`` field.
 from __future__ import annotations
 
 import json
+import re
 import shutil
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urlparse
 
 from . import log, subproc
@@ -133,47 +134,104 @@ def _build_posts_args(cluster_url_id: str, posts_per: int) -> List[str]:
     ]
 
 
-def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
-    """Invoke digg-pp-cli and parse the JSON envelope.
+_LOCAL_FALLBACK_MARKER = "falling back to local FTS5:"
+_FALLBACK_REASON_MAX = 200
+
+
+def _local_fallback_reason(stderr: str) -> str:
+    """Extract the live-search failure from digg-pp-cli's fallback stderr.
 
-    Returns ``{"results": [...]}`` on success, ``{"results": [], "error": "..."}``
-    on failure. Never raises; the pipeline relies on shape consistency.
+    The CLI prints ``falling back to local FTS5: <error>`` and the error can
+    embed the full HTML body of a failed response, so the reason is cut at
+    the HTTP status when present and length-bounded otherwise.
+    """
+    for line in (stderr or "").splitlines():
+        _, marker, rest = line.partition(_LOCAL_FALLBACK_MARKER)
+        if not marker:
+            continue
+        rest = rest.strip()
+        status = re.match(r".*?\bHTTP \d{3}\b", rest)
+        if status:
+            return status.group(0)
+        return rest[:_FALLBACK_REASON_MAX].rstrip()
+    return ""
+
+
+def _empty_local_fallback_error(data: Dict[str, Any], stderr: str = "") -> Optional[str]:
+    """Return an error when the CLI answered from an empty local store.
+
+    ``digg-pp-cli search`` is live by default, but it falls back to its FTS5
+    store on a network error and exits 0 with ``meta.source == "local"``.
+    That store only holds rows for users who ran ``digg-pp-cli sync``
+    themselves, which last30days never does - so a broken live search
+    reaches the user as ``Digg: 0 items (no results)`` instead of an error.
+    A non-empty local store is real data and is left alone.
+
+    The CLI reports the live failure only on stderr; ``meta.reason`` is
+    preferred if a future CLI release adds it to the envelope.
+    """
+    meta = data.get("meta")
+    if not isinstance(meta, dict) or meta.get("source") != "local":
+        return None
+    if data.get("results"):
+        return None
+    reason = str(meta.get("reason") or "").strip()[:_FALLBACK_REASON_MAX]
+    detail = (
+        reason
+        or _local_fallback_reason(stderr)
+        or "live search fell back to an empty local store"
+    )
+    return f"live-search-unavailable: {detail}"
+
+
+def _invoke_cli(cmd: List[str], timeout: int) -> Tuple[Dict[str, Any], str]:
+    """Invoke digg-pp-cli and return the parsed envelope with its stderr.
+
+    The envelope is ``{"results": [...]}`` on success and
+    ``{"results": [], "error": "..."}`` on failure. Never raises; the
+    pipeline relies on shape consistency.
     """
     if not _is_available():
-        return {"results": [], "error": f"{CLI_BIN} not on PATH"}
+        return {"results": [], "error": f"{CLI_BIN} not on PATH"}, ""
     try:
         result = subproc.run_with_timeout(cmd, timeout=timeout)
     except subproc.SubprocTimeout as exc:
         _log(f"Timeout: {exc}")
-        return {"results": [], "error": str(exc)}
+        return {"results": [], "error": str(exc)}, ""
     except FileNotFoundError as exc:
         _log(f"Binary missing: {exc}")
-        return {"results": [], "error": str(exc)}
+        return {"results": [], "error": str(exc)}, ""
     except OSError as exc:
         _log(f"Spawn failed: {exc}")
-        return {"results": [], "error": str(exc)}
+        return {"results": [], "error": str(exc)}, ""
 
+    stderr = result.stderr or ""
     if result.returncode != 0:
-        snippet = (result.stderr or "").strip().splitlines()[:1]
+        snippet = stderr.strip().splitlines()[:1]
         first = snippet[0] if snippet else f"exit {result.returncode}"
         _log(f"CLI exit {result.returncode}: {first}")
-        return {"results": [], "error": first}
+        return {"results": [], "error": first}, stderr
 
     stdout = result.stdout or ""
     if not stdout.strip():
-        return {"results": []}
+        return {"results": []}, stderr
     try:
         data = json.loads(stdout)
     except json.JSONDecodeError as exc:
         _log(f"JSON decode failed: {exc}")
-        return {"results": [], "error": f"json decode: {exc}"}
+        return {"results": [], "error": f"json decode: {exc}"}, stderr
 
     if not isinstance(data, dict):
-        return {"results": []}
+        return {"results": []}, stderr
     results = data.get("results")
     if not isinstance(results, list):
-        return {"results": []}
-    return data
+        return {"results": []}, stderr
+    return data, stderr
+
+
+def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
+    """Invoke digg-pp-cli and return only the parsed envelope."""
+    return _invoke_cli(cmd, timeout)[0]
 
 
 def search_digg(
@@ -199,7 +257,11 @@ def search_digg(
         return {"results": []}
     cmd = _build_search_args(topic, limit)
     _log(f"search '{topic}' (limit={limit}, since=30d)")
-    response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
+    response, stderr = _invoke_cli(cmd, timeout=SEARCH_TIMEOUT)
+    fallback_error = _empty_local_fallback_error(response, stderr)
+    if fallback_error:
+        _log(fallback_error)
+        return {"results": [], "error": fallback_error}
     n = len(response.get("results") or [])
     _log(f"found {n} clusters")
     return response

package/skills/trends/scripts/lib/github.py

@@ -179,7 +179,7 @@ def _compute_relevance(
 # the qualifier so no stray `()`/`""` residue reaches the query (issue #952).
 # The plain regex's boundary also accepts wrapper openers so a qualifier with
 # a missing closer ("(created:>2025-03-20") is still stripped rather than
-# leaking into the query; only the stray opener survives, harmlessly.
+# leaking into the query; qualifier-only wrapper residue is discarded below.
 QUALIFIER_KEYS = frozenset({
     "archived", "assignee", "author", "base", "closed", "comments", "commenter",
     "created", "fork", "forks", "head", "in", "interactions", "involves", "is",
@@ -220,6 +220,11 @@ def strip_search_qualifiers(text: str) -> str:
     """
     stripped = _WRAPPED_QUALIFIER_RE.sub(" ", text)
     stripped = _QUALIFIER_RE.sub(" ", stripped)
+    # Malformed/nested wrappers can leave only punctuation after a qualifier
+    # is removed (#969). There is no subject to search in that case. Keep
+    # wrappers around actual topic text, including unbalanced ones like "ai (foo".
+    if stripped != text and re.fullmatch(r"[\s()\[\]\"']*", stripped):
+        return ""
     while True:
         cleaned = _EMPTY_WRAPPER_RE.sub("", stripped)
         if cleaned == stripped:
@@ -750,8 +755,8 @@ def search_github_person(
     _log(f"Person-mode search for @{username} (since {from_date})")
 
     # Phase 1: PR velocity via search API
-    total_q = urllib.parse.quote(f"author:{username} type:pr created:>{from_date}")
-    merged_q = urllib.parse.quote(f"author:{username} type:pr is:merged created:>{from_date}")
+    total_q = urllib.parse.quote(f"author:{username} type:pr created:{from_date}..{to_date}")
+    merged_q = urllib.parse.quote(f"author:{username} type:pr is:merged created:{from_date}..{to_date}")
 
     total_url = f"{SEARCH_URL}?q={total_q}&per_page=1"
     merged_url = f"{SEARCH_URL}?q={merged_q}&sort=reactions&order=desc&per_page=100"
@@ -823,7 +828,7 @@ def search_github_person(
     num_repos = len(repo_pr_counts)
     velocity_text = (
         f"GitHub Person Profile: @{username}\n\n"
-        f"CONTRIBUTION VELOCITY (last {(to_date > from_date) and 30 or 30} days)\n"
+        f"CONTRIBUTION VELOCITY (last {dates.days_ago(from_date, to_date)} days)\n"
         f"- {merged_count} PRs merged across {num_repos} repos ({merge_rate}% merge rate)\n"
         f"- {total_prs} total PRs submitted, {open_prs} still open\n"
     )

package/skills/trends/scripts/lib/hackernews.py

@@ -10,6 +10,7 @@ import math
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from http.client import HTTPException
 from typing import Any, Dict, List, Optional
 
 import re
@@ -24,6 +25,9 @@ _HN_PREFIXES = re.compile(r"^(Tell HN|Show HN|Ask HN|Launch HN)\s*:\s*", re.IGNO
 ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
 ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
 ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
+# Official HN Firebase item API — `kids` is in the order the thread displays
+# top-level comments (Algolia's /items children do not carry useful points).
+FIREBASE_ITEM_URL = "https://hacker-news.firebaseio.com/v0/item"
 
 DEPTH_CONFIG = {
     "quick": 15,
@@ -309,6 +313,68 @@ def _hn_item_url(item_id: Any) -> str:
     return f"https://news.ycombinator.com/item?id={item_id}"
 
 
+# Ordering is optional, so a slow Firebase must not hold up the report.
+FIREBASE_KIDS_TIMEOUT = 5
+FIREBASE_KIDS_RETRIES = 2
+FIREBASE_KIDS_BUDGET = 8.0
+
+
+def _fetch_hn_kids(object_id: str) -> Optional[List[Any]]:
+    """Return HN's display order for a story's top-level comments, or None on failure.
+
+    Algolia's `/items/<id>` children list is effectively id-order with `points`
+    always null, so sorting by points is a no-op (#1285). The Firebase item
+    endpoint returns `kids` in the same order the HN thread UI shows.
+    """
+    url = f"{FIREBASE_ITEM_URL}/{object_id}.json"
+    # Comment order is a refinement over comments Algolia already returned, so a
+    # Firebase failure must not reach the pipeline's sink and mark HN degraded.
+    try:
+        with http.capture_failures():
+            data = http.request(
+                "GET",
+                url,
+                timeout=FIREBASE_KIDS_TIMEOUT,
+                retries=FIREBASE_KIDS_RETRIES,
+                deadline_monotonic=time.monotonic() + FIREBASE_KIDS_BUDGET,
+            )
+    # http.request wraps HTTP and socket errors, but a truncated or non-UTF-8
+    # body escapes as IncompleteRead or UnicodeDecodeError.
+    except (http.HTTPError, HTTPException, UnicodeDecodeError) as e:
+        _log(f"Failed to fetch HN kids for {object_id}: {e}")
+        return None
+    if not isinstance(data, dict):
+        return None
+    kids = data.get("kids")
+    if not isinstance(kids, list):
+        return None
+    return kids
+
+
+def _order_comments_by_kids(
+    comments: List[Dict[str, Any]],
+    kids: List[Any],
+) -> List[Dict[str, Any]]:
+    """Reorder Algolia children to match HN `kids` order; unknowns keep relative order."""
+    by_id: Dict[Any, Dict[str, Any]] = {}
+    for comment in comments:
+        cid = comment.get("id")
+        if cid is not None and cid not in by_id:
+            by_id[cid] = comment
+    ordered: List[Dict[str, Any]] = []
+    seen = set()
+    for kid in kids:
+        comment = by_id.get(kid)
+        if comment is not None:
+            ordered.append(comment)
+            seen.add(kid)
+    for comment in comments:
+        cid = comment.get("id")
+        if cid not in seen:
+            ordered.append(comment)
+    return ordered
+
+
 def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any]:
     """Fetch top-level comments for a story from Algolia items endpoint.
 
@@ -329,12 +395,18 @@ def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any
 
     children = data.get("children", [])
 
-    # Sort by points (highest first), filter to actual comments
+    # Filter to actual comments. Prefer HN's Firebase `kids` order (#1285);
+    # Algolia children have null points, so a points sort is a no-op and leaves
+    # earliest-posted replies first. Fall back to points only when kids is unavailable.
     real_comments = [
         c for c in children
         if c.get("text") and c.get("author")
     ]
-    real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True)
+    kids = _fetch_hn_kids(object_id)
+    if kids is not None:
+        real_comments = _order_comments_by_kids(real_comments, kids)
+    else:
+        real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True)
 
     comments = []
     insights = []

package/skills/trends/scripts/lib/html_render.py

@@ -20,6 +20,18 @@ PROSE_LABELS = [
 INVITATION_PATTERN = re.compile(r"^---\nI'm now an expert.*?Just ask\.$", re.MULTILINE | re.DOTALL)
 EVIDENCE_BLOCK_PATTERN = re.compile(r"<!-- EVIDENCE FOR SYNTHESIS.*?<!-- END EVIDENCE FOR SYNTHESIS -->", re.DOTALL)
 PASS_THROUGH_FOOTER_PATTERN = re.compile(r"<!-- PASS-THROUGH FOOTER.*?-->\n(.*?)<!-- END PASS-THROUGH FOOTER -->", re.DOTALL)
+# Line-by-line on purpose: no row inside the match may itself open or close a
+# fence, so an earlier ordinary ```text block cannot be stitched to the real
+# footer with the prose between them swallowed into the footer box.
+_FOOTER_ROWS = r"(?:(?!```)[^\n]*\n)*?"
+FENCED_ENGINE_FOOTER_PATTERN = re.compile(
+    r"^```text\n(---\n"
+    + _FOOTER_ROWS
+    + r"[^\n]*All agents reported back![^\n]*\n"
+    + _FOOTER_ROWS
+    + r"---)\n```$",
+    re.MULTILINE,
+)
 CANONICAL_BOUNDARY_PATTERN = re.compile(r"\n?---\n# END OF last30days CANONICAL OUTPUT.*$", re.DOTALL)
 # render_for_html emits metadata as <!-- META: ... --> so it survives the
 # markdown converter (which escapes raw HTML inside paragraphs). Promoted to
@@ -713,7 +725,9 @@ def _protect_engine_footers(md: str) -> tuple[str, dict[str, str]]:
         footers[token] = match.group(1).strip("\n")
         return f"\n{token}\n"
 
-    return PASS_THROUGH_FOOTER_PATTERN.sub(replace, md), footers
+    md = PASS_THROUGH_FOOTER_PATTERN.sub(replace, md)
+    md = FENCED_ENGINE_FOOTER_PATTERN.sub(replace, md)
+    return md, footers
 
 
 def _wrap_engine_footer(body: str) -> str:

package/skills/trends/scripts/lib/pipeline.py

@@ -6116,7 +6116,7 @@ def _retrieve_stream_impl(
         # search_query, so off-topic narrowing does not let weak matches through.
         relevance_topic = raw_topic or topic or subquery.search_query
         return (
-            arxiv.parse_arxiv_response(result, query=relevance_topic),
+            arxiv.parse_arxiv_response(result, query=relevance_topic, to_date=to_date),
             _result_outcome_artifact(source, result),
         )
     if source == "techmeme":

package/skills/trends/scripts/lib/planner.py

@@ -621,6 +621,11 @@ def _trim_subqueries_for_depth(
     # Operator-supplied --plan is a contract: keep per-subquery sources.
     if depth != "quick" and not honor_plan_sources:
         expanded_sources = _default_sources_for_intent(intent, available_sources)
+        # Explicit --search sources are a user override: intent routing must not drop them.
+        expanded_sources += [
+            source for source in available_sources
+            if source in (requested_sources or []) and source not in expanded_sources
+        ]
         return [
             schema.SubQuery(
                 label=subquery.label,

package/skills/trends/scripts/lib/providers.py

@@ -15,7 +15,7 @@ from . import env, http, schema
 GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
 GEMINI_PRO = "gemini-3.1-pro-preview"
 OPENAI_DEFAULT = "gpt-5.4-nano"
-XAI_DEFAULT = "grok-4-1-fast"
+XAI_DEFAULT = "grok-4.20-0309-non-reasoning"
 
 GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
 OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"

package/skills/trends/scripts/lib/reddit_shreddit.py

@@ -98,7 +98,10 @@ def _svc_url(subreddit: str, post_id: str) -> str:
 
 
 def _attr(tag: str, name: str) -> str:
-    m = re.search(rf'\b{name}="([^"]*)"', tag)
+    # HTML attribute names are case-insensitive: the server partial emits
+    # ``thingId``, a DOM round-trip serializes it as ``thingid``. Match the name
+    # either way; the value keeps its case.
+    m = re.search(rf'\b{name}="([^"]*)"', tag, re.IGNORECASE)
     return _html.unescape(m.group(1)) if m else ""