luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/grounding.py

@@ -2,12 +2,45 @@
 
 from __future__ import annotations
 
+import re
 import sys
 import urllib.parse
-from datetime import datetime
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
 from urllib.parse import urlparse
 
-from . import dates, env, http, web_search_keyless
+from . import dates, env, http, parallel_mcp, schema, web_search_keyless
+
+
+@dataclass(frozen=True)
+class GroundedClaimText:
+    """Candidate text with its exact primary evidence item."""
+
+    candidate_id: str
+    title: str
+    summary: str
+    item: schema.SourceItem
+
+
+def claim_source_map(report: schema.Report) -> dict[str, GroundedClaimText]:
+    """Expose only candidate claims that have a clean primary-item trace.
+
+    Freshness verification deliberately starts here instead of scanning all
+    report prose. A candidate without a primary ``SourceItem`` cannot produce
+    an auditable per-claim verdict.
+    """
+    grounded: dict[str, GroundedClaimText] = {}
+    for candidate in report.ranked_candidates:
+        item = schema.candidate_primary_item(candidate)
+        if item is None:
+            continue
+        grounded[candidate.candidate_id] = GroundedClaimText(
+            candidate_id=candidate.candidate_id,
+            title=candidate.title,
+            summary=candidate.snippet or item.snippet or item.body,
+            item=item,
+        )
+    return grounded
 
 
 # ---------------------------------------------------------------------------
@@ -32,7 +65,7 @@ def brave_search(
     for i, r in enumerate((data.get("web", {}).get("results", []))[:count]):
         raw_date = r.get("page_age") or ""
         pub_date = _normalize_date(raw_date[:10]) if raw_date else None
-        if not _in_date_range(pub_date, date_range):
+        if _known_date_out_of_range(pub_date, date_range):
             continue
         items.append({
             "id": f"WB{i + 1}",
@@ -43,6 +76,7 @@ def brave_search(
             "date": pub_date,
             "relevance": 0.8,
             "why_relevant": "Brave web search",
+            "metadata": {"date_window_basis": "server_bounds"},
         })
     artifact = {"label": "brave", "webSearchQueries": [query], "resultCount": len(items)}
     return items, artifact
@@ -77,7 +111,7 @@ def exa_search(
             continue
         raw_date = r.get("publishedDate") or ""
         pub_date = _normalize_date(raw_date.split("T")[0] if "T" in raw_date else raw_date[:10]) if raw_date else None
-        if not _in_date_range(pub_date, date_range):
+        if _known_date_out_of_range(pub_date, date_range):
             continue
         items.append({
             "id": f"WE{i + 1}",
@@ -88,6 +122,7 @@ def exa_search(
             "date": pub_date,
             "relevance": 0.8,
             "why_relevant": "Exa web search",
+            "metadata": {"date_window_basis": "server_bounds"},
         })
     artifact = {"label": "exa", "webSearchQueries": [query], "resultCount": len(items)}
     return items, artifact
@@ -114,7 +149,9 @@ def serper_search(
     for i, r in enumerate((data.get("organic", []))[:count]):
         raw_date = r.get("date") or ""
         pub_date = _parse_serper_date(raw_date)
-        if not _in_date_range(pub_date, date_range):
+        if pub_date is None and _SERPER_RELATIVE_RE.match(raw_date.strip()):
+            continue
+        if _known_date_out_of_range(pub_date, date_range):
             continue
         items.append({
             "id": f"WS{i + 1}",
@@ -125,6 +162,7 @@ def serper_search(
             "date": pub_date,
             "relevance": 0.8,
             "why_relevant": "Serper web search",
+            "metadata": {"date_window_basis": "server_bounds"},
         })
     artifact = {"label": "serper", "webSearchQueries": [query], "resultCount": len(items)}
     return items, artifact
@@ -137,12 +175,16 @@ def serper_search(
 def parallel_search(
     query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
 ) -> tuple[list[dict], dict]:
+    current_window = date_range[1] >= datetime.now(timezone.utc).date().isoformat()
     data = http.request(
         "POST", "https://api.parallel.ai/v1/search",
-        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+        headers={"x-api-key": api_key, "Content-Type": "application/json"},
         json_data={
             "search_queries": [query],
-            "advanced_settings": {"max_results": count},
+            "advanced_settings": {
+                "max_results": count,
+                "source_policy": {"after_date": date_range[0]},
+            },
         },
         timeout=15,
     )
@@ -155,7 +197,9 @@ def parallel_search(
             continue
         raw_date = r.get("publish_date") or ""
         pub_date = _normalize_date(raw_date[:10]) if raw_date else None
-        if not _in_date_range(pub_date, date_range):
+        if _known_date_out_of_range(pub_date, date_range):
+            continue
+        if pub_date is None and not current_window:
             continue
         items.append({
             "id": f"WP{i + 1}",
@@ -166,12 +210,71 @@ def parallel_search(
             "date": pub_date,
             "relevance": 0.8,
             "why_relevant": "Parallel AI web search",
+            "metadata": {
+                "date_window_basis": "server_start_current" if current_window else "server_start"
+            },
         })
     artifact = {"label": "parallel", "webSearchQueries": [query], "resultCount": len(items)}
     return items, artifact
 
 
-def _parse_serper_date(raw: str) -> str | None:
+_SERPER_RELATIVE_RE = re.compile(
+    r"^(?:about\s+)?(\d+)\s+(minute|min|hour|hr|day|week|month|year)s?\s+ago$", re.I
+)
+_SERPER_RELATIVE_DAYS = {"day": 1, "week": 7, "month": 30, "year": 365}
+_SERPER_RELATIVE_WORDS = {"just now": 0, "today": 0, "yesterday": 1}
+
+
+def _now() -> datetime:
+    """Query-time clock. Isolated so tests can pin it."""
+    return datetime.now()
+
+
+def _parse_serper_relative(raw: str, now: datetime | None = None) -> str | None:
+    """Resolve Serper's relative dates ("2 hours ago") to an ISO date.
+
+    Google renders a relative date for recent results, so the fresher a result
+    is the more likely it arrives in this form. Left unparsed it becomes None
+    and `_in_date_range` drops the item, which silently empties the web lane on
+    short research windows.
+
+    The label states the result's age **at query time**, so it resolves against
+    the clock rather than against the end of the research window. The two differ
+    whenever `--as-of` names a past window: a result inside a window ending two
+    weeks ago is still labelled with its age as of today, and anchoring to the
+    window end would place it a further two weeks back and discard it.
+
+    Sub-day units ("N minutes/hours ago") are subtracted from the clock rather
+    than rounded up to the current date, which would name the wrong day across
+    midnight: at 00:30 "23 hours ago" is yesterday. Residual slack comes only
+    from the timezone `_now()` reads against Google's.
+
+    An age large enough to overflow `timedelta` or run past the end of `date`
+    reads as unparseable, like any other junk label, so one malformed value
+    cannot abort the whole result loop in `serper_search`.
+    """
+    anchor = now or _now()
+    text = raw.strip().lower()
+    if text in _SERPER_RELATIVE_WORDS:
+        return (anchor.date() - timedelta(days=_SERPER_RELATIVE_WORDS[text])).isoformat()
+    m = _SERPER_RELATIVE_RE.match(text)
+    if not m:
+        return None
+    unit = {"min": "minute", "hr": "hour"}.get(m.group(2), m.group(2))
+    try:
+        amount = int(m.group(1).lstrip("0") or "0")
+        if unit == "minute":
+            return (anchor - timedelta(minutes=amount)).date().isoformat()
+        if unit == "hour":
+            return (anchor - timedelta(hours=amount)).date().isoformat()
+        return (
+            anchor.date() - timedelta(days=amount * _SERPER_RELATIVE_DAYS[unit])
+        ).isoformat()
+    except (OverflowError, ValueError):
+        return None
+
+
+def _parse_serper_date(raw: str, now: datetime | None = None) -> str | None:
     if not raw:
         return None
     normalized = _normalize_date(raw)
@@ -182,7 +285,7 @@ def _parse_serper_date(raw: str) -> str | None:
             return datetime.strptime(raw.strip(), fmt).date().isoformat()
         except ValueError:
             continue
-    return None
+    return _parse_serper_relative(raw, now)
 
 
 
@@ -236,6 +339,10 @@ def web_search(
         if not key:
             raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
         items, artifact = parallel_search(query, date_range, key)
+    elif backend == "parallel-mcp":
+        items, artifact = parallel_mcp.search(
+            query, date_range, config.get("PARALLEL_API_KEY")
+        )
     elif backend == "keyless":
         items, artifact = web_search_keyless.keyless_search(query, date_range, config)
     elif backend != "none":
@@ -243,7 +350,13 @@ def web_search(
     else:
         return [], {}
     if items and not _reddit_excluded(config):
-        items = _enrich_reddit_items(items)
+        # Reddit enrichment is a best-effort secondary fetch on already-retrieved
+        # web results. Isolate its HTTP failures in a throwaway capture sink so a
+        # reddit.com fetch failure (e.g. a 403 on a datacenter IP) is not
+        # attributed to the web/grounding source itself — which would otherwise
+        # discard the successfully retrieved results and report the source failed.
+        with http.capture_failures():
+            items = _enrich_reddit_items(items)
     return items, artifact
 
 
@@ -327,5 +440,10 @@ def _in_date_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:
     return date_range[0] <= pub_date <= date_range[1]
 
 
+def _known_date_out_of_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:
+    """True when a parseable date falls outside the window. Unknown dates stay."""
+    return pub_date is not None and not _in_date_range(pub_date, date_range)
+
+
 def _domain(url: str) -> str:
     return urlparse(url).netloc.strip().lower()

package/skills/trends/scripts/lib/hackernews.py

@@ -40,6 +40,8 @@ ENRICH_LIMITS = {
     "deep": 10,
 }
 
+DISCOVERY_LIMITS = {"quick": 20, "default": 40, "deep": 60}
+
 
 def _log(msg: str):
     log.source_log("HN", msg, tty_only=False)
@@ -145,6 +147,47 @@ def search_hackernews(
     return response
 
 
+def fetch_discovery_listings(
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Fetch topic-less HN front-page and best-in-window story listings."""
+    limit = DISCOVERY_LIMITS.get(depth, DISCOVERY_LIMITS["default"])
+    from_ts = _date_to_unix(from_date)
+    to_ts = _date_to_unix(to_date) + 86400
+    from urllib.parse import urlencode
+
+    urls = [
+        f"{ALGOLIA_SEARCH_URL}?{urlencode({'tags': 'front_page', 'hitsPerPage': str(limit)})}",
+        f"{ALGOLIA_SEARCH_URL}?{urlencode({
+            'tags': 'story',
+            'numericFilters': f'created_at_i>{from_ts},created_at_i<{to_ts}',
+            'hitsPerPage': str(limit),
+        })}",
+    ]
+    hits: list[dict[str, Any]] = []
+    errors: list[str] = []
+    for url in urls:
+        try:
+            response = http.request("GET", url, timeout=30)
+            hits.extend(response.get("hits") or [])
+        except Exception as exc:
+            errors.append(str(exc))
+
+    seen: set[str] = set()
+    unique_hits: list[dict[str, Any]] = []
+    for hit in hits:
+        object_id = str(hit.get("objectID") or "")
+        if not object_id or object_id in seen:
+            continue
+        seen.add(object_id)
+        unique_hits.append(hit)
+
+    items = parse_hackernews_response({"hits": unique_hits}, query="")
+    return {"items": items, "errors": errors}
+
+
 _WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
 
 
@@ -232,7 +275,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
 
         # Article URL vs HN discussion URL
         article_url = hit.get("url") or ""
-        hn_url = f"https://news.ycombinator.com/item?id={object_id}"
+        hn_url = _hn_item_url(object_id)
 
         # Relevance: blend Algolia rank with token-overlap content matching
         rank_score = max(0.3, 1.0 - (i * 0.02))  # 1.0 -> 0.3 over 35 items
@@ -261,6 +304,11 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
     return items
 
 
+def _hn_item_url(item_id: Any) -> str:
+    """HN discussion/comment permalink for an item id."""
+    return f"https://news.ycombinator.com/item?id={item_id}"
+
+
 def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any]:
     """Fetch top-level comments for a story from Algolia items endpoint.
 
@@ -290,13 +338,18 @@ def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any
 
     comments = []
     insights = []
+    # A quote links to where it was said: the comment's own permalink, else the
+    # story's discussion page -- never the article URL.
+    discussion_url = _hn_item_url(object_id)
     for c in real_comments[:max_comments]:
         text = _strip_html(c.get("text", ""))
         excerpt = text[:300] + "..." if len(text) > 300 else text
+        comment_id = c.get("id")
         comments.append({
             "author": c.get("author", ""),
             "text": excerpt,
-            "points": c.get("points") or 0,
+            "points": c.get("points"),
+            "url": _hn_item_url(comment_id) if comment_id else discussion_url,
         })
         # First sentence as insight
         first_sentence = text.split(". ")[0].split("\n")[0][:200]
@@ -327,7 +380,7 @@ def enrich_top_stories(
     # Sort by points to enrich the most popular stories
     by_points = sorted(
         range(len(items)),
-        key=lambda i: items[i].get("engagement", {}).get("points", 0),
+        key=lambda i: items[i].get("engagement", {}).get("points") or 0,
         reverse=True,
     )
     to_enrich = by_points[:limit]
@@ -335,8 +388,12 @@ def enrich_top_stories(
     _log(f"Enriching top {len(to_enrich)} stories with comments")
 
     with ThreadPoolExecutor(max_workers=5) as executor:
+        # submit_with_context, not executor.submit: a plain submit drops the
+        # pipeline's capture_failures() sink, so failed or rate-limited item
+        # fetches would vanish and the HN source would look fully enriched.
         futures = {
-            executor.submit(
+            http.submit_with_context(
+                executor,
                 _fetch_item_comments,
                 items[idx]["id"],
             ): idx