luckiest-co 1.0.47 → 1.0.48

package/skills/trends/scripts/lib/x_judge.py

@@ -0,0 +1,292 @@
+"""X corpus judging for retrieve-judge-retry.
+
+No I/O: judges items already retrieved. Reuses relevance.token_overlap_relevance.
+
+The judge detects off-topic floods and determines which extracted handles should
+be promoted to the FROM lane based on on-topic post ratio, not frequency.
+"""
+
+from collections import Counter
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from . import relevance
+
+# Minimum on-topic ratio for the overall corpus. Below this, the engine should
+# retry with a wider keyword query. Rome measured ~0.2 (8/40 on-topic).
+CORPUS_ON_TOPIC_FLOOR = 0.4
+
+# Minimum on-topic ratio for a handle's posts to qualify for FROM promotion.
+HANDLE_ON_TOPIC_FLOOR = 0.5
+
+# Minimum on-topic keyword hits before a handle can be promoted to FROM lane.
+# Prevents promoting handles that appeared in thin phrase hits with only 1 match.
+MIN_ON_TOPIC_HITS = 2
+
+
+# Ambiguous short tokens that require case-sensitive matching to avoid
+# pronoun/acronym collisions. E.g., "US" (country) vs "us" (pronoun).
+# For these tokens, require the text to contain the uppercase form (acronym)
+# rather than just the lowercase form (common word).
+_CASE_SENSITIVE_ACRONYMS = frozenset({'us'})
+
+
+def _compute_relevance(query: str, text: str) -> float:
+    """Compute relevance score for a post against the topic query.
+
+    Returns 0.0 for empty/stopword-only queries to avoid treating all items
+    as equally relevant (the shared relevance module returns 0.5 for empty
+    queries as a neutral fallback, but x_judge needs strict filtering).
+
+    Uses case-sensitive matching for ambiguous short tokens like 'us' to
+    distinguish country acronym 'US' from pronoun 'us'.
+    """
+    import re
+    if not query or not text:
+        return 0.0
+
+    q_tokens = relevance.tokenize(query)
+    if not q_tokens:
+        return 0.0  # All query tokens were stopwords
+
+    # Check for ambiguous acronyms that need case-sensitive handling
+    t_tokens = relevance.tokenize(text)
+    filtered_q_tokens = set(q_tokens)
+    for acronym in _CASE_SENSITIVE_ACRONYMS:
+        if acronym in q_tokens and acronym in t_tokens:
+            # Text has the token, but we need to check if it's the acronym (US)
+            # or the common word (us). If text only has lowercase, don't count it.
+            has_uppercase = bool(re.search(rf'\b{acronym.upper()}\b', text))
+            has_lowercase = bool(re.search(rf'\b{acronym}\b', text))
+            if has_lowercase and not has_uppercase:
+                # Text only has lowercase version (pronoun) - don't count this
+                # token in query overlap. This effectively removes "us" from
+                # contributing to the score when text has only the pronoun.
+                t_tokens = t_tokens - {acronym}
+                # Also remove from query for this calculation to avoid
+                # penalizing the overall coverage ratio
+                filtered_q_tokens = filtered_q_tokens - {acronym}
+
+    # If filtering removed all query tokens, fall back to 0
+    if not filtered_q_tokens:
+        return 0.0
+
+    overlap_tokens = filtered_q_tokens & t_tokens
+    if not overlap_tokens:
+        return 0.0
+
+    # Compute simplified relevance: coverage ratio
+    # This is a simpler version of token_overlap_relevance that uses
+    # the filtered tokens rather than re-tokenizing the original text
+    coverage = len(overlap_tokens) / len(filtered_q_tokens)
+    return coverage
+
+
+def judge_x_corpus(
+    items: List[Dict[str, Any]],
+    topic: str,
+    *,
+    ranking_query: str = "",
+) -> Dict[str, Any]:
+    """Judge the retrieved X corpus for on-topic ratio.
+
+    Args:
+        items: List of X items with 'author_handle' and 'text' fields
+        topic: The user topic (e.g., "Rome")
+        ranking_query: Optional ranking query for better relevance scoring
+
+    Returns:
+        Dict with:
+        - on_topic_ratio: float, fraction of posts that are on-topic
+        - is_off_topic_flood: bool, True if corpus fails the on-topic floor
+        - on_topic_items: list, items that passed relevance check
+        - off_topic_items: list, items that failed relevance check
+        - handle_stats: dict, per-handle on-topic counts and totals
+    """
+    if not items:
+        return {
+            "on_topic_ratio": 1.0,
+            "is_off_topic_flood": False,
+            "on_topic_items": [],
+            "off_topic_items": [],
+            "handle_stats": {},
+        }
+
+    # Use ranking_query if provided, otherwise topic
+    query = ranking_query or topic
+
+    on_topic_items = []
+    off_topic_items = []
+    handle_stats: Dict[str, Dict[str, int]] = {}
+
+    for item in items:
+        handle = (item.get("author_handle") or "").lower()
+        text = item.get("text") or ""
+        score = _compute_relevance(query, text)
+
+        # On-topic threshold: relevance.RELEVANCE_FLOOR is 0.1
+        is_on_topic = score >= relevance.RELEVANCE_FLOOR
+
+        if is_on_topic:
+            on_topic_items.append(item)
+        else:
+            off_topic_items.append(item)
+
+        if handle:
+            if handle not in handle_stats:
+                handle_stats[handle] = {"on_topic": 0, "total": 0}
+            handle_stats[handle]["total"] += 1
+            if is_on_topic:
+                handle_stats[handle]["on_topic"] += 1
+
+    on_topic_ratio = len(on_topic_items) / len(items) if items else 1.0
+
+    # Check if top-3 frequency authors have poor on-topic ratio
+    top_handles = sorted(
+        handle_stats.items(),
+        key=lambda x: x[1]["total"],
+        reverse=True,
+    )[:3]
+    top_authors_off_topic = all(
+        stats["on_topic"] / stats["total"] < HANDLE_ON_TOPIC_FLOOR
+        for _, stats in top_handles
+        if stats["total"] > 0
+    ) if top_handles else False
+
+    is_off_topic_flood = (
+        on_topic_ratio < CORPUS_ON_TOPIC_FLOOR
+        or (top_authors_off_topic and len(on_topic_items) < MIN_ON_TOPIC_HITS)
+    )
+
+    return {
+        "on_topic_ratio": on_topic_ratio,
+        "is_off_topic_flood": is_off_topic_flood,
+        "on_topic_items": on_topic_items,
+        "off_topic_items": off_topic_items,
+        "handle_stats": handle_stats,
+    }
+
+
+def promotable_handles(
+    items: List[Dict[str, Any]],
+    topic: str,
+    extracted_handles: List[str],
+    *,
+    explicit_handles: Optional[List[str]] = None,
+    ranking_query: str = "",
+) -> Tuple[List[str], List[str]]:
+    """Determine which handles should be promoted to the FROM lane.
+
+    Split FROM promotion:
+    - Explicit handles (--x-handle/--x-related): always promoted, no AND topic
+    - Extracted handles: promoted only if:
+      - ≥MIN_ON_TOPIC_HITS on-topic keyword hits AND
+      - author on-topic ratio ≥ HANDLE_ON_TOPIC_FLOOR
+      - These pulls AND the topic (from:handle Rome)
+
+    Args:
+        items: List of X items with 'author_handle' and 'text' fields
+        topic: The user topic
+        extracted_handles: Handles extracted from entity_extract
+        explicit_handles: Explicit --x-handle/--x-related handles
+        ranking_query: Optional ranking query for relevance scoring
+
+    Returns:
+        Tuple of (explicit_promotable, extracted_promotable):
+        - explicit_promotable: handles that get FROM without AND topic
+        - extracted_promotable: handles that get FROM with AND topic
+    """
+    explicit_set = {
+        h.lower().lstrip("@")
+        for h in (explicit_handles or [])
+        if h and h.strip()
+    }
+
+    # Judge corpus to get handle stats
+    judgment = judge_x_corpus(items, topic, ranking_query=ranking_query)
+    handle_stats = judgment["handle_stats"]
+
+    explicit_promotable = []
+    extracted_promotable = []
+
+    for handle in extracted_handles:
+        handle_lower = handle.lower().lstrip("@")
+
+        # Explicit handles always promoted (no AND topic)
+        if handle_lower in explicit_set:
+            explicit_promotable.append(handle)
+            continue
+
+        # Check if handle qualifies for extracted promotion
+        stats = handle_stats.get(handle_lower, {"on_topic": 0, "total": 0})
+
+        # Need ≥MIN_ON_TOPIC_HITS on-topic posts
+        if stats["on_topic"] < MIN_ON_TOPIC_HITS:
+            continue
+
+        # Need ≥HANDLE_ON_TOPIC_FLOOR ratio
+        if stats["total"] > 0:
+            ratio = stats["on_topic"] / stats["total"]
+            if ratio >= HANDLE_ON_TOPIC_FLOOR:
+                extracted_promotable.append(handle)
+
+    # Also check explicit handles not in extracted list
+    for handle in (explicit_handles or []):
+        handle_lower = handle.lower().lstrip("@")
+        if handle_lower not in [h.lower() for h in explicit_promotable]:
+            if handle_lower not in [h.lower() for h in extracted_handles]:
+                explicit_promotable.append(handle)
+
+    return explicit_promotable, extracted_promotable
+
+
+def should_retry_x_search(
+    items: List[Dict[str, Any]],
+    topic: str,
+    *,
+    ranking_query: str = "",
+    depth: str = "default",
+) -> bool:
+    """Determine if X search should retry with wider keyword query.
+
+    Skip retry on quick/mock (same as Phase 2).
+
+    Args:
+        items: Retrieved X items
+        topic: The user topic
+        ranking_query: Optional ranking query
+        depth: Search depth ("quick", "default", "deep")
+
+    Returns:
+        True if retry is warranted
+    """
+    if depth == "quick":
+        return False
+
+    if not items:
+        return False  # Nothing to judge, no retry
+
+    judgment = judge_x_corpus(items, topic, ranking_query=ranking_query)
+    return judgment["is_off_topic_flood"]
+
+
+def prune_off_topic_items(
+    items: List[Dict[str, Any]],
+    topic: str,
+    *,
+    ranking_query: str = "",
+) -> List[Dict[str, Any]]:
+    """Prune off-topic items before the pool.
+
+    Eight on-topic items with 32 pruned → ok with 8.
+    Zero on-topic → no-results, not ok with 40 junk.
+
+    Args:
+        items: Retrieved X items
+        topic: The user topic
+        ranking_query: Optional ranking query
+
+    Returns:
+        Only on-topic items
+    """
+    judgment = judge_x_corpus(items, topic, ranking_query=ranking_query)
+    return judgment["on_topic_items"]

package/skills/trends/scripts/lib/xai_x.py

@@ -5,7 +5,7 @@ import re
 import sys
 from typing import Any, Dict, List, Optional
 
-from . import http, log
+from . import http, log, providers
 
 
 def _safe_text(val) -> str:
@@ -74,6 +74,8 @@ def search_x(
     to_date: str,
     depth: str = "default",
     mock_response: Optional[Dict] = None,
+    deadline_monotonic: Optional[float] = None,
+    cancel: Any = None,
 ) -> Dict[str, Any]:
     """Search X for relevant posts using xAI API with live search.
 
@@ -85,6 +87,14 @@ def search_x(
         to_date: End date (YYYY-MM-DD)
         depth: Research depth - "quick", "default", or "deep"
         mock_response: Mock response for testing
+        deadline_monotonic: Optional shared wall-clock deadline (``time.monotonic()``
+            instant) from the X backend chain. Bounds the whole call including
+            retries; without one this call owns its own timeout only.
+
+    The transport runs with ``retries=1`` (a global default change from the
+    inherited ``retries=5``): one 90-180s live-search call must not become a
+    10-15 minute stall. DNS-resolution failures are the exception: the
+    transport widens those to three attempts.
 
     Returns:
         Raw API response
@@ -122,7 +132,20 @@ def search_x(
         ],
     }
 
-    return http.post(XAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
+    # Honor XAI_BASE_URL like the planner/rerank client does (providers.py), so
+    # an override redirects every request that carries the xAI key, not only some.
+    url = providers.resolve_endpoint("XAI_BASE_URL", XAI_RESPONSES_URL)
+    # A single non-DNS attempt: the per-call timeout already spans the model's
+    # full live-search latency, and the inherited http default (retries=5)
+    # would turn one 90-180s call into a 10-15 minute stall. The shared chain
+    # deadline bounds the wait wall-clock; failover to the next X backend
+    # covers the miss.
+    return http.post(
+        url, payload, headers=headers, timeout=timeout,
+        retries=1, deadline_monotonic=deadline_monotonic,
+        cancel=cancel,
+        bypass_proxy=providers.is_loopback_http_endpoint(url),
+    )
 
 
 def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
@@ -141,7 +164,7 @@ def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
         error = response["error"]
         err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
         _log_error(f"xAI API error: {err_msg}")
-        if http.DEBUG:
+        if log.is_debug():
             _log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
         return items
 

package/skills/trends/scripts/lib/xquik.py

@@ -7,11 +7,13 @@ bookmarks). Requires an API key from xquik.com.
 
 from __future__ import annotations
 
+import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 from . import http, log
 from .relevance import token_overlap_relevance as _compute_relevance
+from .x_api import is_own_post
 
 # Per-process probe cache: (state, reason). state is "unset" until probed, then
 # True (funded/working) | False (auth/payment failure) | None (inconclusive).
@@ -78,6 +80,8 @@ def search_xquik(
     to_date: str,
     depth: str = "default",
     token: str = "",
+    deadline: float | None = None,
+    cancel: Any = None,
 ) -> Dict[str, Any]:
     """Search X via Xquik REST API.
 
@@ -87,6 +91,10 @@ def search_xquik(
         to_date: End date (YYYY-MM-DD)
         depth: Research depth - "quick", "default", or "deep"
         token: Xquik API key
+        deadline: Optional shared wall-clock deadline (``time.monotonic()``
+            instant) from the X backend chain. Queries past the deadline are
+            never started; the per-request timeout/retry pair still bounds
+            each call (mirrors ``x_api``'s TIMEOUT_SECONDS/RETRIES pattern).
 
     Returns:
         Dict with "items" list and optional "error" string.
@@ -100,12 +108,17 @@ def search_xquik(
     seen_ids: set[str] = set()
 
     for query_text in queries:
+        if (cancel is not None and cancel.is_set()) or (deadline is not None and time.monotonic() >= deadline):
+            _log("chain deadline reached; skipping remaining queries")
+            return {"items": all_items, "error": "Xquik research cancelled or timed out"}
         q = f"{query_text} since:{from_date} until:{to_date}"
         items, auth_error = _execute_search(
             q, cfg["limit"], token,
             label=query_text, id_prefix="XQ",
             seen_ids=seen_ids, relevance_query=query_text,
             index_offset=len(all_items),
+            deadline_monotonic=deadline,
+            cancel=cancel,
         )
         if auth_error:
             # Auth/payment failure is fatal for the whole source (e.g. 401/403,
@@ -114,6 +127,8 @@ def search_xquik(
             return {"items": [], "error": auth_error}
         all_items.extend(items)
 
+    if (cancel is not None and cancel.is_set()) or (deadline is not None and time.monotonic() >= deadline):
+        return {"items": all_items, "error": "Xquik research cancelled or timed out"}
     return {"items": all_items}
 
 
@@ -127,38 +142,64 @@ def _execute_search(
     seen_ids: set[str],
     relevance_query: str,
     index_offset: int = 0,
+    deadline_monotonic: float | None = None,
+    failure_out: Optional[List[str]] = None,
+    cancel: Any = None,
 ) -> tuple[List[Dict[str, Any]], str | None]:
     """Run one Xquik search call and parse its tweets.
 
     Returns ``(items, auth_error)``. ``auth_error`` is a non-empty string only
     on a fatal auth/payment failure (401/403); transient/HTTP errors log and
     return ``([], None)`` so one bad lane never discards another's results.
+
+    Non-fatal failures (429, 5xx, network error, unparseable payload) append a
+    reason to ``failure_out`` instead. They stay out of the return value
+    because the caller must keep going to the next handle, but a caller that
+    stopped there would report the lane as empty rather than as failed.
     ``relevance_query`` (the topic) is what items are scored against — for the
     handle lanes that differs from the search query (``from:handle``).
     ``index_offset`` keeps item ids unique across multiple calls that share an
     accumulator (multi-query topic search, per-handle lanes).
+    ``deadline_monotonic`` bounds the request including retries at the
+    transport (mirrors ``x_api._get``).
     """
     full_url = f"{_BASE_URL}/x/tweets/search?q={_url_encode(q)}&queryType=Top&limit={limit}"
     _log(f"Searching: {label}")
     try:
-        response = http.get(full_url, headers={"X-Api-Key": token}, timeout=30, retries=2)
+        request_headers = {"X-Api-Key": token}
+        response = http.get(
+            full_url, headers=request_headers, timeout=30, retries=2,
+            deadline_monotonic=deadline_monotonic,
+            cancel=cancel,
+        )
     except http.HTTPError as exc:
         status = getattr(exc, "status_code", None)
         if status == 402:
             # Unpaid key — fatal for the source, and surfaced on the real search
             # path (not just --diagnose) so a live run reports it instead of
-            # settling silently empty.
-            return [], "Xquik key unpaid (402)"
+            # settling silently empty. The X retrieval branch classifies by
+            # message text only, so the detail carries the "payment required"
+            # marker that http.classify_failure maps to PAYMENT_REQUIRED.
+            return [], "Xquik key unpaid: payment required (402)"
         if status in (401, 403):
             return [], f"Xquik auth failed ({status})"
         _log(f"HTTP error for '{label}': {exc}")
+        if failure_out is not None:
+            failure_out.append(f"Xquik HTTP error for '{label}': {exc}")
         return [], None
     except Exception as exc:
         _log(f"Error for '{label}': {exc}")
+        if failure_out is not None:
+            failure_out.append(f"Xquik request failed for '{label}': {exc}")
         return [], None
 
     tweets = response.get("tweets", [])
     if not isinstance(tweets, list):
+        if failure_out is not None:
+            failure_out.append(
+                f"Xquik returned no tweets array for '{label}' "
+                f"(got {type(tweets).__name__})"
+            )
         return [], None
     items: List[Dict[str, Any]] = []
     for tweet in tweets:
@@ -174,15 +215,10 @@ def _execute_search(
     return items, None
 
 
-def _is_own(url: str, handle: str) -> bool:
-    """True when a tweet URL is authored by ``handle`` (their own post).
-
-    Used by the ABOUT lane to drop the subject's own tweets so only mentions
-    *by others* remain. Handles both x.com and twitter.com permalinks.
-    """
-    u = (url or "").lower()
-    h = handle.lower().lstrip("@").strip()
-    return bool(h) and (f"x.com/{h}/status" in u or f"twitter.com/{h}/status" in u)
+# True when a tweet URL is authored by ``handle`` (their own post). Used by
+# the ABOUT lane to drop the subject's own tweets so only mentions *by
+# others* remain; the implementation is ``x_api.is_own_post``.
+_is_own = is_own_post
 
 
 def search_handles(
@@ -193,18 +229,30 @@ def search_handles(
     *,
     count_per: int = 8,
     token: str = "",
+    failure_out: Optional[List[str]] = None,
+    deadline: float | None = None,
+    cancel: Any = None,
 ) -> List[Dict[str, Any]]:
     """FROM lane: tweets authored BY each handle (their own timeline).
 
     The topic is NOT AND'd into the query (that was the from:-AND bug, #610) —
     we pull the raw timeline and use ``topic`` for relevance ranking only.
     Returns a flat list of item dicts (mirrors ``bird_x.search_handles``).
+
+    When ``failure_out`` is provided, any failure reason is appended — the
+    fatal auth/payment one that stops the lane, and the per-handle transient
+    ones (429, 5xx, network, unparseable payload) that do not — so the caller
+    can distinguish a failed request from a handle that posted nothing.
     """
     if not token or not handles:
         return []
     items: List[Dict[str, Any]] = []
     seen_ids: set[str] = set()
     for raw in handles:
+        if (cancel is not None and cancel.is_set()) or (deadline is not None and time.monotonic() >= deadline):
+            if failure_out is not None:
+                failure_out.append("Xquik research cancelled or timed out")
+            break
         handle = str(raw).lstrip("@").strip()
         if not handle:
             continue
@@ -214,9 +262,17 @@ def search_handles(
             label=f"from:{handle}", id_prefix="XF",
             seen_ids=seen_ids, relevance_query=topic,
             index_offset=len(items),
+            failure_out=failure_out,
+            deadline_monotonic=deadline,
+            cancel=cancel,
         )
         if auth_error:
-            break  # fatal auth/payment failure — stop, keep what we have
+            # Fatal auth/payment failure — stop, keep what we have. Surface the
+            # reason: an empty FROM lane is otherwise indistinguishable from a
+            # subject who simply did not post.
+            if failure_out is not None:
+                failure_out.append(auth_error)
+            break
         items.extend(got)
     return items
 
@@ -229,17 +285,29 @@ def search_mentions(
     topic: str = "",
     count_per: int = 5,
     token: str = "",
+    failure_out: Optional[List[str]] = None,
+    deadline: float | None = None,
+    cancel: Any = None,
 ) -> List[Dict[str, Any]]:
     """ABOUT lane: tweets mentioning each handle, authored by OTHERS.
 
     Queries ``@handle`` then drops the handle's own tweets (``_is_own``) so only
     third-party mentions remain. Returns a flat list of item dicts.
+
+    When ``failure_out`` is provided, any failure reason is appended — the
+    fatal auth/payment one that stops the lane, and the per-handle transient
+    ones (429, 5xx, network, unparseable payload) that do not — so the caller
+    can distinguish a failed request from a handle nobody mentioned.
     """
     if not token or not handles:
         return []
     items: List[Dict[str, Any]] = []
     seen_ids: set[str] = set()
     for raw in handles:
+        if (cancel is not None and cancel.is_set()) or (deadline is not None and time.monotonic() >= deadline):
+            if failure_out is not None:
+                failure_out.append("Xquik research cancelled or timed out")
+            break
         handle = str(raw).lstrip("@").strip()
         if not handle:
             continue
@@ -249,8 +317,13 @@ def search_mentions(
             label=f"@{handle}", id_prefix="XA",
             seen_ids=seen_ids, relevance_query=topic,
             index_offset=len(items),
+            failure_out=failure_out,
+            deadline_monotonic=deadline,
+            cancel=cancel,
         )
         if auth_error:
+            if failure_out is not None:
+                failure_out.append(auth_error)
             break
         items.extend(it for it in got if not _is_own(it.get("url", ""), handle))
     return items
@@ -280,11 +353,12 @@ def probe_works(token: str, timeout: int = 8) -> Optional[bool]:
     q = f"from:x since:{since}"
     full_url = f"{_BASE_URL}/x/tweets/search?q={_url_encode(q)}&queryType=Top&limit=1"
     try:
-        http.get(full_url, headers={"X-Api-Key": token}, timeout=timeout, retries=0)
+        request_headers = {"X-Api-Key": token}
+        http.get(full_url, headers=request_headers, timeout=timeout, retries=0)
     except http.HTTPError as exc:
         status = getattr(exc, "status_code", None)
         if status == 402:
-            _probe_cache = (False, "xquik key unpaid (402)")
+            _probe_cache = (False, "xquik key unpaid: payment required (402)")
         elif status in (401, 403):
             _probe_cache = (False, f"xquik auth failed ({status})")
         else: