luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/polymarket.py

@@ -10,12 +10,13 @@ import re
 import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Dict, List, Optional
-from urllib.parse import quote_plus, urlencode
+from urllib.parse import quote, quote_plus, urlencode
 
 from . import http, log
 from .relevance import LOW_SIGNAL_QUERY_TOKENS, token_overlap_relevance
 
 GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
+GAMMA_EVENTS_URL = "https://gamma-api.polymarket.com/events"
 
 # Pages to fetch per query (API returns 5 events per page, limit param is a no-op)
 DEPTH_CONFIG = {
@@ -111,10 +112,7 @@ _NOISE_WORDS = frozenset({
     "springs", "heights", "ridge", "bridge", "harbor", "port", "station", "center",
     "square", "field", "forest", "garden", "tower", "school", "church", "camp",
     "ranch", "crossing", "shore", "rock", "summit", "falls", "grove", "haven",
-    # Generic tech terms that match too broadly on Polymarket
-    # "cli" -> any CLI tool market; "mcp" -> protocol markets; "ai" -> every AI market
-    "cli", "mcp", "protocol", "tool", "app", "code", "model", "ai", "api",
-    "software", "plugin", "skill", "agent", "bot", "search", "research",
+    # Generic tech terms — see _DOMAIN_WORDS below, which is folded in here
     # Generic prediction market terms
     "market", "odds", "prediction", "forecast", "chance", "probability",
     # Comparison-query conjunctions — should not count as informative filter tokens
@@ -122,6 +120,109 @@ _NOISE_WORDS = frozenset({
     "vs", "versus",
 })
 
+# Generic tech terms that match too broadly to be the sole signal for a NARROW
+# topic ("cli" -> any CLI tool market; "ai" -> every AI market), but which ARE
+# the subject when the topic is a domain sweep rather than one product. Kept
+# separate from the rest of _NOISE_WORDS — the directional/sports/place words
+# there exist to PREVENT false matches ("NFC West" vs a "Kanye West" search),
+# so they must never be used as a positive signal.
+_DOMAIN_WORDS = frozenset({
+    "cli", "mcp", "protocol", "tool", "app", "code", "model", "ai", "api",
+    "software", "plugin", "skill", "agent", "bot", "search", "research",
+})
+
+# Soft residue left after stripping domain words from a sweep topic
+# ("AI frontier developments"). Domain-word fallback may fire when these are
+# the only informative leftovers. Distinctive terms like "benchmark" block it.
+_SWEEP_RESIDUE = frozenset({
+    "frontier", "developments", "development", "news", "trends", "trend",
+    "latest", "industry", "space", "ecosystem", "landscape", "overview",
+    "updates", "update", "future", "outlook", "sector", "field", "world",
+})
+
+_NOISE_WORDS = _NOISE_WORDS | _DOMAIN_WORDS
+
+
+def _domain_stem(word: str) -> str | None:
+    """Return the canonical domain token if ``word`` is a domain term or plural.
+
+    Exact-set membership alone treats ``models`` as a hard narrowing term even
+    though ``model`` is a domain word — which blocked soft AI sweeps and broke
+    ``AI models`` → ``New AI prediction``.
+    """
+    if word in _DOMAIN_WORDS:
+        return word
+    if word.endswith("ies") and len(word) > 4:
+        stem = word[:-3] + "y"
+        if stem in _DOMAIN_WORDS:
+            return stem
+    if len(word) > 3 and word.endswith("es") and word[:-2] in _DOMAIN_WORDS:
+        return word[:-2]
+    if len(word) > 2 and word.endswith("s") and word[:-1] in _DOMAIN_WORDS:
+        return word[:-1]
+    return None
+
+
+def _informative_words(core_words: list[str]) -> list[str]:
+    """Topic words that are neither noise nor (possibly plural) domain terms."""
+    return [
+        w for w in core_words
+        if w not in _NOISE_WORDS and _domain_stem(w) is None
+    ]
+
+
+def _domain_word_fallback_allows(core_words: list[str], informative: list[str],
+                                 title_lower: str, title_words: set[str]) -> bool:
+    """Allow domain-word title matches only for pure/soft domain sweeps.
+
+    Blocks mixed topics like \"MCP protocol benchmark\" from accepting a Kyoto
+    Protocol market via the shared domain token \"protocol\" when the distinctive
+    informative word (\"benchmark\") missed.
+    """
+    hard_informative = [w for w in informative if w not in _SWEEP_RESIDUE]
+    if hard_informative:
+        return False
+    domain_stems = []
+    seen: set[str] = set()
+    for w in core_words:
+        stem = _domain_stem(w)
+        if stem and stem not in seen:
+            seen.add(stem)
+            domain_stems.append(stem)
+    if not domain_stems:
+        return False
+    for word in domain_stems:
+        if word in title_words or f"{word}s" in title_words or f"{word}es" in title_words:
+            return True
+        if len(word) >= 4 and word in title_lower:
+            return True
+    return False
+
+
+def _acronym_credit(core_words: list[str], title_words: set[str]) -> int:
+    """Credit matches when the title abbreviates a phrase the topic spells out.
+
+    Prediction-market titles use shorthand ("AGI by 2030?") while topics arrive
+    spelled out ("artificial general intelligence"), so word overlap scores zero
+    on a title that is squarely on topic. For each run of 3+ consecutive
+    informative words, build its initialism and, if the title carries it as a
+    whole word, credit one match per abbreviated word. Requiring at least three
+    letters avoids treating ambiguous tokens such as "ML" as expanded phrases.
+    """
+    informative_set = set(_informative_words(core_words))
+    credit = 0
+    run: list[str] = []
+    for word in core_words + [""]:
+        if word in informative_set:
+            run.append(word)
+            continue
+        if len(run) >= 3:
+            acronym = "".join(w[0] for w in run)
+            if len(acronym) >= 3 and acronym in title_words:
+                credit = max(credit, len(run))
+        run = []
+    return credit
+
 
 def _passes_topic_filter(topic: str, event_title: str) -> bool:
     """Check if event title contains enough informative words from the topic.
@@ -138,8 +239,8 @@ def _passes_topic_filter(topic: str, event_title: str) -> bool:
     if not core_words:
         return True  # No words to check against
 
-    # Split into informative vs generic
-    informative = [w for w in core_words if w not in _NOISE_WORDS]
+    # Split into informative vs generic (domain plurals count as domain, not hard)
+    informative = _informative_words(core_words)
 
     # If ALL words are generic, we can't meaningfully filter — keep everything
     if not informative:
@@ -160,12 +261,22 @@ def _passes_topic_filter(topic: str, event_title: str) -> bool:
         if len(word) >= 4 and word in title_lower:
             match_count += 1
 
+    # A title that abbreviates what the topic spells out ("AGI" for
+    # "artificial general intelligence") scores zero above; credit it here.
+    if match_count < 2:
+        match_count = max(match_count,
+                          _acronym_credit(core_words, title_words))
+
     # For topics with 3+ informative words, require at least 2 matches.
     # This prevents single-word false positives like "mill" in "Meek Mill"
     # when the topic is "Mill.com food recycler" (3 informative words).
     min_matches = 2 if len(informative) >= 3 else 1
 
-    return match_count >= min_matches
+    if match_count >= min_matches:
+        return True
+
+    # Domain-word fallback for soft domain sweeps only (see helper).
+    return _domain_word_fallback_allows(core_words, informative, title_lower, title_words)
 
 
 def _passes_any_informative_word(topic: str, event_title: str) -> bool:
@@ -182,7 +293,7 @@ def _passes_any_informative_word(topic: str, event_title: str) -> bool:
     core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1]
     if not core_words:
         return True
-    informative = [w for w in core_words if w not in _NOISE_WORDS]
+    informative = _informative_words(core_words)
     if not informative:
         return True
 
@@ -194,7 +305,8 @@ def _passes_any_informative_word(topic: str, event_title: str) -> bool:
             return True
         if len(word) >= 4 and word in title_lower:
             return True
-    return False
+
+    return _domain_word_fallback_allows(core_words, informative, title_lower, title_words)
 
 
 def filter_items_against_topic(topic: str, items: List[Any]) -> List[Any]:
@@ -343,7 +455,7 @@ def _run_queries_parallel(
         futures = {}
         for i, q in enumerate(queries, start=start_idx):
             for p in range(1, pages + 1):
-                future = executor.submit(_search_single_query, q, p)
+                future = http.submit_with_context(executor, _search_single_query, q, p)
                 futures[future] = i
 
         for future in as_completed(futures):
@@ -513,6 +625,14 @@ def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None)
     if core in title_lower:
         return 1.0
 
+    # Same match, abbreviated: "AGI" standing in for an informative phrase.
+    # Use the filter's matcher so modifiers and minimum acronym length cannot
+    # produce different decisions at the filtering and scoring stages.
+    core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1]
+    title_words = set(re.sub(r"[^\w\s]", " ", title_lower).split())
+    if _acronym_credit(core_words, title_words):
+        return 1.0
+
     query_type = _infer_query_intent(topic)
     title_score = token_overlap_relevance(core, title)
     best_score = title_score
@@ -562,7 +682,13 @@ def _safe_float(val, default=0.0) -> float:
         return default
 
 
-def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List[Dict[str, Any]]:
+def parse_polymarket_response(
+    response: Dict[str, Any],
+    topic: str = "",
+    *,
+    include_all_outcomes: bool = False,
+    include_closed: bool = False,
+) -> List[Dict[str, Any]]:
     """Parse Gamma API response into normalized item dicts.
 
     Each event becomes one item showing its title and top markets.
@@ -584,10 +710,11 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
         slug = event.get("slug", "")
 
         # Filter: skip closed/resolved events
-        if event.get("closed", False):
-            continue
-        if not event.get("active", True):
-            continue
+        if not include_closed:
+            if event.get("closed", False):
+                continue
+            if not event.get("active", True):
+                continue
 
         # Filter: skip events that don't match the topic's core subject
         # This prevents "NFC West" from matching a "Kanye West" search
@@ -603,16 +730,17 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
         # Filter to active, open markets with liquidity (excludes resolved markets)
         active_markets = []
         for m in markets:
-            if m.get("closed", False):
-                continue
-            if not m.get("active", True):
-                continue
+            if not include_closed:
+                if m.get("closed", False):
+                    continue
+                if not m.get("active", True):
+                    continue
             # Must have liquidity (resolved markets have 0 or None)
             try:
                 liq = float(m.get("liquidity", 0) or 0)
             except (ValueError, TypeError):
                 liq = 0
-            if liq > 0:
+            if include_closed or liq > 0:
                 active_markets.append(m)
 
         if not active_markets:
@@ -743,8 +871,9 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
             if reordered:
                 outcome_prices = reordered + rest
 
-        # Top 3 outcomes for multi-outcome markets
-        top_outcomes = outcome_prices[:3]
+        # Normal display payloads stay compact. Verification requests the
+        # complete snapshot so topic-promoted outcomes remain re-checkable.
+        top_outcomes = outcome_prices if include_all_outcomes else outcome_prices[:3]
         remaining = len(outcome_prices) - 3
         if remaining < 0:
             remaining = 0
@@ -792,3 +921,136 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
 
     cap = response.get("_cap", len(items))
     return items[:cap]
+
+
+def refetch_datum(item: Any, datum_key: str) -> dict[str, Any]:
+    """Re-fetch one event datum through the replay-aware HTTP wrapper."""
+    event_id = str(getattr(item, "metadata", {}).get("event_id") or "").strip()
+    slug_match = re.search(r"/event/([^/?#]+)", str(getattr(item, "url", "")))
+    cached_item_id = str(getattr(item, "item_id", "") or "").strip()
+    # On the slug fallback, a slug can be re-used by a re-created event. When
+    # the cached item still carries the original Gamma event id (numeric; the
+    # synthetic PM<N> parse fallback carries no identity), the response id
+    # must match it too, or the verdict would come from another market.
+    expected_id = (
+        cached_item_id
+        if not event_id and re.fullmatch(r"\d+", cached_item_id)
+        else ""
+    )
+    if event_id:
+        payload = http.request(
+            "GET", f"{GAMMA_EVENTS_URL}/{quote(event_id)}", timeout=10, retries=2,
+        )
+    elif slug_match:
+        if not expected_id:
+            # No event id anywhere: slug equality alone cannot verify event
+            # identity, so fail closed (unsupported) instead of re-deriving a
+            # verdict from whatever event currently owns the slug.
+            raise ValueError(
+                "Polymarket item carries no event id; slug equality alone "
+                "cannot verify event identity"
+            )
+        requested_slug = slug_match.group(1)
+        payload = http.request(
+            "GET", GAMMA_EVENTS_URL, params={"slug": requested_slug},
+            timeout=10, retries=2,
+        )
+    else:
+        raise ValueError("Polymarket item has no event id or slug")
+
+    requested_slug = slug_match.group(1) if slug_match else None
+
+    def _matches_identity(entry: dict) -> bool:
+        if str(entry.get("slug") or "").strip() != requested_slug:
+            return False
+        if expected_id and str(entry.get("id") or "").strip() != expected_id:
+            return False
+        return True
+
+    def _pick_event(events: list) -> Any:
+        candidates = [entry for entry in events if isinstance(entry, dict)]
+        if requested_slug is None:
+            return candidates[0] if candidates else None
+        # Verify identity: Gamma slug queries can return multiple or loosely
+        # matched events, and verifying a claim against another market's
+        # prices would fabricate current/stale verdicts.
+        for entry in candidates:
+            if _matches_identity(entry):
+                return entry
+        return None
+
+    if isinstance(payload, list):
+        event = _pick_event(payload)
+    elif isinstance(payload, dict) and isinstance(payload.get("events"), list):
+        event = _pick_event(payload.get("events") or [])
+    else:
+        event = payload
+        if (
+            requested_slug is not None
+            and isinstance(event, dict)
+            and (
+                str(event.get("slug") or "").strip() not in ("", requested_slug)
+                or (
+                    expected_id
+                    and str(event.get("id") or "").strip() not in ("", expected_id)
+                )
+            )
+        ):
+            event = None
+    if not isinstance(event, dict):
+        raise KeyError("Polymarket event was not found")
+    # Mixed events: an active event can carry resolved child markets whose
+    # high volume would win the parse and swap the outcome labels. Only fall
+    # back to closed markets when nothing is active (fully resolved event -
+    # the stale-odds transition verification exists to catch).
+    markets = event.get("markets") or []
+    has_active = any(
+        isinstance(m, dict) and m.get("active", True) and not m.get("closed", False)
+        for m in markets
+    )
+    parsed = parse_polymarket_response(
+        {"events": [event]},
+        include_all_outcomes=True,
+        include_closed=not has_active,
+    )
+    if not parsed:
+        raise KeyError("Polymarket event is closed, unavailable, or malformed")
+    refreshed = parsed[0]
+    values: dict[str, Any] = {}
+    outcome_pairs = refreshed.get("outcome_prices") or []
+    outcome_totals: dict[str, int] = {}
+    for name, _price in outcome_pairs:
+        normalized = str(name).casefold()
+        outcome_totals[normalized] = outcome_totals.get(normalized, 0) + 1
+    outcome_counts: dict[str, int] = {}
+    for name, price in outcome_pairs:
+        normalized = str(name).casefold()
+        occurrence = outcome_counts.get(normalized, 0)
+        outcome_counts[normalized] = occurrence + 1
+        key = f"{name}\x1f{occurrence}" if outcome_totals[normalized] > 1 else str(name)
+        values[key] = price
+    if refreshed.get("end_date") is not None:
+        values["end_date"] = refreshed["end_date"]
+
+    if datum_key == "end_date":
+        value = values.get("end_date")
+    else:
+        if "\x1f" in datum_key:
+            outcome_name, raw_occurrence = datum_key.rsplit("\x1f", 1)
+            occurrence = int(raw_occurrence)
+        else:
+            outcome_name, occurrence = datum_key, 0
+        matches = [
+            price
+            for name, price in refreshed.get("outcome_prices") or []
+            if str(name).casefold() == outcome_name.casefold()
+        ]
+        value = matches[occurrence] if occurrence < len(matches) else None
+    if value is None:
+        raise KeyError(f"Polymarket datum {datum_key!r} was not found")
+    return {
+        "value": value,
+        "values": values,
+        "url": str(getattr(item, "url", "")),
+        "timestamp": event.get("updatedAt"),
+    }