luckiest-co 1.0.0 → 1.0.3

package/skills/luckiest-trends/scripts/lib/cluster.py

@@ -0,0 +1,271 @@
+"""Candidate clustering and representative selection."""
+
+from __future__ import annotations
+
+import re
+
+from . import dedupe, schema
+
+CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
+
+# Words too common to signal shared topic between clusters.
+_ENTITY_STOPWORDS = frozenset({
+    "the", "a", "an", "to", "for", "how", "is", "in", "of", "on", "and",
+    "with", "from", "by", "at", "this", "that", "it", "what", "are", "do",
+    "can", "his", "her", "he", "she", "its", "was", "has", "new", "just",
+    "says", "said", "will", "about", "after", "now", "all", "been", "here",
+    "not", "out", "up", "more", "also", "but", "who", "year", "first",
+    "make", "being", "making", "over", "into", "than", "they", "their",
+    "would", "could", "get", "got", "some", "like", "back", "going",
+    "breaking", "https", "http", "www", "com",
+})
+
+
+def _candidate_text(candidate: schema.Candidate) -> str:
+    return " ".join(part for part in [candidate.title, candidate.snippet] if part).strip()
+
+
+def _extract_entities(text: str) -> set[str]:
+    """Extract significant words (proper nouns, numbers, capitalized words) from text.
+
+    Used for cross-source cluster merging where phrasing differs but entities overlap.
+    """
+    # Normalize but preserve word boundaries
+    words = re.sub(r"[^\w\s]", " ", text).split()
+    entities = set()
+    for word in words:
+        lower = word.lower()
+        if lower in _ENTITY_STOPWORDS or len(word) <= 2:
+            continue
+        # Keep words that are: capitalized, ALL CAPS, contain digits, or 4+ chars
+        if word[0].isupper() or word.isupper() or any(c.isdigit() for c in word) or len(word) >= 4:
+            entities.add(lower)
+    return entities
+
+
+def _entity_overlap(entities_a: set[str], entities_b: set[str]) -> float:
+    """Jaccard-style overlap on extracted entities."""
+    if not entities_a or not entities_b:
+        return 0.0
+    intersection = entities_a & entities_b
+    smaller = min(len(entities_a), len(entities_b))
+    # Use overlap coefficient (intersection / min) instead of Jaccard,
+    # because a short tweet about the same event as a long Reddit post
+    # will have fewer total entities but high overlap with the larger set.
+    return len(intersection) / smaller if smaller > 0 else 0.0
+
+
+def _mmr_representatives(
+    candidates: list[schema.Candidate],
+    text_cache: dict[str, dedupe._PreparedText],
+    limit: int = 3,
+    diversity_lambda: float = 0.75,
+) -> list[str]:
+    selected: list[schema.Candidate] = []
+    remaining_set = {c.candidate_id for c in candidates}
+    remaining = list(candidates)
+    while remaining and len(selected) < limit:
+        if not selected:
+            best = max(remaining, key=lambda candidate: candidate.final_score)
+            selected.append(best)
+            remaining_set.discard(best.candidate_id)
+            remaining = [c for c in remaining if c.candidate_id in remaining_set]
+            continue
+
+        selected_preps = [text_cache[c.candidate_id] for c in selected]
+
+        def score(candidate: schema.Candidate) -> float:
+            prep = text_cache[candidate.candidate_id]
+            diversity_penalty = max(
+                dedupe.prepared_similarity(prep, sp) for sp in selected_preps
+            )
+            return (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
+
+        best = max(remaining, key=score)
+        selected.append(best)
+        remaining_set.discard(best.candidate_id)
+        remaining = [c for c in remaining if c.candidate_id in remaining_set]
+    return [candidate.candidate_id for candidate in selected]
+
+
+def cluster_candidates(
+    candidates: list[schema.Candidate],
+    plan: schema.QueryPlan,
+) -> list[schema.Cluster]:
+    """Greedy clustering around high-ranked leaders."""
+    if plan.intent not in CLUSTERABLE_INTENTS or plan.cluster_mode == "none":
+        clusters = []
+        for index, candidate in enumerate(candidates, start=1):
+            cluster_id = f"cluster-{index}"
+            candidate.cluster_id = cluster_id
+            clusters.append(
+                schema.Cluster(
+                    cluster_id=cluster_id,
+                    title=candidate.title,
+                    candidate_ids=[candidate.candidate_id],
+                    representative_ids=[candidate.candidate_id],
+                    sources=sorted(schema.candidate_sources(candidate)),
+                    score=candidate.final_score,
+                    uncertainty=None,
+                )
+            )
+        return clusters
+
+    text_cache: dict[str, dedupe._PreparedText] = {
+        c.candidate_id: dedupe._PreparedText(_candidate_text(c))
+        for c in candidates
+    }
+
+    groups: list[list[schema.Candidate]] = []
+    # Lower threshold for breaking_news: related articles share fewer exact
+    # words but cover the same event.
+    threshold = 0.42 if plan.intent == "breaking_news" else 0.48
+    for candidate in candidates:
+        assigned = False
+        cand_prep = text_cache[candidate.candidate_id]
+        for group in groups:
+            leader = group[0]
+            similarity = dedupe.prepared_similarity(cand_prep, text_cache[leader.candidate_id])
+            if similarity >= threshold:
+                group.append(candidate)
+                assigned = True
+                break
+        if not assigned:
+            groups.append([candidate])
+
+    clusters: list[schema.Cluster] = []
+    for index, group in enumerate(groups, start=1):
+        group.sort(key=lambda candidate: candidate.final_score, reverse=True)
+        cluster_id = f"cluster-{index}"
+        representatives = _mmr_representatives(group, text_cache)
+        for candidate in group:
+            candidate.cluster_id = cluster_id
+        clusters.append(
+            schema.Cluster(
+                cluster_id=cluster_id,
+                title=group[0].title,
+                candidate_ids=[candidate.candidate_id for candidate in group],
+                representative_ids=representatives,
+                sources=sorted({source for candidate in group for source in schema.candidate_sources(candidate)}),
+                score=max(candidate.final_score for candidate in group),
+                uncertainty=_cluster_uncertainty(group),
+            )
+        )
+
+    # Second pass: merge small clusters that share entities across sources.
+    clusters = _merge_entity_clusters(clusters, candidates)
+
+    return sorted(clusters, key=lambda cluster: cluster.score, reverse=True)
+
+
+def _merge_entity_clusters(
+    clusters: list[schema.Cluster],
+    all_candidates: list[schema.Candidate],
+) -> list[schema.Cluster]:
+    """Merge small clusters that cover the same story across different sources.
+
+    The initial greedy pass uses text similarity which misses cross-source
+    matches where phrasing differs. This second pass looks at entity overlap
+    (proper nouns, names, numbers) to catch cases like:
+      - Reddit: "Kanye West to headline all three nights of Wireless Festival 2026"
+      - X: "BREAKING: Kanye West (Ye) is making his massive UK comeback!"
+    """
+    if len(clusters) < 2:
+        return clusters
+
+    candidate_map = {c.candidate_id: c for c in all_candidates}
+
+    # Build entity sets per cluster
+    cluster_entities: list[set[str]] = []
+    for cl in clusters:
+        entities: set[str] = set()
+        for cid in cl.candidate_ids:
+            cand = candidate_map.get(cid)
+            if cand:
+                entities |= _extract_entities(_candidate_text(cand))
+        cluster_entities.append(entities)
+
+    # Only merge clusters with <= 3 items (don't merge already-large clusters)
+    merged_into: dict[int, int] = {}  # index -> merge target index
+    for i in range(len(clusters)):
+        if i in merged_into or len(clusters[i].candidate_ids) > 3:
+            continue
+        for j in range(i + 1, len(clusters)):
+            if j in merged_into or len(clusters[j].candidate_ids) > 3:
+                continue
+            # Require different sources to merge (same-source should already be grouped)
+            sources_i = set(clusters[i].sources)
+            sources_j = set(clusters[j].sources)
+            if sources_i == sources_j and len(sources_i) == 1:
+                continue
+            # Prevent Polymarket clusters from merging with non-Polymarket
+            # clusters. Prediction markets about "Sam Altman equity" should not
+            # merge into a news cluster about "Sam Altman rivalry" just because
+            # both mention the same entity.
+            poly_i = "polymarket" in sources_i
+            poly_j = "polymarket" in sources_j
+            if poly_i != poly_j:
+                continue
+
+            overlap = _entity_overlap(cluster_entities[i], cluster_entities[j])
+            if overlap >= 0.45:
+                merged_into[j] = i
+
+    if not merged_into:
+        return clusters
+
+    # Build merged cluster list
+    result: list[schema.Cluster] = []
+    for i, cl in enumerate(clusters):
+        if i in merged_into:
+            continue
+        # Collect all clusters merged into this one
+        merge_sources = [i] + [j for j, target in merged_into.items() if target == i]
+        if len(merge_sources) == 1:
+            result.append(cl)
+            continue
+
+        # Combine candidates from all merged clusters
+        combined_cids: list[str] = []
+        combined_sources: set[str] = set()
+        best_score = 0.0
+        for idx in merge_sources:
+            combined_cids.extend(clusters[idx].candidate_ids)
+            combined_sources.update(clusters[idx].sources)
+            best_score = max(best_score, clusters[idx].score)
+
+        # Pick representatives from combined pool
+        combined_candidates = [candidate_map[cid] for cid in combined_cids if cid in candidate_map]
+        combined_candidates.sort(key=lambda c: c.final_score, reverse=True)
+        merge_text_cache = {
+            c.candidate_id: dedupe._PreparedText(_candidate_text(c))
+            for c in combined_candidates
+        }
+        reps = _mmr_representatives(combined_candidates, merge_text_cache)
+
+        cluster_id = cl.cluster_id
+        for cid in combined_cids:
+            cand = candidate_map.get(cid)
+            if cand:
+                cand.cluster_id = cluster_id
+
+        result.append(schema.Cluster(
+            cluster_id=cluster_id,
+            title=combined_candidates[0].title if combined_candidates else cl.title,
+            candidate_ids=combined_cids,
+            representative_ids=reps,
+            sources=sorted(combined_sources),
+            score=best_score,
+            uncertainty=_cluster_uncertainty(combined_candidates),
+        ))
+
+    return result
+
+
+def _cluster_uncertainty(group: list[schema.Candidate]) -> str | None:
+    sources = {source for candidate in group for source in schema.candidate_sources(candidate)}
+    if len(sources) == 1:
+        return "single-source"
+    if max(candidate.final_score for candidate in group) < 55:
+        return "thin-evidence"
+    return None

package/skills/luckiest-trends/scripts/lib/competitors.py

@@ -0,0 +1,198 @@
+"""Discover peer entities ("competitors") for a topic via web search.
+
+Mirrors the `resolve.auto_resolve()` pattern: fan out 2-3 web searches via
+`grounding.web_search()`, then extract capitalized entity candidates from
+titles and snippets with deterministic text mining. No LLM call — the
+hosting reasoning model can always override discovery via
+`--competitors-list`.
+
+Returned list is ordered by score (frequency across queries) and capped to
+the caller's requested count.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from . import dates, grounding, log
+from .resolve import _has_backend
+
+# A "brand-shaped" token starts with uppercase OR is camelCase with an
+# uppercase letter later. Catches "Anthropic", "OpenAI", "xAI", "iPhone",
+# "eBay", "Hugging", "Face".
+_BRAND_TOKEN = (
+    r"(?:[A-Z][A-Za-z0-9&.\-]*"
+    r"|[a-z][A-Za-z0-9&.\-]*[A-Z][A-Za-z0-9&.\-]*)"
+)
+
+# A capitalized phrase of 1-4 brand tokens separated by whitespace.
+_CAPITALIZED_PHRASE = re.compile(
+    rf"\b{_BRAND_TOKEN}(?:\s+{_BRAND_TOKEN}){{0,3}}\b"
+)
+
+# Title-case fillers common in listicle SERPs. Kept flat — extraction
+# rejects a candidate whose entire tokens are stopwords, not candidates
+# that merely contain one.
+_STOPWORD_TOKENS: frozenset[str] = frozenset(
+    token.lower()
+    for token in (
+        # Listicle fillers
+        "Top", "Best", "Worst", "Popular", "Leading", "Similar",
+        "Alternatives", "Alternative", "Competitor", "Competitors",
+        "vs", "Vs", "Versus", "Review", "Reviews", "Comparison",
+        "Guide", "List", "Lists", "Full", "Complete", "Free", "Paid",
+        "Tools", "Tool", "Options", "Rivals", "Rival", "Similar",
+        "Pick", "Picks", "Ranking", "Ranked", "Recommended",
+        # Grammar / time
+        "The", "A", "An", "Of", "In", "For", "To", "With", "On", "At",
+        "By", "From", "Is", "Are", "And", "Or", "But", "Than", "As",
+        "This", "That", "These", "Those", "Our", "Your", "Their",
+        "January", "February", "March", "April", "May", "June", "July",
+        "August", "September", "October", "November", "December",
+        # Years likely to appear as standalone tokens
+        *(str(year) for year in range(2018, 2031)),
+        # Miscellaneous SERP noise
+        "AI", "Apps", "App", "Software", "Platform", "Service", "Startups",
+        "Companies", "Company", "Products", "Product", "Brands", "Brand",
+    )
+)
+
+
+def _log(msg: str) -> None:
+    log.source_log("Competitors", msg, tty_only=False)
+
+
+def _topic_tokens(topic: str) -> set[str]:
+    """Return lowercase alphanumeric tokens of the topic for filtering."""
+    return {tok for tok in re.findall(r"[A-Za-z0-9]+", topic.lower()) if tok}
+
+
+def _candidate_ok(candidate: str, topic_tokens: set[str]) -> bool:
+    """Filter a candidate phrase against stopwords and topic overlap."""
+    tokens = [t for t in re.findall(r"[A-Za-z0-9&.\-]+", candidate) if t]
+    if not tokens:
+        return False
+    # Reject candidates made entirely of stopwords (e.g., "Top Alternatives").
+    if all(tok.lower() in _STOPWORD_TOKENS for tok in tokens):
+        return False
+    # Reject candidates that overlap with the topic (e.g., topic="OpenAI"
+    # should not return "OpenAI Alternatives" or "OpenAI").
+    lower_tokens = {tok.lower() for tok in tokens}
+    if lower_tokens & topic_tokens:
+        return False
+    # Reject too-short one-letter tokens like "I" or single digits.
+    if len(tokens) == 1 and len(tokens[0]) < 2:
+        return False
+    return True
+
+
+def _normalize_candidate(candidate: str) -> str:
+    """Collapse whitespace and strip trailing punctuation."""
+    return re.sub(r"\s+", " ", candidate).strip(".,;:!?'\"()[] ")
+
+
+def _extract_peer_entities(
+    items: list[dict], topic: str, limit: int,
+) -> list[str]:
+    """Score capitalized candidates across SERP items and return top `limit`.
+
+    Scoring is bag-of-phrases frequency across all items in the input. Ties
+    are broken by first-seen order so the output is deterministic.
+    """
+    topic_tokens = _topic_tokens(topic)
+    counts: Counter[str] = Counter()
+    first_seen: dict[str, int] = {}
+    order = 0
+    # Group candidates into a frequency map keyed by lowercased normalized
+    # form so "xAI" and "xAI" count together regardless of case.
+    canonical: dict[str, str] = {}
+    for item in items:
+        text = f"{item.get('title', '')} {item.get('snippet', '')}"
+        for raw in _CAPITALIZED_PHRASE.findall(text):
+            candidate = _normalize_candidate(raw)
+            if not _candidate_ok(candidate, topic_tokens):
+                continue
+            key = candidate.lower()
+            if key not in canonical:
+                canonical[key] = candidate
+                first_seen[key] = order
+                order += 1
+            counts[key] += 1
+
+    ranked_keys = sorted(
+        counts.keys(),
+        key=lambda k: (-counts[k], first_seen[k]),
+    )
+    return [canonical[k] for k in ranked_keys[:limit]]
+
+
+def _queries_for(topic: str) -> dict[str, str]:
+    return {
+        "competitors": f"{topic} competitors",
+        "alternatives": f"{topic} alternatives",
+        "vs": f"{topic} vs",
+    }
+
+
+def discover_competitors(
+    topic: str,
+    count: int,
+    config: dict,
+    *,
+    lookback_days: int = 30,
+) -> list[str]:
+    """Discover `count` peer entities for `topic` via web search.
+
+    Args:
+        topic: The primary research topic.
+        count: Desired number of competitor entities (1..N).
+        config: Runtime config dict — expects the same shape as the engine
+            config (BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / etc.).
+        lookback_days: Date range for freshness. Defaults to 30.
+
+    Returns:
+        A list of up to `count` entity names, deduped and ordered by score.
+        Empty list when no web backend is configured or every search fails
+        or returns zero usable candidates.
+    """
+    if count < 1:
+        return []
+    if not _has_backend(config):
+        _log("No web search backend available, skipping competitor discovery")
+        return []
+
+    date_range = dates.get_date_range(lookback_days)
+    queries = _queries_for(topic)
+    collected: list[dict] = []
+    searches_run = 0
+
+    def _search(label: str, query: str) -> tuple[str, list[dict]]:
+        items, _artifact = grounding.web_search(query, date_range, config)
+        return label, items
+
+    with ThreadPoolExecutor(max_workers=len(queries)) as executor:
+        futures = {
+            executor.submit(_search, label, q): label
+            for label, q in queries.items()
+        }
+        for future in as_completed(futures):
+            label = futures[future]
+            try:
+                _label, items = future.result()
+                collected.extend(items)
+                searches_run += 1
+            except Exception as exc:
+                _log(f"Search failed for {label}: {exc}")
+
+    if not collected:
+        _log(f"No SERP results for {topic!r} across {searches_run}/{len(queries)} queries")
+        return []
+
+    entities = _extract_peer_entities(collected, topic, limit=count)
+    _log(
+        f"Discovered {len(entities)} competitor(s) for {topic!r} "
+        f"from {searches_run}/{len(queries)} queries: {entities}"
+    )
+    return entities