luckiest-co 1.0.0 → 1.0.3

package/skills/luckiest-trends/scripts/lib/techmeme.py

@@ -0,0 +1,239 @@
+"""Techmeme tech-news source for last30days.
+
+Shells out to ``techmeme-pp-cli`` (no auth) to surface current tech-news
+headlines relevant to a topic. Techmeme caches headlines locally, so the
+adapter ensures a one-time sync per run, then searches the cache.
+
+Activation gate: only available when ``techmeme-pp-cli`` is on PATH.
+``pipeline.available_sources`` checks ``shutil.which`` before including
+``techmeme``. The functions below also detect the missing-binary case.
+
+Surface choice: ``search "<topic>" --json`` (NOT ``--agent``). ``--agent``
+implies ``--compact``, which on older binaries stripped headline records to
+``{}`` (fixed upstream in printing-press-library PR #1383); ``--json`` without
+``--compact`` returns the populated ``{num, source, headline, link}`` shape on
+every binary version, so the adapter is robust regardless of the installed
+build.
+
+The search shape carries no per-item date or velocity, so headlines are dated
+to the sync time (Techmeme's cache is the current news cycle) and ranked on
+topic relevance plus source quality. Publication-name header rows (very short
+``headline`` values) are dropped.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any, Dict, List
+
+from . import log, subproc
+from .relevance import token_overlap_relevance
+
+
+CLI_BIN = "techmeme-pp-cli"
+
+DEPTH_CONFIG = {
+    "quick": 8,
+    "default": 16,
+    "deep": 30,
+}
+
+# A real story headline is a sentence; bare publication-name rows ("TechCrunch",
+# "New York Times") are section headers in the feed, not stories. Require at
+# least this many words to keep a record.
+MIN_HEADLINE_WORDS = 4
+
+SEARCH_TIMEOUT = 30
+SYNC_TIMEOUT = 40
+
+# Refresh the local cache at most once per TTL window, guarded by a lock so
+# concurrent fan-out threads do not double-sync (two processes writing the same
+# cache file). A monotonic timestamp -- not a bare bool -- so a long-lived host
+# process or multi-report fan-out re-syncs instead of serving a stale cache
+# indefinitely (headlines are dated to the sync time, so staleness would
+# misrepresent old news as current).
+_SYNC_LOCK = threading.Lock()
+_SYNC_TTL = 600.0  # seconds
+_LAST_SYNC = 0.0
+# Sentinel used by tests to force a sync regardless of the TTL window.
+_NEVER_SYNCED = float("-inf")
+
+
+def _log(msg: str) -> None:
+    log.source_log("Techmeme", msg, tty_only=False)
+
+
+def _is_available() -> bool:
+    """True when the techmeme-pp-cli binary is on PATH."""
+    return shutil.which(CLI_BIN) is not None
+
+
+def _today_iso() -> str:
+    return datetime.now(timezone.utc).date().isoformat()
+
+
+def _ensure_synced() -> None:
+    """Refresh the local headline cache at most once per TTL window.
+
+    Lock-guarded so concurrent fan-out threads do not double-sync. Best-effort:
+    a sync failure is logged and ignored (search can still run on an existing
+    cache), and the timestamp is stamped before the call so a failing sync does
+    not retry on every subquery within the window."""
+    global _LAST_SYNC
+    with _SYNC_LOCK:
+        if time.monotonic() - _LAST_SYNC < _SYNC_TTL:
+            return
+        _LAST_SYNC = time.monotonic()  # stamp first: a failure does not retry
+        try:
+            subproc.run_with_timeout([CLI_BIN, "sync", "--agent"], timeout=SYNC_TIMEOUT)
+            _log("synced headline cache")
+        except (subproc.SubprocTimeout, FileNotFoundError, OSError) as exc:
+            _log(f"sync skipped: {exc}")
+
+
+def _build_search_args(topic: str) -> List[str]:
+    # --json (not --agent) avoids --compact, which blanks headline records on
+    # pre-PR-1383 binaries. Techmeme's `search` has no result-limit flag, so the
+    # depth cap is applied client-side after parsing.
+    return [CLI_BIN, "search", topic, "--json"]
+
+
+def _coerce_list(data: Any) -> List[Dict[str, Any]]:
+    """Techmeme search returns a bare JSON array; tolerate a results-wrapped
+    envelope too."""
+    if isinstance(data, list):
+        return [r for r in data if isinstance(r, dict)]
+    if isinstance(data, dict):
+        results = data.get("results")
+        if isinstance(results, list):
+            return [r for r in results if isinstance(r, dict)]
+    return []
+
+
+def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
+    """Invoke techmeme-pp-cli and return ``{"results": [...records...]}``.
+    Never raises."""
+    if not _is_available():
+        return {"results": [], "error": f"{CLI_BIN} not on PATH"}
+    try:
+        result = subproc.run_with_timeout(cmd, timeout=timeout)
+    except subproc.SubprocTimeout as exc:
+        _log(f"Timeout: {exc}")
+        return {"results": [], "error": str(exc)}
+    except FileNotFoundError as exc:
+        _log(f"Binary missing: {exc}")
+        return {"results": [], "error": str(exc)}
+    except OSError as exc:
+        _log(f"Spawn failed: {exc}")
+        return {"results": [], "error": str(exc)}
+
+    if result.returncode != 0:
+        snippet = (result.stderr or "").strip().splitlines()[:1]
+        first = snippet[0] if snippet else f"exit {result.returncode}"
+        _log(f"CLI exit {result.returncode}: {first}")
+        return {"results": [], "error": first}
+
+    stdout = result.stdout or ""
+    if not stdout.strip():
+        return {"results": []}
+    try:
+        data = json.loads(stdout)
+    except json.JSONDecodeError as exc:
+        _log(f"JSON decode failed: {exc}")
+        return {"results": [], "error": f"json decode: {exc}"}
+
+    return {"results": _coerce_list(data)}
+
+
+def search_techmeme(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Search Techmeme headlines via techmeme-pp-cli.
+
+    Ensures a one-time cache sync, then searches. Returns a dict with a
+    ``results`` list of raw records. On failure, ``results`` is empty.
+    """
+    if not topic or not topic.strip():
+        return {"results": []}
+    if not _is_available():
+        return {"results": [], "error": f"{CLI_BIN} not on PATH"}
+    _ensure_synced()
+    limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    cmd = _build_search_args(topic)
+    _log(f"search '{topic}' (cap={limit})")
+    response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
+    # Techmeme returns all matches; apply the depth cap client-side.
+    records = response.get("results") or []
+    if isinstance(records, list) and len(records) > limit:
+        response["results"] = records[:limit]
+    _log(f"found {len(response.get('results') or [])} records")
+    return response
+
+
+def _is_story_headline(headline: str, source: str) -> bool:
+    """Reject bare publication-name header rows; keep sentence-shaped stories."""
+    if not headline:
+        return False
+    if len(headline.split()) < MIN_HEADLINE_WORDS:
+        return False
+    # A row whose headline is just the publication name is a header.
+    if source and headline.strip().lower() == source.strip().lower():
+        return False
+    return True
+
+
+def parse_techmeme_response(
+    response: Dict[str, Any],
+    query: str = "",
+) -> List[Dict[str, Any]]:
+    """Parse a Techmeme search envelope into normalized item dicts.
+
+    Drops publication-name header rows and records missing a link. Dates each
+    headline to today (the cache is the current news cycle) and computes a
+    token-overlap relevance hint. Returns dicts ready for
+    ``normalize._normalize_techmeme``.
+    """
+    raw = response.get("results") if isinstance(response, dict) else None
+    if not isinstance(raw, list):
+        return []
+
+    today = _today_iso()
+    items: List[Dict[str, Any]] = []
+    for i, rec in enumerate(raw):
+        if not isinstance(rec, dict):
+            continue
+        headline = " ".join(str(rec.get("headline") or "").split()).strip()
+        source_name = str(rec.get("source") or "").strip()
+        if not _is_story_headline(headline, source_name):
+            continue
+        link = str(rec.get("link") or "").strip()
+        if not link:
+            continue
+
+        rank_decay = max(0.3, 1.0 - (i * 0.03))
+        content_score = token_overlap_relevance(query, headline) if query else 0.5
+        relevance = min(1.0, 0.55 * rank_decay + 0.45 * content_score)
+
+        items.append(
+            {
+                "id": link,
+                "title": headline,
+                "url": link,
+                "source_name": source_name,
+                "date": today,
+                "engagement": {},
+                "relevance": round(relevance, 2),
+                "why_relevant": (
+                    f"Techmeme headline ({source_name})" if source_name else "Techmeme headline"
+                ),
+            }
+        )
+
+    return items

package/skills/luckiest-trends/scripts/lib/threads.py

@@ -0,0 +1,207 @@
+"""Threads keyword search via ScrapeCreators API for /last30days.
+
+Uses ScrapeCreators REST API to search Threads by keyword, extracting
+engagement metrics (likes, replies) from short text posts.
+
+Requires SCRAPECREATORS_API_KEY in config. Opt-in source via INCLUDE_SOURCES.
+API docs: https://scrapecreators.com/docs
+"""
+
+import math
+import re
+from typing import Any, Dict, List, Optional
+
+from . import dates, http, log
+from .relevance import token_overlap_relevance as _compute_relevance
+
+SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/threads"
+
+# Depth configurations: how many results to fetch
+DEPTH_CONFIG = {
+    "quick":   {"results": 10},
+    "default": {"results": 20},
+    "deep":    {"results": 40},
+}
+
+
+def _log(msg: str):
+    log.source_log("Threads", msg, tty_only=False)
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for Threads search.
+
+    The ScrapeCreators Threads keyword endpoint only returns hits for short
+    (1-2 word) queries; 3+ words or leaked boolean operators (the planner
+    emits "A OR B") return zero. Strip boolean operators and cap to the two
+    most salient words.
+    """
+    from .query import SOCIAL_NOISE, extract_core_subject
+    core = extract_core_subject(topic, noise=SOCIAL_NOISE, max_words=2)
+    return " ".join(core.rstrip("?!.").split()[:2])
+
+
+
+def _parse_date(item: Dict[str, Any]) -> Optional[str]:
+    """Parse date from Threads item to YYYY-MM-DD.
+
+    Tries common timestamp fields in order: taken_at and create_time
+    (unix timestamps in Meta APIs), then created_at, published_at, and
+    date (ISO 8601 strings). dates.parse_date() handles both.
+    """
+    for key in ("taken_at", "create_time", "created_at", "published_at", "date"):
+        val = item.get(key)
+        if val is None:
+            continue
+        dt = dates.parse_date(str(val))
+        if dt:
+            return dt.strftime("%Y-%m-%d")
+    return None
+
+
+def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
+    """Parse raw Threads items into normalized dicts."""
+    items = []
+    for i, raw in enumerate(raw_items):
+        post_id = str(
+            raw.get("id")
+            or raw.get("pk")
+            or raw.get("code")
+            or f"TH{i + 1}"
+        )
+        text = raw.get("text") or raw.get("caption") or raw.get("content") or ""
+        if isinstance(text, dict):
+            text = text.get("text", "")
+
+        # Author extraction
+        user = raw.get("user") or raw.get("author") or {}
+        if isinstance(user, dict):
+            handle = user.get("username") or user.get("handle") or ""
+            display_name = user.get("full_name") or user.get("displayName") or handle
+        elif isinstance(user, str):
+            handle = user
+            display_name = user
+        else:
+            handle = ""
+            display_name = ""
+
+        # Engagement metrics
+        likes = raw.get("like_count") or raw.get("likes") or 0
+        replies = raw.get("reply_count") or raw.get("replies") or 0
+        reposts = raw.get("repost_count") or raw.get("reposts") or 0
+        quotes = raw.get("quote_count") or raw.get("quotes") or 0
+
+        date_str = _parse_date(raw)
+
+        # Build URL
+        code = raw.get("code") or raw.get("shortcode") or ""
+        url = raw.get("url") or raw.get("share_url") or ""
+        if not url and code:
+            url = f"https://www.threads.net/post/{code}"
+        elif not url and handle and post_id:
+            url = f"https://www.threads.net/@{handle}/post/{post_id}"
+
+        # Relevance: position-based + engagement boost (similar to bluesky)
+        rank_score = max(0.3, 1.0 - (i * 0.02))
+        engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
+        text_relevance = _compute_relevance(core_topic, text)
+        relevance = min(1.0, text_relevance * 0.5 + rank_score * 0.3 + engagement_boost + 0.1)
+
+        items.append({
+            "id": post_id,
+            "handle": handle,
+            "display_name": display_name,
+            "text": text,
+            "url": url,
+            "date": date_str,
+            "engagement": {
+                "likes": likes,
+                "replies": replies,
+                "reposts": reposts,
+                "quotes": quotes,
+            },
+            "relevance": round(relevance, 2),
+            "why_relevant": f"Threads: @{handle}: {text[:60]}" if text else f"Threads: {handle}",
+        })
+    return items
+
+
+def search_threads(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = None,
+) -> Dict[str, Any]:
+    """Search Threads via ScrapeCreators API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+        token: ScrapeCreators API key
+
+    Returns:
+        Dict with 'items' list and optional 'error'.
+    """
+    if not token:
+        return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
+
+    config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    core_topic = _extract_core_subject(topic)
+
+    _log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
+
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/search",
+            params={"query": core_topic},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+
+    # Extract items from response (try common SC response shapes)
+    raw_items = (
+        data.get("items")
+        or data.get("data")
+        or data.get("threads")
+        or data.get("posts")
+        or data.get("search_results")
+        or []
+    )
+
+    # Limit to configured count
+    raw_items = raw_items[:config["results"]]
+
+    # Parse items
+    items = _parse_items(raw_items, core_topic)
+
+    # Date filter
+    in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
+    out_of_range = len(items) - len(in_range)
+    if in_range:
+        items = in_range
+        if out_of_range:
+            _log(f"Filtered {out_of_range} posts outside date range")
+    else:
+        _log(f"No posts within date range, keeping all {len(items)}")
+
+    # Sort by engagement (likes) descending
+    items.sort(key=lambda x: x["engagement"]["likes"], reverse=True)
+
+    _log(f"Found {len(items)} Threads posts")
+    return {"items": items}
+
+
+def parse_threads_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse Threads search response to normalized format.
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    return response.get("items", [])