@w3kits-com/plugin-opendesign 0.1.8 → 0.1.10

package/dist/__w3kits/assets/design-templates/last30days/scripts/lib/xai_x.py

@@ -0,0 +1,228 @@
+"""xAI API client for X (Twitter) discovery."""
+
+import json
+import re
+import sys
+from typing import Any, Dict, List, Optional
+
+from . import http, log
+
+
+def _safe_text(val) -> str:
+    """Extract text from string or localized object."""
+    if isinstance(val, str):
+        return val
+    if isinstance(val, dict):
+        return str(val.get("text", val.get("en", "")))
+    return str(val) if val is not None else ""
+
+
+def _log(msg: str):
+    log.source_log("xAI", msg, tty_only=False)
+
+
+def _log_error(msg: str):
+    log.source_log("xAI ERROR", msg, tty_only=False)
+
+# xAI uses responses endpoint with Agent Tools API
+XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
+
+# Depth configurations: (min, max) posts to request
+DEPTH_CONFIG = {
+    "quick": (8, 12),
+    "default": (20, 30),
+    "deep": (40, 60),
+}
+
+X_SEARCH_PROMPT = """You have access to real-time X (Twitter) data. Search for posts about: {topic}
+
+Focus on posts from {from_date} to {to_date}. Find {min_items}-{max_items} high-quality, relevant posts.
+
+IMPORTANT: Return ONLY valid JSON in this exact format, no other text:
+{{
+  "items": [
+    {{
+      "text": "Post text content (truncated if long)",
+      "url": "https://x.com/user/status/...",
+      "author_handle": "username",
+      "date": "YYYY-MM-DD or null if unknown",
+      "engagement": {{
+        "likes": 100,
+        "reposts": 25,
+        "replies": 15,
+        "quotes": 5
+      }},
+      "why_relevant": "Brief explanation of relevance",
+      "relevance": 0.85
+    }}
+  ]
+}}
+
+Rules:
+- relevance is 0.0 to 1.0 (1.0 = highly relevant)
+- date must be YYYY-MM-DD format or null
+- engagement can be null if unknown
+- Include diverse voices/accounts if applicable
+- Prefer posts with substantive content, not just links"""
+
+
+def search_x(
+    api_key: str,
+    model: str,
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    mock_response: Optional[Dict] = None,
+) -> Dict[str, Any]:
+    """Search X for relevant posts using xAI API with live search.
+
+    Args:
+        api_key: xAI API key
+        model: Model to use
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: Research depth - "quick", "default", or "deep"
+        mock_response: Mock response for testing
+
+    Returns:
+        Raw API response
+    """
+    if mock_response is not None:
+        return mock_response
+
+    min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    # Adjust timeout based on depth (generous for API response time)
+    timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
+
+    # Use Agent Tools API with x_search tool (native date filtering)
+    payload = {
+        "model": model,
+        "tools": [
+            {"type": "x_search", "from_date": from_date, "to_date": to_date}
+        ],
+        "input": [
+            {
+                "role": "user",
+                "content": X_SEARCH_PROMPT.format(
+                    topic=topic,
+                    from_date=from_date,
+                    to_date=to_date,
+                    min_items=min_items,
+                    max_items=max_items,
+                ),
+            }
+        ],
+    }
+
+    return http.post(XAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
+
+
+def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse xAI response to extract X items.
+
+    Args:
+        response: Raw API response
+
+    Returns:
+        List of item dicts
+    """
+    items = []
+
+    # Check for API errors first
+    if "error" in response and response["error"]:
+        error = response["error"]
+        err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
+        _log_error(f"xAI API error: {err_msg}")
+        if http.DEBUG:
+            _log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
+        return items
+
+    # Try to find the output text
+    output_text = ""
+    if "output" in response:
+        output = response["output"]
+        if isinstance(output, str):
+            output_text = output
+        elif isinstance(output, list):
+            for item in output:
+                if isinstance(item, dict):
+                    if item.get("type") == "message":
+                        content = item.get("content", [])
+                        for c in content:
+                            if isinstance(c, dict) and c.get("type") == "output_text":
+                                output_text = c.get("text", "")
+                                break
+                    elif "text" in item:
+                        output_text = item["text"]
+                elif isinstance(item, str):
+                    output_text = item
+                if output_text:
+                    break
+
+    # Also check for choices (older format)
+    if not output_text and "choices" in response:
+        for choice in response["choices"]:
+            if "message" in choice:
+                output_text = choice["message"].get("content", "")
+                break
+
+    if not output_text:
+        return items
+
+    # Extract JSON from the response
+    json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
+    if json_match:
+        try:
+            data = json.loads(json_match.group())
+            items = data.get("items", [])
+        except json.JSONDecodeError:
+            _log(f"Failed to parse xAI response JSON: {output_text[:200]}")
+
+    # Validate and clean items
+    clean_items = []
+    for i, item in enumerate(items):
+        if not isinstance(item, dict):
+            continue
+
+        url = item.get("url", "")
+        if not url:
+            continue
+
+        # Parse engagement
+        engagement = None
+        eng_raw = item.get("engagement")
+        if isinstance(eng_raw, dict):
+            engagement = {
+                "likes": int(eng_raw["likes"]) if eng_raw.get("likes") is not None else None,
+                "reposts": int(eng_raw["reposts"]) if eng_raw.get("reposts") is not None else None,
+                "replies": int(eng_raw["replies"]) if eng_raw.get("replies") is not None else None,
+                "quotes": int(eng_raw["quotes"]) if eng_raw.get("quotes") is not None else None,
+            }
+
+        clean_item = {
+            "id": f"X{i+1}",
+            "text": _safe_text(item.get("text", "")).strip()[:500],  # Truncate long text
+            "url": url,
+            "author_handle": _safe_text(item.get("author_handle", "")).strip().lstrip("@"),
+            "date": item.get("date"),
+            "engagement": engagement,
+            "why_relevant": _safe_text(item.get("why_relevant", "")).strip(),
+            "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
+        }
+
+        # Validate date format
+        if clean_item["date"]:
+            if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])):
+                clean_item["date"] = None
+
+        clean_items.append(clean_item)
+
+    return clean_items

package/dist/__w3kits/assets/design-templates/last30days/scripts/lib/xiaohongshu_api.py

@@ -0,0 +1,162 @@
+"""Xiaohongshu HTTP API search client for last30days.
+
+Uses xpzouying/xiaohongshu-mcp REST endpoints:
+- GET/POST /api/v1/feeds/search
+- GET /api/v1/login/status
+"""
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+
+def _to_int(value: Any) -> int:
+    """Convert Xiaohongshu count strings to int.
+
+    Supports plain ints and Chinese suffixes like 1.2万 / 3亿.
+    """
+    if value is None:
+        return 0
+    if isinstance(value, (int, float)):
+        return int(value)
+
+    text = str(value).strip().lower().replace(",", "")
+    if not text:
+        return 0
+
+    try:
+        if text.endswith("万"):
+            return int(float(text[:-1]) * 10000)
+        if text.endswith("亿"):
+            return int(float(text[:-1]) * 100000000)
+        return int(float(text))
+    except (TypeError, ValueError):
+        return 0
+
+
+def _timestamp_to_date_ms(ts: Any) -> Optional[str]:
+    """Convert millisecond timestamp to YYYY-MM-DD."""
+    try:
+        iv = int(ts)
+        if iv <= 0:
+            return None
+        # API examples use milliseconds.
+        dt = datetime.fromtimestamp(iv / 1000.0, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d")
+    except (TypeError, ValueError, OSError):
+        return None
+
+
+def _relevance_from_interactions(likes: int, comments: int, favorites: int) -> float:
+    """Heuristic relevance score from engagement metrics."""
+    # Weighted engagement with soft caps to [0, 1].
+    weighted = (likes * 1.0) + (comments * 2.5) + (favorites * 1.5)
+    # 5000 weighted engagement ~= strong relevance.
+    score = min(1.0, max(0.05, weighted / 5000.0))
+    return round(score, 3)
+
+
+def _build_note_url(feed_id: str, xsec_token: str) -> str:
+    """Build a stable Xiaohongshu note URL."""
+    if xsec_token:
+        return f"https://www.xiaohongshu.com/explore/{feed_id}?xsec_token={xsec_token}"
+    return f"https://www.xiaohongshu.com/explore/{feed_id}"
+
+
+def search_feeds(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    base_url: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search Xiaohongshu feeds and normalize to web-item shape."""
+    base = (base_url or "").rstrip("/")
+    if not base:
+        raise ValueError("Missing Xiaohongshu API base URL")
+
+    # Quick login sanity check.
+    login = http.get(f"{base}/api/v1/login/status", timeout=8, retries=1)
+    is_logged_in = (
+        login.get("data", {}).get("is_logged_in")
+        if isinstance(login, dict) else False
+    )
+    if not is_logged_in:
+        raise http.HTTPError("Xiaohongshu API reachable but not logged in")
+
+    # API supports filters; use recency-oriented defaults.
+    publish_time = "一天内" if depth == "quick" else "一周内" if depth == "default" else "半年内"
+    payload = {
+        "keyword": topic,
+        "filters": {
+            "sort_by": "综合",
+            "note_type": "不限",
+            "publish_time": publish_time,
+            "search_scope": "不限",
+            "location": "不限",
+        },
+    }
+
+    resp = http.post(f"{base}/api/v1/feeds/search", payload, timeout=20, retries=1)
+    feeds = resp.get("data", {}).get("feeds", []) if isinstance(resp, dict) else []
+    if not isinstance(feeds, list):
+        feeds = []
+
+    # Cap source volume similarly to other web sources.
+    limit = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
+    items: List[Dict[str, Any]] = []
+
+    for i, feed in enumerate(feeds[:limit]):
+        if not isinstance(feed, dict):
+            continue
+        note = feed.get("noteCard") or {}
+        if not isinstance(note, dict):
+            note = {}
+        interact = note.get("interactInfo") or {}
+        if not isinstance(interact, dict):
+            interact = {}
+
+        feed_id = str(feed.get("id") or note.get("noteId") or "").strip()
+        if not feed_id:
+            continue
+
+        xsec_token = str(feed.get("xsecToken") or note.get("xsecToken") or "").strip()
+        title = str(
+            note.get("displayTitle")
+            or note.get("title")
+            or ""
+        ).strip()
+        snippet = str(
+            note.get("desc")
+            or note.get("displayDesc")
+            or title
+            or ""
+        ).strip()
+
+        likes = _to_int(interact.get("likedCount"))
+        comments = _to_int(interact.get("commentCount"))
+        favorites = _to_int(interact.get("collectedCount"))
+
+        date_value = _timestamp_to_date_ms(note.get("time"))
+        why = f"Xiaohongshu engagement: likes={likes}, comments={comments}, favorites={favorites}"
+
+        items.append({
+            "id": f"XHS{i+1}",
+            "title": title[:200] if title else f"Xiaohongshu note {feed_id}",
+            "url": _build_note_url(feed_id, xsec_token),
+            "source_domain": "xiaohongshu.com",
+            "snippet": snippet[:500],
+            "date": date_value,
+            "date_confidence": "high" if date_value else "low",
+            "relevance": _relevance_from_interactions(likes, comments, favorites),
+            "why_relevant": why,
+            # Keep raw engagement for debugging/possible future rendering.
+            "engagement": {
+                "likes": likes,
+                "comments": comments,
+                "favorites": favorites,
+            },
+        })
+
+    return items

package/dist/__w3kits/assets/design-templates/last30days/scripts/lib/xquik.py

@@ -0,0 +1,227 @@
+"""Xquik X search source for the v3.0.0 last30days pipeline.
+
+Uses the Xquik REST API (https://xquik.com/api/v1) to search X/Twitter
+with full engagement metrics (likes, retweets, replies, quotes, views,
+bookmarks). Requires an API key from xquik.com.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, List
+
+from . import http, log
+from .relevance import token_overlap_relevance as _compute_relevance
+
+# Depth configurations: number of results to request per query
+DEPTH_CONFIG = {
+    "quick": {"limit": 10, "queries": 1},
+    "default": {"limit": 20, "queries": 2},
+    "deep": {"limit": 40, "queries": 3},
+}
+
+_BASE_URL = "https://xquik.com/api/v1"
+
+
+def _log(msg: str):
+    log.source_log("Xquik", msg, tty_only=False)
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject for X search queries."""
+    from .query import extract_core_subject
+    return extract_core_subject(topic, max_words=5, strip_suffixes=True)
+
+
+def expand_xquik_queries(topic: str, depth: str) -> List[str]:
+    """Generate query variants based on depth.
+
+    Args:
+        topic: Research topic
+        depth: "quick", "default", or "deep"
+
+    Returns:
+        List of query strings (1 for quick, 2 for default, 3 for deep).
+    """
+    core = _extract_core_subject(topic)
+    queries = [core]
+
+    # Add original topic if meaningfully different
+    if topic.lower().strip() != core.lower().strip():
+        queries.append(topic.strip())
+
+    # Add compound term variant for deep searches
+    if len(queries) < 3:
+        from .query import extract_compound_terms
+        compounds = extract_compound_terms(topic)
+        if compounds:
+            or_parts = " OR ".join(f'"{t}"' for t in compounds[:3])
+            queries.append(f"({or_parts})")
+
+    cap = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])["queries"]
+    return queries[:cap]
+
+
+def search_xquik(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = "",
+) -> Dict[str, Any]:
+    """Search X via Xquik REST API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: Research depth - "quick", "default", or "deep"
+        token: Xquik API key
+
+    Returns:
+        Dict with "items" list and optional "error" string.
+    """
+    if not token:
+        return {"items": [], "error": "No XQUIK_API_KEY configured"}
+
+    cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    queries = expand_xquik_queries(topic, depth)
+    all_items: List[Dict[str, Any]] = []
+    seen_ids: set[str] = set()
+
+    for query_text in queries:
+        try:
+            url = f"{_BASE_URL}/x/tweets/search"
+            # Build query with date filter
+            q = f"{query_text} since:{from_date} until:{to_date}"
+            params = f"q={_url_encode(q)}&queryType=Top&limit={cfg['limit']}"
+            full_url = f"{url}?{params}"
+
+            _log(f"Searching: {query_text}")
+            response = http.get(
+                full_url,
+                headers={"X-Api-Key": token},
+                timeout=30,
+                retries=2,
+            )
+
+            tweets = response.get("tweets", [])
+            if not isinstance(tweets, list):
+                continue
+
+            for i, tweet in enumerate(tweets):
+                if not isinstance(tweet, dict):
+                    continue
+                tweet_id = str(tweet.get("id", ""))
+                if tweet_id in seen_ids:
+                    continue
+                seen_ids.add(tweet_id)
+
+                item = _parse_tweet(tweet, i + len(all_items), query_text)
+                if item:
+                    all_items.append(item)
+
+        except http.HTTPError as exc:
+            status = getattr(exc, "status_code", None)
+            if status in (401, 403):
+                return {"items": [], "error": f"Xquik auth failed ({status})"}
+            _log(f"HTTP error for query '{query_text}': {exc}")
+        except Exception as exc:
+            _log(f"Error for query '{query_text}': {exc}")
+
+    return {"items": all_items}
+
+
+def search_and_enrich(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = "",
+) -> Dict[str, Any]:
+    """Search X via Xquik and return results.
+
+    Xquik API returns full engagement data by default, so no separate
+    enrichment step is needed.
+    """
+    return search_xquik(topic, from_date, to_date, depth=depth, token=token)
+
+
+def parse_xquik_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extract items from search response.
+
+    Args:
+        response: Response dict from search_xquik()
+
+    Returns:
+        List of normalized item dicts.
+    """
+    return response.get("items", [])
+
+
+def _parse_tweet(
+    tweet: Dict[str, Any], index: int, query: str
+) -> Dict[str, Any] | None:
+    """Parse a single tweet from the API response into the standard item format."""
+    author = tweet.get("author") or {}
+    username = str(author.get("username", "")).lstrip("@")
+    tweet_id = str(tweet.get("id", ""))
+
+    # Build URL
+    url = ""
+    if username and tweet_id:
+        url = f"https://x.com/{username}/status/{tweet_id}"
+    if not url:
+        return None
+
+    # Parse date
+    date = None
+    created_at = tweet.get("createdAt") or ""
+    if created_at:
+        try:
+            if len(created_at) > 10 and created_at[10] == "T":
+                dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+            else:
+                dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
+            date = dt.strftime("%Y-%m-%d")
+        except (ValueError, TypeError):
+            pass
+
+    text = str(tweet.get("text", "")).strip()[:500]
+
+    # Build engagement dict with full metrics
+    engagement = {
+        "likes": _safe_int(tweet.get("likeCount")),
+        "reposts": _safe_int(tweet.get("retweetCount")),
+        "replies": _safe_int(tweet.get("replyCount")),
+        "quotes": _safe_int(tweet.get("quoteCount")),
+        "views": _safe_int(tweet.get("viewCount")),
+        "bookmarks": _safe_int(tweet.get("bookmarkCount")),
+    }
+
+    return {
+        "id": f"XQ{index + 1}",
+        "text": text,
+        "url": url,
+        "author_handle": username,
+        "date": date,
+        "engagement": engagement,
+        "relevance": _compute_relevance(query, text) if query else 0.7,
+        "why_relevant": "",
+    }
+
+
+def _safe_int(value: Any) -> int | None:
+    """Convert value to int, returning None on failure."""
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _url_encode(text: str) -> str:
+    """URL-encode a string using stdlib."""
+    from urllib.parse import quote
+    return quote(text, safe="")