bps-kit 1.0.1 → 1.0.2

package/templates/vault/last30days/scripts/lib/websearch.py

@@ -0,0 +1,401 @@
+"""WebSearch module for last30days skill.
+
+NOTE: WebSearch uses Claude's built-in WebSearch tool, which runs INSIDE Claude Code.
+Unlike Reddit/X which use external APIs, WebSearch results are obtained by Claude
+directly and passed to this module for normalization and scoring.
+
+The typical flow is:
+1. Claude invokes WebSearch tool with the topic
+2. Claude passes results to parse_websearch_results()
+3. Results are normalized into WebSearchItem objects
+"""
+
+import re
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import urlparse
+
+from . import schema
+
+
+# Month name mappings for date parsing
+MONTH_MAP = {
+    "jan": 1, "january": 1,
+    "feb": 2, "february": 2,
+    "mar": 3, "march": 3,
+    "apr": 4, "april": 4,
+    "may": 5,
+    "jun": 6, "june": 6,
+    "jul": 7, "july": 7,
+    "aug": 8, "august": 8,
+    "sep": 9, "sept": 9, "september": 9,
+    "oct": 10, "october": 10,
+    "nov": 11, "november": 11,
+    "dec": 12, "december": 12,
+}
+
+
+def extract_date_from_url(url: str) -> Optional[str]:
+    """Try to extract a date from URL path.
+
+    Many sites embed dates in URLs like:
+    - /2026/01/24/article-title
+    - /2026-01-24/article
+    - /blog/20260124/title
+
+    Args:
+        url: URL to parse
+
+    Returns:
+        Date string in YYYY-MM-DD format, or None
+    """
+    # Pattern 1: /YYYY/MM/DD/ (most common)
+    match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', url)
+    if match:
+        year, month, day = match.groups()
+        if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
+            return f"{year}-{month}-{day}"
+
+    # Pattern 2: /YYYY-MM-DD/ or /YYYY-MM-DD-
+    match = re.search(r'/(\d{4})-(\d{2})-(\d{2})[-/]', url)
+    if match:
+        year, month, day = match.groups()
+        if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
+            return f"{year}-{month}-{day}"
+
+    # Pattern 3: /YYYYMMDD/ (compact)
+    match = re.search(r'/(\d{4})(\d{2})(\d{2})/', url)
+    if match:
+        year, month, day = match.groups()
+        if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
+            return f"{year}-{month}-{day}"
+
+    return None
+
+
+def extract_date_from_snippet(text: str) -> Optional[str]:
+    """Try to extract a date from text snippet or title.
+
+    Looks for patterns like:
+    - January 24, 2026 or Jan 24, 2026
+    - 24 January 2026
+    - 2026-01-24
+    - "3 days ago", "yesterday", "last week"
+
+    Args:
+        text: Text to parse
+
+    Returns:
+        Date string in YYYY-MM-DD format, or None
+    """
+    if not text:
+        return None
+
+    text_lower = text.lower()
+
+    # Pattern 1: Month DD, YYYY (e.g., "January 24, 2026")
+    match = re.search(
+        r'\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
+        r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
+        r'\s+(\d{1,2})(?:st|nd|rd|th)?,?\s*(\d{4})\b',
+        text_lower
+    )
+    if match:
+        month_str, day, year = match.groups()
+        month = MONTH_MAP.get(month_str[:3])
+        if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
+            return f"{year}-{month:02d}-{int(day):02d}"
+
+    # Pattern 2: DD Month YYYY (e.g., "24 January 2026")
+    match = re.search(
+        r'\b(\d{1,2})(?:st|nd|rd|th)?\s+'
+        r'(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
+        r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
+        r'\s+(\d{4})\b',
+        text_lower
+    )
+    if match:
+        day, month_str, year = match.groups()
+        month = MONTH_MAP.get(month_str[:3])
+        if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
+            return f"{year}-{month:02d}-{int(day):02d}"
+
+    # Pattern 3: YYYY-MM-DD (ISO format)
+    match = re.search(r'\b(\d{4})-(\d{2})-(\d{2})\b', text)
+    if match:
+        year, month, day = match.groups()
+        if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
+            return f"{year}-{month}-{day}"
+
+    # Pattern 4: Relative dates ("3 days ago", "yesterday", etc.)
+    today = datetime.now()
+
+    if "yesterday" in text_lower:
+        date = today - timedelta(days=1)
+        return date.strftime("%Y-%m-%d")
+
+    if "today" in text_lower:
+        return today.strftime("%Y-%m-%d")
+
+    # "N days ago"
+    match = re.search(r'\b(\d+)\s*days?\s*ago\b', text_lower)
+    if match:
+        days = int(match.group(1))
+        if days <= 60:  # Reasonable range
+            date = today - timedelta(days=days)
+            return date.strftime("%Y-%m-%d")
+
+    # "N hours ago" -> today
+    match = re.search(r'\b(\d+)\s*hours?\s*ago\b', text_lower)
+    if match:
+        return today.strftime("%Y-%m-%d")
+
+    # "last week" -> ~7 days ago
+    if "last week" in text_lower:
+        date = today - timedelta(days=7)
+        return date.strftime("%Y-%m-%d")
+
+    # "this week" -> ~3 days ago (middle of week)
+    if "this week" in text_lower:
+        date = today - timedelta(days=3)
+        return date.strftime("%Y-%m-%d")
+
+    return None
+
+
+def extract_date_signals(
+    url: str,
+    snippet: str,
+    title: str,
+) -> Tuple[Optional[str], str]:
+    """Extract date from any available signal.
+
+    Tries URL first (most reliable), then snippet, then title.
+
+    Args:
+        url: Page URL
+        snippet: Page snippet/description
+        title: Page title
+
+    Returns:
+        Tuple of (date_string, confidence)
+        - date from URL: 'high' confidence
+        - date from snippet/title: 'med' confidence
+        - no date found: None, 'low' confidence
+    """
+    # Try URL first (most reliable)
+    url_date = extract_date_from_url(url)
+    if url_date:
+        return url_date, "high"
+
+    # Try snippet
+    snippet_date = extract_date_from_snippet(snippet)
+    if snippet_date:
+        return snippet_date, "med"
+
+    # Try title
+    title_date = extract_date_from_snippet(title)
+    if title_date:
+        return title_date, "med"
+
+    return None, "low"
+
+
+# Domains to exclude (Reddit and X are handled separately)
+EXCLUDED_DOMAINS = {
+    "reddit.com",
+    "www.reddit.com",
+    "old.reddit.com",
+    "twitter.com",
+    "www.twitter.com",
+    "x.com",
+    "www.x.com",
+    "mobile.twitter.com",
+}
+
+
+def extract_domain(url: str) -> str:
+    """Extract the domain from a URL.
+
+    Args:
+        url: Full URL
+
+    Returns:
+        Domain string (e.g., "medium.com")
+    """
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        # Remove www. prefix for cleaner display
+        if domain.startswith("www."):
+            domain = domain[4:]
+        return domain
+    except Exception:
+        return ""
+
+
+def is_excluded_domain(url: str) -> bool:
+    """Check if URL is from an excluded domain (Reddit/X).
+
+    Args:
+        url: URL to check
+
+    Returns:
+        True if URL should be excluded
+    """
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        return domain in EXCLUDED_DOMAINS
+    except Exception:
+        return False
+
+
+def parse_websearch_results(
+    results: List[Dict[str, Any]],
+    topic: str,
+    from_date: str = "",
+    to_date: str = "",
+) -> List[Dict[str, Any]]:
+    """Parse WebSearch results into normalized format.
+
+    This function expects results from Claude's WebSearch tool.
+    Each result should have: title, url, snippet, and optionally date/relevance.
+
+    Uses "Date Detective" approach:
+    1. Extract dates from URLs (high confidence)
+    2. Extract dates from snippets/titles (med confidence)
+    3. Hard filter: exclude items with verified old dates
+    4. Keep items with no date signals (with low confidence penalty)
+
+    Args:
+        results: List of WebSearch result dicts
+        topic: Original search topic (for context)
+        from_date: Start date for filtering (YYYY-MM-DD)
+        to_date: End date for filtering (YYYY-MM-DD)
+
+    Returns:
+        List of normalized item dicts ready for WebSearchItem creation
+    """
+    items = []
+
+    for i, result in enumerate(results):
+        if not isinstance(result, dict):
+            continue
+
+        url = result.get("url", "")
+        if not url:
+            continue
+
+        # Skip Reddit/X URLs (handled separately)
+        if is_excluded_domain(url):
+            continue
+
+        title = str(result.get("title", "")).strip()
+        snippet = str(result.get("snippet", result.get("description", ""))).strip()
+
+        if not title and not snippet:
+            continue
+
+        # Use Date Detective to extract date signals
+        date = result.get("date")  # Use provided date if available
+        date_confidence = "low"
+
+        if date and re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)):
+            # Provided date is valid
+            date_confidence = "med"
+        else:
+            # Try to extract date from URL/snippet/title
+            extracted_date, confidence = extract_date_signals(url, snippet, title)
+            if extracted_date:
+                date = extracted_date
+                date_confidence = confidence
+
+        # Hard filter: if we found a date and it's too old, skip
+        if date and from_date and date < from_date:
+            continue  # DROP - verified old content
+
+        # Hard filter: if date is in the future, skip (parsing error)
+        if date and to_date and date > to_date:
+            continue  # DROP - future date
+
+        # Get relevance if provided, default to 0.5
+        relevance = result.get("relevance", 0.5)
+        try:
+            relevance = min(1.0, max(0.0, float(relevance)))
+        except (TypeError, ValueError):
+            relevance = 0.5
+
+        item = {
+            "id": f"W{i+1}",
+            "title": title[:200],  # Truncate long titles
+            "url": url,
+            "source_domain": extract_domain(url),
+            "snippet": snippet[:500],  # Truncate long snippets
+            "date": date,
+            "date_confidence": date_confidence,
+            "relevance": relevance,
+            "why_relevant": str(result.get("why_relevant", "")).strip(),
+        }
+
+        items.append(item)
+
+    return items
+
+
+def normalize_websearch_items(
+    items: List[Dict[str, Any]],
+    from_date: str,
+    to_date: str,
+) -> List[schema.WebSearchItem]:
+    """Convert parsed dicts to WebSearchItem objects.
+
+    Args:
+        items: List of parsed item dicts
+        from_date: Start of date range (YYYY-MM-DD)
+        to_date: End of date range (YYYY-MM-DD)
+
+    Returns:
+        List of WebSearchItem objects
+    """
+    result = []
+
+    for item in items:
+        web_item = schema.WebSearchItem(
+            id=item["id"],
+            title=item["title"],
+            url=item["url"],
+            source_domain=item["source_domain"],
+            snippet=item["snippet"],
+            date=item.get("date"),
+            date_confidence=item.get("date_confidence", "low"),
+            relevance=item.get("relevance", 0.5),
+            why_relevant=item.get("why_relevant", ""),
+        )
+        result.append(web_item)
+
+    return result
+
+
+def dedupe_websearch(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
+    """Remove duplicate WebSearch items.
+
+    Deduplication is based on URL.
+
+    Args:
+        items: List of WebSearchItem objects
+
+    Returns:
+        Deduplicated list
+    """
+    seen_urls = set()
+    result = []
+
+    for item in items:
+        # Normalize URL for comparison
+        url_key = item.url.lower().rstrip("/")
+        if url_key not in seen_urls:
+            seen_urls.add(url_key)
+            result.append(item)
+
+    return result

package/templates/vault/last30days/scripts/lib/xai_x.py

@@ -0,0 +1,217 @@
+"""xAI API client for X (Twitter) discovery."""
+
+import json
+import re
+import sys
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+
+def _log_error(msg: str):
+    """Log error to stderr."""
+    sys.stderr.write(f"[X ERROR] {msg}\n")
+    sys.stderr.flush()
+
+# xAI uses responses endpoint with Agent Tools API
+XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
+
+# Depth configurations: (min, max) posts to request
+DEPTH_CONFIG = {
+    "quick": (8, 12),
+    "default": (20, 30),
+    "deep": (40, 60),
+}
+
+X_SEARCH_PROMPT = """You have access to real-time X (Twitter) data. Search for posts about: {topic}
+
+Focus on posts from {from_date} to {to_date}. Find {min_items}-{max_items} high-quality, relevant posts.
+
+IMPORTANT: Return ONLY valid JSON in this exact format, no other text:
+{{
+  "items": [
+    {{
+      "text": "Post text content (truncated if long)",
+      "url": "https://x.com/user/status/...",
+      "author_handle": "username",
+      "date": "YYYY-MM-DD or null if unknown",
+      "engagement": {{
+        "likes": 100,
+        "reposts": 25,
+        "replies": 15,
+        "quotes": 5
+      }},
+      "why_relevant": "Brief explanation of relevance",
+      "relevance": 0.85
+    }}
+  ]
+}}
+
+Rules:
+- relevance is 0.0 to 1.0 (1.0 = highly relevant)
+- date must be YYYY-MM-DD format or null
+- engagement can be null if unknown
+- Include diverse voices/accounts if applicable
+- Prefer posts with substantive content, not just links"""
+
+
+def search_x(
+    api_key: str,
+    model: str,
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    mock_response: Optional[Dict] = None,
+) -> Dict[str, Any]:
+    """Search X for relevant posts using xAI API with live search.
+
+    Args:
+        api_key: xAI API key
+        model: Model to use
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: Research depth - "quick", "default", or "deep"
+        mock_response: Mock response for testing
+
+    Returns:
+        Raw API response
+    """
+    if mock_response is not None:
+        return mock_response
+
+    min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    # Adjust timeout based on depth (generous for API response time)
+    timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
+
+    # Use Agent Tools API with x_search tool
+    payload = {
+        "model": model,
+        "tools": [
+            {"type": "x_search"}
+        ],
+        "input": [
+            {
+                "role": "user",
+                "content": X_SEARCH_PROMPT.format(
+                    topic=topic,
+                    from_date=from_date,
+                    to_date=to_date,
+                    min_items=min_items,
+                    max_items=max_items,
+                ),
+            }
+        ],
+    }
+
+    return http.post(XAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
+
+
+def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse xAI response to extract X items.
+
+    Args:
+        response: Raw API response
+
+    Returns:
+        List of item dicts
+    """
+    items = []
+
+    # Check for API errors first
+    if "error" in response and response["error"]:
+        error = response["error"]
+        err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
+        _log_error(f"xAI API error: {err_msg}")
+        if http.DEBUG:
+            _log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
+        return items
+
+    # Try to find the output text
+    output_text = ""
+    if "output" in response:
+        output = response["output"]
+        if isinstance(output, str):
+            output_text = output
+        elif isinstance(output, list):
+            for item in output:
+                if isinstance(item, dict):
+                    if item.get("type") == "message":
+                        content = item.get("content", [])
+                        for c in content:
+                            if isinstance(c, dict) and c.get("type") == "output_text":
+                                output_text = c.get("text", "")
+                                break
+                    elif "text" in item:
+                        output_text = item["text"]
+                elif isinstance(item, str):
+                    output_text = item
+                if output_text:
+                    break
+
+    # Also check for choices (older format)
+    if not output_text and "choices" in response:
+        for choice in response["choices"]:
+            if "message" in choice:
+                output_text = choice["message"].get("content", "")
+                break
+
+    if not output_text:
+        return items
+
+    # Extract JSON from the response
+    json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
+    if json_match:
+        try:
+            data = json.loads(json_match.group())
+            items = data.get("items", [])
+        except json.JSONDecodeError:
+            pass
+
+    # Validate and clean items
+    clean_items = []
+    for i, item in enumerate(items):
+        if not isinstance(item, dict):
+            continue
+
+        url = item.get("url", "")
+        if not url:
+            continue
+
+        # Parse engagement
+        engagement = None
+        eng_raw = item.get("engagement")
+        if isinstance(eng_raw, dict):
+            engagement = {
+                "likes": int(eng_raw.get("likes", 0)) if eng_raw.get("likes") else None,
+                "reposts": int(eng_raw.get("reposts", 0)) if eng_raw.get("reposts") else None,
+                "replies": int(eng_raw.get("replies", 0)) if eng_raw.get("replies") else None,
+                "quotes": int(eng_raw.get("quotes", 0)) if eng_raw.get("quotes") else None,
+            }
+
+        clean_item = {
+            "id": f"X{i+1}",
+            "text": str(item.get("text", "")).strip()[:500],  # Truncate long text
+            "url": url,
+            "author_handle": str(item.get("author_handle", "")).strip().lstrip("@"),
+            "date": item.get("date"),
+            "engagement": engagement,
+            "why_relevant": str(item.get("why_relevant", "")).strip(),
+            "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
+        }
+
+        # Validate date format
+        if clean_item["date"]:
+            if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])):
+                clean_item["date"] = None
+
+        clean_items.append(clean_item)
+
+    return clean_items