bps-kit 1.0.1 → 1.0.2

package/templates/vault/last30days/scripts/lib/models.py

@@ -0,0 +1,175 @@
+"""Model auto-selection for last30days skill."""
+
+import re
+from typing import Dict, List, Optional, Tuple
+
+from . import cache, http
+
+# OpenAI API
+OPENAI_MODELS_URL = "https://api.openai.com/v1/models"
+OPENAI_FALLBACK_MODELS = ["gpt-5.2", "gpt-5.1", "gpt-5", "gpt-4o"]
+
+# xAI API - Agent Tools API requires grok-4 family
+XAI_MODELS_URL = "https://api.x.ai/v1/models"
+XAI_ALIASES = {
+    "latest": "grok-4-1-fast",  # Required for x_search tool
+    "stable": "grok-4-1-fast",
+}
+
+
+def parse_version(model_id: str) -> Optional[Tuple[int, ...]]:
+    """Parse semantic version from model ID.
+
+    Examples:
+        gpt-5 -> (5,)
+        gpt-5.2 -> (5, 2)
+        gpt-5.2.1 -> (5, 2, 1)
+    """
+    match = re.search(r'(\d+(?:\.\d+)*)', model_id)
+    if match:
+        return tuple(int(x) for x in match.group(1).split('.'))
+    return None
+
+
+def is_mainline_openai_model(model_id: str) -> bool:
+    """Check if model is a mainline GPT model (not mini/nano/chat/codex/pro)."""
+    model_lower = model_id.lower()
+
+    # Must be gpt-5 series
+    if not re.match(r'^gpt-5(\.\d+)*$', model_lower):
+        return False
+
+    # Exclude variants
+    excludes = ['mini', 'nano', 'chat', 'codex', 'pro', 'preview', 'turbo']
+    for exc in excludes:
+        if exc in model_lower:
+            return False
+
+    return True
+
+
+def select_openai_model(
+    api_key: str,
+    policy: str = "auto",
+    pin: Optional[str] = None,
+    mock_models: Optional[List[Dict]] = None,
+) -> str:
+    """Select the best OpenAI model based on policy.
+
+    Args:
+        api_key: OpenAI API key
+        policy: 'auto' or 'pinned'
+        pin: Model to use if policy is 'pinned'
+        mock_models: Mock model list for testing
+
+    Returns:
+        Selected model ID
+    """
+    if policy == "pinned" and pin:
+        return pin
+
+    # Check cache first
+    cached = cache.get_cached_model("openai")
+    if cached:
+        return cached
+
+    # Fetch model list
+    if mock_models is not None:
+        models = mock_models
+    else:
+        try:
+            headers = {"Authorization": f"Bearer {api_key}"}
+            response = http.get(OPENAI_MODELS_URL, headers=headers)
+            models = response.get("data", [])
+        except http.HTTPError:
+            # Fall back to known models
+            return OPENAI_FALLBACK_MODELS[0]
+
+    # Filter to mainline models
+    candidates = [m for m in models if is_mainline_openai_model(m.get("id", ""))]
+
+    if not candidates:
+        # No gpt-5 models found, use fallback
+        return OPENAI_FALLBACK_MODELS[0]
+
+    # Sort by version (descending), then by created timestamp
+    def sort_key(m):
+        version = parse_version(m.get("id", "")) or (0,)
+        created = m.get("created", 0)
+        return (version, created)
+
+    candidates.sort(key=sort_key, reverse=True)
+    selected = candidates[0]["id"]
+
+    # Cache the selection
+    cache.set_cached_model("openai", selected)
+
+    return selected
+
+
+def select_xai_model(
+    api_key: str,
+    policy: str = "latest",
+    pin: Optional[str] = None,
+    mock_models: Optional[List[Dict]] = None,
+) -> str:
+    """Select the best xAI model based on policy.
+
+    Args:
+        api_key: xAI API key
+        policy: 'latest', 'stable', or 'pinned'
+        pin: Model to use if policy is 'pinned'
+        mock_models: Mock model list for testing
+
+    Returns:
+        Selected model ID
+    """
+    if policy == "pinned" and pin:
+        return pin
+
+    # Use alias system
+    if policy in XAI_ALIASES:
+        alias = XAI_ALIASES[policy]
+
+        # Check cache first
+        cached = cache.get_cached_model("xai")
+        if cached:
+            return cached
+
+        # Cache the alias
+        cache.set_cached_model("xai", alias)
+        return alias
+
+    # Default to latest
+    return XAI_ALIASES["latest"]
+
+
+def get_models(
+    config: Dict,
+    mock_openai_models: Optional[List[Dict]] = None,
+    mock_xai_models: Optional[List[Dict]] = None,
+) -> Dict[str, Optional[str]]:
+    """Get selected models for both providers.
+
+    Returns:
+        Dict with 'openai' and 'xai' keys
+    """
+    result = {"openai": None, "xai": None}
+
+    if config.get("OPENAI_API_KEY"):
+        result["openai"] = select_openai_model(
+            config["OPENAI_API_KEY"],
+            config.get("OPENAI_MODEL_POLICY", "auto"),
+            config.get("OPENAI_MODEL_PIN"),
+            mock_openai_models,
+        )
+
+    if config.get("XAI_API_KEY"):
+        result["xai"] = select_xai_model(
+            config["XAI_API_KEY"],
+            config.get("XAI_MODEL_POLICY", "latest"),
+            config.get("XAI_MODEL_PIN"),
+            mock_xai_models,
+        )
+
+    return result

package/templates/vault/last30days/scripts/lib/normalize.py

@@ -0,0 +1,160 @@
+"""Normalization of raw API data to canonical schema."""
+
+from typing import Any, Dict, List, TypeVar, Union
+
+from . import dates, schema
+
+T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem)
+
+
+def filter_by_date_range(
+    items: List[T],
+    from_date: str,
+    to_date: str,
+    require_date: bool = False,
+) -> List[T]:
+    """Hard filter: Remove items outside the date range.
+
+    This is the safety net - even if the prompt lets old content through,
+    this filter will exclude it.
+
+    Args:
+        items: List of items to filter
+        from_date: Start date (YYYY-MM-DD) - exclude items before this
+        to_date: End date (YYYY-MM-DD) - exclude items after this
+        require_date: If True, also remove items with no date
+
+    Returns:
+        Filtered list with only items in range (or unknown dates if not required)
+    """
+    result = []
+    for item in items:
+        if item.date is None:
+            if not require_date:
+                result.append(item)  # Keep unknown dates (with scoring penalty)
+            continue
+
+        # Hard filter: if date is before from_date, exclude
+        if item.date < from_date:
+            continue  # DROP - too old
+
+        # Hard filter: if date is after to_date, exclude (likely parsing error)
+        if item.date > to_date:
+            continue  # DROP - future date
+
+        result.append(item)
+
+    return result
+
+
+def normalize_reddit_items(
+    items: List[Dict[str, Any]],
+    from_date: str,
+    to_date: str,
+) -> List[schema.RedditItem]:
+    """Normalize raw Reddit items to schema.
+
+    Args:
+        items: Raw Reddit items from API
+        from_date: Start of date range
+        to_date: End of date range
+
+    Returns:
+        List of RedditItem objects
+    """
+    normalized = []
+
+    for item in items:
+        # Parse engagement
+        engagement = None
+        eng_raw = item.get("engagement")
+        if isinstance(eng_raw, dict):
+            engagement = schema.Engagement(
+                score=eng_raw.get("score"),
+                num_comments=eng_raw.get("num_comments"),
+                upvote_ratio=eng_raw.get("upvote_ratio"),
+            )
+
+        # Parse comments
+        top_comments = []
+        for c in item.get("top_comments", []):
+            top_comments.append(schema.Comment(
+                score=c.get("score", 0),
+                date=c.get("date"),
+                author=c.get("author", ""),
+                excerpt=c.get("excerpt", ""),
+                url=c.get("url", ""),
+            ))
+
+        # Determine date confidence
+        date_str = item.get("date")
+        date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
+
+        normalized.append(schema.RedditItem(
+            id=item.get("id", ""),
+            title=item.get("title", ""),
+            url=item.get("url", ""),
+            subreddit=item.get("subreddit", ""),
+            date=date_str,
+            date_confidence=date_confidence,
+            engagement=engagement,
+            top_comments=top_comments,
+            comment_insights=item.get("comment_insights", []),
+            relevance=item.get("relevance", 0.5),
+            why_relevant=item.get("why_relevant", ""),
+        ))
+
+    return normalized
+
+
+def normalize_x_items(
+    items: List[Dict[str, Any]],
+    from_date: str,
+    to_date: str,
+) -> List[schema.XItem]:
+    """Normalize raw X items to schema.
+
+    Args:
+        items: Raw X items from API
+        from_date: Start of date range
+        to_date: End of date range
+
+    Returns:
+        List of XItem objects
+    """
+    normalized = []
+
+    for item in items:
+        # Parse engagement
+        engagement = None
+        eng_raw = item.get("engagement")
+        if isinstance(eng_raw, dict):
+            engagement = schema.Engagement(
+                likes=eng_raw.get("likes"),
+                reposts=eng_raw.get("reposts"),
+                replies=eng_raw.get("replies"),
+                quotes=eng_raw.get("quotes"),
+            )
+
+        # Determine date confidence
+        date_str = item.get("date")
+        date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
+
+        normalized.append(schema.XItem(
+            id=item.get("id", ""),
+            text=item.get("text", ""),
+            url=item.get("url", ""),
+            author_handle=item.get("author_handle", ""),
+            date=date_str,
+            date_confidence=date_confidence,
+            engagement=engagement,
+            relevance=item.get("relevance", 0.5),
+            why_relevant=item.get("why_relevant", ""),
+        ))
+
+    return normalized
+
+
+def items_to_dicts(items: List) -> List[Dict[str, Any]]:
+    """Convert schema items to dicts for JSON serialization."""
+    return [item.to_dict() for item in items]

package/templates/vault/last30days/scripts/lib/openai_reddit.py

@@ -0,0 +1,230 @@
+"""OpenAI Responses API client for Reddit discovery."""
+
+import json
+import re
+import sys
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+
+def _log_error(msg: str):
+    """Log error to stderr."""
+    sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
+    sys.stderr.flush()
+
+OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
+
+# Depth configurations: (min, max) threads to request
+# Request MORE than needed since many get filtered by date
+DEPTH_CONFIG = {
+    "quick": (15, 25),
+    "default": (30, 50),
+    "deep": (70, 100),
+}
+
+REDDIT_SEARCH_PROMPT = """Find Reddit discussion threads about: {topic}
+
+STEP 1: EXTRACT THE CORE SUBJECT
+Get the MAIN NOUN/PRODUCT/TOPIC:
+- "best nano banana prompting practices" → "nano banana"
+- "killer features of clawdbot" → "clawdbot"
+- "top Claude Code skills" → "Claude Code"
+DO NOT include "best", "top", "tips", "practices", "features" in your search.
+
+STEP 2: SEARCH BROADLY
+Search for the core subject:
+1. "[core subject] site:reddit.com"
+2. "reddit [core subject]"
+3. "[core subject] reddit"
+
+Return as many relevant threads as you find. We filter by date server-side.
+
+STEP 3: INCLUDE ALL MATCHES
+- Include ALL threads about the core subject
+- Set date to "YYYY-MM-DD" if you can determine it, otherwise null
+- We verify dates and filter old content server-side
+- DO NOT pre-filter aggressively - include anything relevant
+
+REQUIRED: URLs must contain "/r/" AND "/comments/"
+REJECT: developers.reddit.com, business.reddit.com
+
+Find {min_items}-{max_items} threads. Return MORE rather than fewer.
+
+Return JSON:
+{{
+  "items": [
+    {{
+      "title": "Thread title",
+      "url": "https://www.reddit.com/r/sub/comments/xyz/title/",
+      "subreddit": "subreddit_name",
+      "date": "YYYY-MM-DD or null",
+      "why_relevant": "Why relevant",
+      "relevance": 0.85
+    }}
+  ]
+}}"""
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for retry."""
+    noise = ['best', 'top', 'how to', 'tips for', 'practices', 'features',
+             'killer', 'guide', 'tutorial', 'recommendations', 'advice',
+             'prompting', 'using', 'for', 'with', 'the', 'of', 'in', 'on']
+    words = topic.lower().split()
+    result = [w for w in words if w not in noise]
+    return ' '.join(result[:3]) or topic  # Keep max 3 words
+
+
+def search_reddit(
+    api_key: str,
+    model: str,
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    mock_response: Optional[Dict] = None,
+    _retry: bool = False,
+) -> Dict[str, Any]:
+    """Search Reddit for relevant threads using OpenAI Responses API.
+
+    Args:
+        api_key: OpenAI API key
+        model: Model to use
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD) - only include threads after this
+        to_date: End date (YYYY-MM-DD) - only include threads before this
+        depth: Research depth - "quick", "default", or "deep"
+        mock_response: Mock response for testing
+
+    Returns:
+        Raw API response
+    """
+    if mock_response is not None:
+        return mock_response
+
+    min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    # Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
+    timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
+
+    # Note: allowed_domains accepts base domain, not subdomains
+    # We rely on prompt to filter out developers.reddit.com, etc.
+    payload = {
+        "model": model,
+        "tools": [
+            {
+                "type": "web_search",
+                "filters": {
+                    "allowed_domains": ["reddit.com"]
+                }
+            }
+        ],
+        "include": ["web_search_call.action.sources"],
+        "input": REDDIT_SEARCH_PROMPT.format(
+            topic=topic,
+            from_date=from_date,
+            to_date=to_date,
+            min_items=min_items,
+            max_items=max_items,
+        ),
+    }
+
+    return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
+
+
+def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse OpenAI response to extract Reddit items.
+
+    Args:
+        response: Raw API response
+
+    Returns:
+        List of item dicts
+    """
+    items = []
+
+    # Check for API errors first
+    if "error" in response and response["error"]:
+        error = response["error"]
+        err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
+        _log_error(f"OpenAI API error: {err_msg}")
+        if http.DEBUG:
+            _log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
+        return items
+
+    # Try to find the output text
+    output_text = ""
+    if "output" in response:
+        output = response["output"]
+        if isinstance(output, str):
+            output_text = output
+        elif isinstance(output, list):
+            for item in output:
+                if isinstance(item, dict):
+                    if item.get("type") == "message":
+                        content = item.get("content", [])
+                        for c in content:
+                            if isinstance(c, dict) and c.get("type") == "output_text":
+                                output_text = c.get("text", "")
+                                break
+                    elif "text" in item:
+                        output_text = item["text"]
+                elif isinstance(item, str):
+                    output_text = item
+                if output_text:
+                    break
+
+    # Also check for choices (older format)
+    if not output_text and "choices" in response:
+        for choice in response["choices"]:
+            if "message" in choice:
+                output_text = choice["message"].get("content", "")
+                break
+
+    if not output_text:
+        print(f"[REDDIT WARNING] No output text found in OpenAI response. Keys present: {list(response.keys())}", flush=True)
+        return items
+
+    # Extract JSON from the response
+    json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
+    if json_match:
+        try:
+            data = json.loads(json_match.group())
+            items = data.get("items", [])
+        except json.JSONDecodeError:
+            pass
+
+    # Validate and clean items
+    clean_items = []
+    for i, item in enumerate(items):
+        if not isinstance(item, dict):
+            continue
+
+        url = item.get("url", "")
+        if not url or "reddit.com" not in url:
+            continue
+
+        clean_item = {
+            "id": f"R{i+1}",
+            "title": str(item.get("title", "")).strip(),
+            "url": url,
+            "subreddit": str(item.get("subreddit", "")).strip().lstrip("r/"),
+            "date": item.get("date"),
+            "why_relevant": str(item.get("why_relevant", "")).strip(),
+            "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
+        }
+
+        # Validate date format
+        if clean_item["date"]:
+            if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])):
+                clean_item["date"] = None
+
+        clean_items.append(clean_item)
+
+    return clean_items