bps-kit 1.0.1 → 1.0.2

package/templates/vault/last30days/scripts/lib/dates.py

@@ -0,0 +1,124 @@
+"""Date utilities for last30days skill."""
+
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Tuple
+
+
+def get_date_range(days: int = 30) -> Tuple[str, str]:
+    """Get the date range for the last N days.
+
+    Returns:
+        Tuple of (from_date, to_date) as YYYY-MM-DD strings
+    """
+    today = datetime.now(timezone.utc).date()
+    from_date = today - timedelta(days=days)
+    return from_date.isoformat(), today.isoformat()
+
+
+def parse_date(date_str: Optional[str]) -> Optional[datetime]:
+    """Parse a date string in various formats.
+
+    Supports: YYYY-MM-DD, ISO 8601, Unix timestamp
+    """
+    if not date_str:
+        return None
+
+    # Try Unix timestamp (from Reddit)
+    try:
+        ts = float(date_str)
+        return datetime.fromtimestamp(ts, tz=timezone.utc)
+    except (ValueError, TypeError):
+        pass
+
+    # Try ISO formats
+    formats = [
+        "%Y-%m-%d",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%dT%H:%M:%SZ",
+        "%Y-%m-%dT%H:%M:%S%z",
+        "%Y-%m-%dT%H:%M:%S.%f%z",
+    ]
+
+    for fmt in formats:
+        try:
+            return datetime.strptime(date_str, fmt).replace(tzinfo=timezone.utc)
+        except ValueError:
+            continue
+
+    return None
+
+
+def timestamp_to_date(ts: Optional[float]) -> Optional[str]:
+    """Convert Unix timestamp to YYYY-MM-DD string."""
+    if ts is None:
+        return None
+    try:
+        dt = datetime.fromtimestamp(ts, tz=timezone.utc)
+        return dt.date().isoformat()
+    except (ValueError, TypeError, OSError):
+        return None
+
+
+def get_date_confidence(date_str: Optional[str], from_date: str, to_date: str) -> str:
+    """Determine confidence level for a date.
+
+    Args:
+        date_str: The date to check (YYYY-MM-DD or None)
+        from_date: Start of valid range (YYYY-MM-DD)
+        to_date: End of valid range (YYYY-MM-DD)
+
+    Returns:
+        'high', 'med', or 'low'
+    """
+    if not date_str:
+        return 'low'
+
+    try:
+        dt = datetime.strptime(date_str, "%Y-%m-%d").date()
+        start = datetime.strptime(from_date, "%Y-%m-%d").date()
+        end = datetime.strptime(to_date, "%Y-%m-%d").date()
+
+        if start <= dt <= end:
+            return 'high'
+        elif dt < start:
+            # Older than range
+            return 'low'
+        else:
+            # Future date (suspicious)
+            return 'low'
+    except ValueError:
+        return 'low'
+
+
+def days_ago(date_str: Optional[str]) -> Optional[int]:
+    """Calculate how many days ago a date is.
+
+    Returns None if date is invalid or missing.
+    """
+    if not date_str:
+        return None
+
+    try:
+        dt = datetime.strptime(date_str, "%Y-%m-%d").date()
+        today = datetime.now(timezone.utc).date()
+        delta = today - dt
+        return delta.days
+    except ValueError:
+        return None
+
+
+def recency_score(date_str: Optional[str], max_days: int = 30) -> int:
+    """Calculate recency score (0-100).
+
+    0 days ago = 100, max_days ago = 0, clamped.
+    """
+    age = days_ago(date_str)
+    if age is None:
+        return 0  # Unknown date gets worst score
+
+    if age < 0:
+        return 100  # Future date (treat as today)
+    if age >= max_days:
+        return 0
+
+    return int(100 * (1 - age / max_days))

package/templates/vault/last30days/scripts/lib/dedupe.py

@@ -0,0 +1,120 @@
+"""Near-duplicate detection for last30days skill."""
+
+import re
+from typing import List, Set, Tuple, Union
+
+from . import schema
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for comparison.
+
+    - Lowercase
+    - Remove punctuation
+    - Collapse whitespace
+    """
+    text = text.lower()
+    text = re.sub(r'[^\w\s]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    return text.strip()
+
+
+def get_ngrams(text: str, n: int = 3) -> Set[str]:
+    """Get character n-grams from text."""
+    text = normalize_text(text)
+    if len(text) < n:
+        return {text}
+    return {text[i:i+n] for i in range(len(text) - n + 1)}
+
+
+def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
+    """Compute Jaccard similarity between two sets."""
+    if not set1 or not set2:
+        return 0.0
+    intersection = len(set1 & set2)
+    union = len(set1 | set2)
+    return intersection / union if union > 0 else 0.0
+
+
+def get_item_text(item: Union[schema.RedditItem, schema.XItem]) -> str:
+    """Get comparable text from an item."""
+    if isinstance(item, schema.RedditItem):
+        return item.title
+    else:
+        return item.text
+
+
+def find_duplicates(
+    items: List[Union[schema.RedditItem, schema.XItem]],
+    threshold: float = 0.7,
+) -> List[Tuple[int, int]]:
+    """Find near-duplicate pairs in items.
+
+    Args:
+        items: List of items to check
+        threshold: Similarity threshold (0-1)
+
+    Returns:
+        List of (i, j) index pairs where i < j and items are similar
+    """
+    duplicates = []
+
+    # Pre-compute n-grams
+    ngrams = [get_ngrams(get_item_text(item)) for item in items]
+
+    for i in range(len(items)):
+        for j in range(i + 1, len(items)):
+            similarity = jaccard_similarity(ngrams[i], ngrams[j])
+            if similarity >= threshold:
+                duplicates.append((i, j))
+
+    return duplicates
+
+
+def dedupe_items(
+    items: List[Union[schema.RedditItem, schema.XItem]],
+    threshold: float = 0.7,
+) -> List[Union[schema.RedditItem, schema.XItem]]:
+    """Remove near-duplicates, keeping highest-scored item.
+
+    Args:
+        items: List of items (should be pre-sorted by score descending)
+        threshold: Similarity threshold
+
+    Returns:
+        Deduplicated items
+    """
+    if len(items) <= 1:
+        return items
+
+    # Find duplicate pairs
+    dup_pairs = find_duplicates(items, threshold)
+
+    # Mark indices to remove (always remove the lower-scored one)
+    # Since items are pre-sorted by score, the second index is always lower
+    to_remove = set()
+    for i, j in dup_pairs:
+        # Keep the higher-scored one (lower index in sorted list)
+        if items[i].score >= items[j].score:
+            to_remove.add(j)
+        else:
+            to_remove.add(i)
+
+    # Return items not marked for removal
+    return [item for idx, item in enumerate(items) if idx not in to_remove]
+
+
+def dedupe_reddit(
+    items: List[schema.RedditItem],
+    threshold: float = 0.7,
+) -> List[schema.RedditItem]:
+    """Dedupe Reddit items."""
+    return dedupe_items(items, threshold)
+
+
+def dedupe_x(
+    items: List[schema.XItem],
+    threshold: float = 0.7,
+) -> List[schema.XItem]:
+    """Dedupe X items."""
+    return dedupe_items(items, threshold)

package/templates/vault/last30days/scripts/lib/env.py

@@ -0,0 +1,149 @@
+"""Environment and API key management for last30days skill."""
+
+import os
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+CONFIG_DIR = Path.home() / ".config" / "last30days"
+CONFIG_FILE = CONFIG_DIR / ".env"
+
+
+def load_env_file(path: Path) -> Dict[str, str]:
+    """Load environment variables from a file."""
+    env = {}
+    if not path.exists():
+        return env
+
+    with open(path, 'r') as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            if '=' in line:
+                key, _, value = line.partition('=')
+                key = key.strip()
+                value = value.strip()
+                # Remove quotes if present
+                if value and value[0] in ('"', "'") and value[-1] == value[0]:
+                    value = value[1:-1]
+                if key and value:
+                    env[key] = value
+    return env
+
+
+def get_config() -> Dict[str, Any]:
+    """Load configuration from ~/.config/last30days/.env and environment."""
+    # Load from config file first
+    file_env = load_env_file(CONFIG_FILE)
+
+    # Environment variables override file
+    config = {
+        'OPENAI_API_KEY': os.environ.get('OPENAI_API_KEY') or file_env.get('OPENAI_API_KEY'),
+        'XAI_API_KEY': os.environ.get('XAI_API_KEY') or file_env.get('XAI_API_KEY'),
+        'OPENAI_MODEL_POLICY': os.environ.get('OPENAI_MODEL_POLICY') or file_env.get('OPENAI_MODEL_POLICY', 'auto'),
+        'OPENAI_MODEL_PIN': os.environ.get('OPENAI_MODEL_PIN') or file_env.get('OPENAI_MODEL_PIN'),
+        'XAI_MODEL_POLICY': os.environ.get('XAI_MODEL_POLICY') or file_env.get('XAI_MODEL_POLICY', 'latest'),
+        'XAI_MODEL_PIN': os.environ.get('XAI_MODEL_PIN') or file_env.get('XAI_MODEL_PIN'),
+    }
+
+    return config
+
+
+def config_exists() -> bool:
+    """Check if configuration file exists."""
+    return CONFIG_FILE.exists()
+
+
+def get_available_sources(config: Dict[str, Any]) -> str:
+    """Determine which sources are available based on API keys.
+
+    Returns: 'both', 'reddit', 'x', or 'web' (fallback when no keys)
+    """
+    has_openai = bool(config.get('OPENAI_API_KEY'))
+    has_xai = bool(config.get('XAI_API_KEY'))
+
+    if has_openai and has_xai:
+        return 'both'
+    elif has_openai:
+        return 'reddit'
+    elif has_xai:
+        return 'x'
+    else:
+        return 'web'  # Fallback: WebSearch only (no API keys needed)
+
+
+def get_missing_keys(config: Dict[str, Any]) -> str:
+    """Determine which API keys are missing.
+
+    Returns: 'both', 'reddit', 'x', or 'none'
+    """
+    has_openai = bool(config.get('OPENAI_API_KEY'))
+    has_xai = bool(config.get('XAI_API_KEY'))
+
+    if has_openai and has_xai:
+        return 'none'
+    elif has_openai:
+        return 'x'  # Missing xAI key
+    elif has_xai:
+        return 'reddit'  # Missing OpenAI key
+    else:
+        return 'both'  # Missing both keys
+
+
+def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
+    """Validate requested sources against available keys.
+
+    Args:
+        requested: 'auto', 'reddit', 'x', 'both', or 'web'
+        available: Result from get_available_sources()
+        include_web: If True, add WebSearch to available sources
+
+    Returns:
+        Tuple of (effective_sources, error_message)
+    """
+    # WebSearch-only mode (no API keys)
+    if available == 'web':
+        if requested == 'auto':
+            return 'web', None
+        elif requested == 'web':
+            return 'web', None
+        else:
+            return 'web', f"No API keys configured. Using WebSearch fallback. Add keys to ~/.config/last30days/.env for Reddit/X."
+
+    if requested == 'auto':
+        # Add web to sources if include_web is set
+        if include_web:
+            if available == 'both':
+                return 'all', None  # reddit + x + web
+            elif available == 'reddit':
+                return 'reddit-web', None
+            elif available == 'x':
+                return 'x-web', None
+        return available, None
+
+    if requested == 'web':
+        return 'web', None
+
+    if requested == 'both':
+        if available not in ('both',):
+            missing = 'xAI' if available == 'reddit' else 'OpenAI'
+            return 'none', f"Requested both sources but {missing} key is missing. Use --sources=auto to use available keys."
+        if include_web:
+            return 'all', None
+        return 'both', None
+
+    if requested == 'reddit':
+        if available == 'x':
+            return 'none', "Requested Reddit but only xAI key is available."
+        if include_web:
+            return 'reddit-web', None
+        return 'reddit', None
+
+    if requested == 'x':
+        if available == 'reddit':
+            return 'none', "Requested X but only OpenAI key is available."
+        if include_web:
+            return 'x-web', None
+        return 'x', None
+
+    return requested, None

package/templates/vault/last30days/scripts/lib/http.py

@@ -0,0 +1,152 @@
+"""HTTP utilities for last30days skill (stdlib only)."""
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+
+DEFAULT_TIMEOUT = 30
+DEBUG = os.environ.get("LAST30DAYS_DEBUG", "").lower() in ("1", "true", "yes")
+
+
+def log(msg: str):
+    """Log debug message to stderr."""
+    if DEBUG:
+        sys.stderr.write(f"[DEBUG] {msg}\n")
+        sys.stderr.flush()
+MAX_RETRIES = 3
+RETRY_DELAY = 1.0
+USER_AGENT = "last30days-skill/1.0 (Claude Code Skill)"
+
+
+class HTTPError(Exception):
+    """HTTP request error with status code."""
+    def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+def request(
+    method: str,
+    url: str,
+    headers: Optional[Dict[str, str]] = None,
+    json_data: Optional[Dict[str, Any]] = None,
+    timeout: int = DEFAULT_TIMEOUT,
+    retries: int = MAX_RETRIES,
+) -> Dict[str, Any]:
+    """Make an HTTP request and return JSON response.
+
+    Args:
+        method: HTTP method (GET, POST, etc.)
+        url: Request URL
+        headers: Optional headers dict
+        json_data: Optional JSON body (for POST)
+        timeout: Request timeout in seconds
+        retries: Number of retries on failure
+
+    Returns:
+        Parsed JSON response
+
+    Raises:
+        HTTPError: On request failure
+    """
+    headers = headers or {}
+    headers.setdefault("User-Agent", USER_AGENT)
+
+    data = None
+    if json_data is not None:
+        data = json.dumps(json_data).encode('utf-8')
+        headers.setdefault("Content-Type", "application/json")
+
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+
+    log(f"{method} {url}")
+    if json_data:
+        log(f"Payload keys: {list(json_data.keys())}")
+
+    last_error = None
+    for attempt in range(retries):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as response:
+                body = response.read().decode('utf-8')
+                log(f"Response: {response.status} ({len(body)} bytes)")
+                return json.loads(body) if body else {}
+        except urllib.error.HTTPError as e:
+            body = None
+            try:
+                body = e.read().decode('utf-8')
+            except:
+                pass
+            log(f"HTTP Error {e.code}: {e.reason}")
+            if body:
+                log(f"Error body: {body[:500]}")
+            last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
+
+            # Don't retry client errors (4xx) except rate limits
+            if 400 <= e.code < 500 and e.code != 429:
+                raise last_error
+
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+        except urllib.error.URLError as e:
+            log(f"URL Error: {e.reason}")
+            last_error = HTTPError(f"URL Error: {e.reason}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+        except json.JSONDecodeError as e:
+            log(f"JSON decode error: {e}")
+            last_error = HTTPError(f"Invalid JSON response: {e}")
+            raise last_error
+        except (OSError, TimeoutError, ConnectionResetError) as e:
+            # Handle socket-level errors (connection reset, timeout, etc.)
+            log(f"Connection error: {type(e).__name__}: {e}")
+            last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+
+    if last_error:
+        raise last_error
+    raise HTTPError("Request failed with no error details")
+
+
+def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a GET request."""
+    return request("GET", url, headers=headers, **kwargs)
+
+
+def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a POST request with JSON body."""
+    return request("POST", url, headers=headers, json_data=json_data, **kwargs)
+
+
+def get_reddit_json(path: str) -> Dict[str, Any]:
+    """Fetch Reddit thread JSON.
+
+    Args:
+        path: Reddit path (e.g., /r/subreddit/comments/id/title)
+
+    Returns:
+        Parsed JSON response
+    """
+    # Ensure path starts with /
+    if not path.startswith('/'):
+        path = '/' + path
+
+    # Remove trailing slash and add .json
+    path = path.rstrip('/')
+    if not path.endswith('.json'):
+        path = path + '.json'
+
+    url = f"https://www.reddit.com{path}?raw_json=1"
+
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+    }
+
+    return get(url, headers=headers)