@w3kits-com/plugin-opendesign 0.1.8 → 0.1.9

package/dist/__w3kits/assets/design-templates/last30days/scripts/lib/grounding.py

@@ -0,0 +1,259 @@
+"""Web search retrieval via Brave Search, Exa, and Serper."""
+
+from __future__ import annotations
+
+import urllib.parse
+from datetime import datetime
+from urllib.parse import urlparse
+
+from . import dates, http
+
+
+# ---------------------------------------------------------------------------
+# Brave Search API
+# ---------------------------------------------------------------------------
+
+def brave_search(
+    query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
+) -> tuple[list[dict], dict]:
+    url = (
+        "https://api.search.brave.com/res/v1/web/search?"
+        + urllib.parse.urlencode(
+            {
+                "q": query,
+                "count": count,
+                "freshness": f"{date_range[0]}to{date_range[1]}",
+            }
+        )
+    )
+    data = http.request("GET", url, headers={"X-Subscription-Token": api_key}, timeout=15)
+    items = []
+    for i, r in enumerate((data.get("web", {}).get("results", []))[:count]):
+        raw_date = r.get("page_age") or ""
+        pub_date = _normalize_date(raw_date[:10]) if raw_date else None
+        if not _in_date_range(pub_date, date_range):
+            continue
+        items.append({
+            "id": f"WB{i + 1}",
+            "title": r.get("title", ""),
+            "url": r.get("url", ""),
+            "source_domain": _domain(r.get("url", "")),
+            "snippet": r.get("description", ""),
+            "date": pub_date,
+            "relevance": 0.8,
+            "why_relevant": "Brave web search",
+        })
+    artifact = {"label": "brave", "webSearchQueries": [query], "resultCount": len(items)}
+    return items, artifact
+
+
+# ---------------------------------------------------------------------------
+# Exa AI Search
+# ---------------------------------------------------------------------------
+
+def exa_search(
+    query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
+) -> tuple[list[dict], dict]:
+    data = http.request(
+        "POST", "https://api.exa.ai/search",
+        headers={"x-api-key": api_key},
+        json_data={
+            "query": query,
+            "type": "auto",
+            "numResults": count,
+            "startPublishedDate": f"{date_range[0]}T00:00:00.000Z",
+            "endPublishedDate": f"{date_range[1]}T23:59:59.999Z",
+            "contents": {"text": {"maxCharacters": 2000}},
+        },
+        timeout=15,
+    )
+    items = []
+    for i, r in enumerate((data.get("results", []))[:count]):
+        if not isinstance(r, dict):
+            continue
+        url = r.get("url", "")
+        if not url:
+            continue
+        raw_date = r.get("publishedDate") or ""
+        pub_date = _normalize_date(raw_date.split("T")[0] if "T" in raw_date else raw_date[:10]) if raw_date else None
+        if not _in_date_range(pub_date, date_range):
+            continue
+        items.append({
+            "id": f"WE{i + 1}",
+            "title": r.get("title", ""),
+            "url": url,
+            "source_domain": _domain(url),
+            "snippet": (r.get("text") or "")[:500],
+            "date": pub_date,
+            "relevance": 0.8,
+            "why_relevant": "Exa web search",
+        })
+    artifact = {"label": "exa", "webSearchQueries": [query], "resultCount": len(items)}
+    return items, artifact
+
+
+# ---------------------------------------------------------------------------
+# Serper (Google Search wrapper)
+# ---------------------------------------------------------------------------
+
+def serper_search(
+    query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
+) -> tuple[list[dict], dict]:
+    data = http.request(
+        "POST", "https://google.serper.dev/search",
+        headers={"X-API-KEY": api_key},
+        json_data={
+            "q": query,
+            "num": count,
+            "tbs": f"cdr:1,cd_min:{_serper_date_param(date_range[0])},cd_max:{_serper_date_param(date_range[1])}",
+        },
+        timeout=15,
+    )
+    items = []
+    for i, r in enumerate((data.get("organic", []))[:count]):
+        raw_date = r.get("date") or ""
+        pub_date = _parse_serper_date(raw_date)
+        if not _in_date_range(pub_date, date_range):
+            continue
+        items.append({
+            "id": f"WS{i + 1}",
+            "title": r.get("title", ""),
+            "url": r.get("link", ""),
+            "source_domain": _domain(r.get("link", "")),
+            "snippet": r.get("snippet", ""),
+            "date": pub_date,
+            "relevance": 0.8,
+            "why_relevant": "Serper web search",
+        })
+    artifact = {"label": "serper", "webSearchQueries": [query], "resultCount": len(items)}
+    return items, artifact
+
+
+# ---------------------------------------------------------------------------
+# Parallel AI Search
+# ---------------------------------------------------------------------------
+
+def parallel_search(
+    query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
+) -> tuple[list[dict], dict]:
+    data = http.request(
+        "POST", "https://api.parallel.ai/v1/search",
+        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+        json_data={"query": query, "max_results": count},
+        timeout=15,
+    )
+    items = []
+    for i, r in enumerate((data.get("results", []))[:count]):
+        if not isinstance(r, dict):
+            continue
+        url = r.get("url", "")
+        if not url:
+            continue
+        raw_date = r.get("published_date") or ""
+        pub_date = _normalize_date(raw_date[:10]) if raw_date else None
+        if not _in_date_range(pub_date, date_range):
+            continue
+        items.append({
+            "id": f"WP{i + 1}",
+            "title": r.get("title", ""),
+            "url": url,
+            "source_domain": _domain(url),
+            "snippet": r.get("snippet", ""),
+            "date": pub_date,
+            "relevance": 0.8,
+            "why_relevant": "Parallel AI web search",
+        })
+    artifact = {"label": "parallel", "webSearchQueries": [query], "resultCount": len(items)}
+    return items, artifact
+
+
+def _parse_serper_date(raw: str) -> str | None:
+    if not raw:
+        return None
+    normalized = _normalize_date(raw)
+    if normalized:
+        return normalized
+    for fmt in ("%b %d, %Y", "%B %d, %Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(raw.strip(), fmt).date().isoformat()
+        except ValueError:
+            continue
+    return None
+
+
+
+
+# ---------------------------------------------------------------------------
+# Dispatcher
+# ---------------------------------------------------------------------------
+
+def web_search(
+    query: str,
+    date_range: tuple[str, str],
+    config: dict,
+    backend: str = "auto",
+) -> tuple[list[dict], dict]:
+    """Run web search with the specified or auto-detected backend."""
+    if backend == "auto":
+        if config.get("BRAVE_API_KEY"):
+            backend = "brave"
+        elif config.get("EXA_API_KEY"):
+            backend = "exa"
+        elif config.get("SERPER_API_KEY"):
+            backend = "serper"
+        elif config.get("PARALLEL_API_KEY"):
+            backend = "parallel"
+        else:
+            return [], {}
+    if backend == "brave":
+        key = config.get("BRAVE_API_KEY")
+        if not key:
+            raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
+        return brave_search(query, date_range, key)
+    if backend == "exa":
+        key = config.get("EXA_API_KEY")
+        if not key:
+            raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
+        return exa_search(query, date_range, key)
+    if backend == "serper":
+        key = config.get("SERPER_API_KEY")
+        if not key:
+            raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
+        return serper_search(query, date_range, key)
+    if backend == "parallel":
+        key = config.get("PARALLEL_API_KEY")
+        if not key:
+            raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
+        return parallel_search(query, date_range, key)
+    if backend != "none":
+        raise ValueError(f"Unsupported web backend: {backend!r}")
+    return [], {}
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _normalize_date(value: object) -> str | None:
+    if value is None:
+        return None
+    parsed = dates.parse_date(str(value).strip())
+    if not parsed:
+        return None
+    return parsed.date().isoformat()
+
+
+def _serper_date_param(iso_date: str) -> str:
+    """Convert YYYY-MM-DD to MM/DD/YYYY for Serper tbs parameter."""
+    parts = iso_date.split("-")
+    return f"{parts[1]}/{parts[2]}/{parts[0]}"
+
+
+def _in_date_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:
+    if not pub_date:
+        return False
+    return date_range[0] <= pub_date <= date_range[1]
+
+
+def _domain(url: str) -> str:
+    return urlparse(url).netloc.strip().lower()

package/dist/__w3kits/assets/design-templates/last30days/scripts/lib/hackernews.py

@@ -0,0 +1,301 @@
+"""Hacker News search via Algolia API (free, no auth required).
+
+Uses hn.algolia.com/api/v1 for story discovery and comment enrichment.
+No API key needed - just HTTP calls via stdlib urllib.
+"""
+
+import datetime
+import html
+import math
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict, List, Optional
+
+import re
+
+from . import http, log
+from .query import extract_core_subject
+from .relevance import token_overlap_relevance
+
+# Common HN prefixes that can cause false-positive keyword matches
+_HN_PREFIXES = re.compile(r"^(Tell HN|Show HN|Ask HN|Launch HN)\s*:\s*", re.IGNORECASE)
+
+ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
+ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
+ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
+
+DEPTH_CONFIG = {
+    "quick": 15,
+    "default": 30,
+    "deep": 60,
+}
+
+ENRICH_LIMITS = {
+    "quick": 3,
+    "default": 5,
+    "deep": 10,
+}
+
+
+def _log(msg: str):
+    log.source_log("HN", msg)
+
+
+def _date_to_unix(date_str: str) -> int:
+    """Convert YYYY-MM-DD to Unix timestamp (start of day UTC)."""
+    parts = date_str.split("-")
+    year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
+    dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc)
+    return int(dt.timestamp())
+
+
+def _unix_to_date(ts: int) -> str:
+    """Convert Unix timestamp to YYYY-MM-DD."""
+    dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
+    return dt.strftime("%Y-%m-%d")
+
+
+def _strip_html(text: str) -> str:
+    """Strip HTML tags and decode entities from HN comment text."""
+    import re
+    text = html.unescape(text)
+    text = re.sub(r'<p>', '\n', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    return text.strip()
+
+
+def search_hackernews(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Search Hacker News via Algolia API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        Dict with Algolia response (contains 'hits' list).
+    """
+    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    from_ts = _date_to_unix(from_date)
+    to_ts = _date_to_unix(to_date) + 86400  # Include the end date
+
+    # Use extracted core subject instead of raw topic for cleaner Algolia matching
+    core = extract_core_subject(topic)
+    _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
+
+    # Use relevance-sorted search with minimum engagement filter.
+    # NOTE: restrictSearchableAttributes=title omitted intentionally — it would
+    # miss Ask HN/Show HN threads where the topic appears in the body.
+    params = {
+        "query": core,
+        "tags": "story",
+        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
+        "hitsPerPage": str(count),
+    }
+
+    from urllib.parse import urlencode
+    url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
+
+    try:
+        response = http.request("GET", url, timeout=30)
+    except http.HTTPError as e:
+        _log(f"Search failed: {e}")
+        return {"hits": [], "error": str(e)}
+    except Exception as e:
+        _log(f"Search failed: {e}")
+        return {"hits": [], "error": str(e)}
+
+    hits = response.get("hits", [])
+    _log(f"Found {len(hits)} stories")
+    return response
+
+
+def _title_matches_query(title: str, query: str, author: str = "") -> bool:
+    """Check if the query term appears in the title content, not just an HN prefix or author.
+
+    Returns True if the query (or any multi-word token) appears in the title
+    after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
+    and ignoring the author name.  Returns True when query is empty (no filter).
+    """
+    if not query:
+        return True
+    stripped = _HN_PREFIXES.sub("", title).strip()
+    # Also check that the match isn't solely in the author's username
+    check_text = stripped.lower()
+    query_lower = query.lower()
+    # Check each word of the query independently; all must appear somewhere
+    # in the stripped title (not just the prefix).
+    query_words = query_lower.split()
+    for word in query_words:
+        if word in check_text:
+            continue
+        # Word not found in stripped title — reject
+        return False
+    return True
+
+
+def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
+    """Parse Algolia response into normalized item dicts.
+
+    Args:
+        response: Algolia search response
+        query: Original search query for token-overlap relevance scoring
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    hits = response.get("hits", [])
+    # Post-filter: remove items where query only matched an HN prefix like "Tell HN:"
+    if query:
+        before = len(hits)
+        hits = [
+            h for h in hits
+            if _title_matches_query(h.get("title", ""), query, h.get("author", ""))
+        ]
+        dropped = before - len(hits)
+        if dropped:
+            _log(f"Prefix filter removed {dropped}/{before} false-positive hits for '{query}'")
+    items = []
+
+    for i, hit in enumerate(hits):
+        object_id = hit.get("objectID", "")
+        points = hit.get("points") or 0
+        num_comments = hit.get("num_comments") or 0
+        created_at_i = hit.get("created_at_i")
+
+        date_str = None
+        if created_at_i:
+            date_str = _unix_to_date(created_at_i)
+
+        # Article URL vs HN discussion URL
+        article_url = hit.get("url") or ""
+        hn_url = f"https://news.ycombinator.com/item?id={object_id}"
+
+        # Relevance: blend Algolia rank with token-overlap content matching
+        rank_score = max(0.3, 1.0 - (i * 0.02))  # 1.0 -> 0.3 over 35 items
+        engagement_boost = min(0.2, math.log1p(points) / 40)
+        if query:
+            content_score = token_overlap_relevance(query, hit.get("title", ""))
+            relevance = min(1.0, 0.6 * rank_score + 0.4 * content_score + engagement_boost)
+        else:
+            relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
+
+        items.append({
+            "id": object_id,
+            "title": hit.get("title", ""),
+            "url": article_url,
+            "hn_url": hn_url,
+            "author": hit.get("author", ""),
+            "date": date_str,
+            "engagement": {
+                "points": points,
+                "comments": num_comments,
+            },
+            "relevance": round(relevance, 2),
+            "why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}",
+        })
+
+    return items
+
+
+def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any]:
+    """Fetch top-level comments for a story from Algolia items endpoint.
+
+    Args:
+        object_id: HN story ID
+        max_comments: Max comments to return
+
+    Returns:
+        Dict with 'comments' list and 'comment_insights' list.
+    """
+    url = f"{ALGOLIA_ITEM_URL}/{object_id}"
+
+    try:
+        data = http.request("GET", url, timeout=15)
+    except Exception as e:
+        _log(f"Failed to fetch comments for {object_id}: {e}")
+        return {"comments": [], "comment_insights": []}
+
+    children = data.get("children", [])
+
+    # Sort by points (highest first), filter to actual comments
+    real_comments = [
+        c for c in children
+        if c.get("text") and c.get("author")
+    ]
+    real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True)
+
+    comments = []
+    insights = []
+    for c in real_comments[:max_comments]:
+        text = _strip_html(c.get("text", ""))
+        excerpt = text[:300] + "..." if len(text) > 300 else text
+        comments.append({
+            "author": c.get("author", ""),
+            "text": excerpt,
+            "points": c.get("points") or 0,
+        })
+        # First sentence as insight
+        first_sentence = text.split(". ")[0].split("\n")[0][:200]
+        if first_sentence:
+            insights.append(first_sentence)
+
+    return {"comments": comments, "comment_insights": insights}
+
+
+def enrich_top_stories(
+    items: List[Dict[str, Any]],
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Fetch comments for top N stories by points.
+
+    Args:
+        items: Parsed HN items
+        depth: Research depth (controls how many to enrich)
+
+    Returns:
+        Items with top_comments and comment_insights added.
+    """
+    if not items:
+        return items
+
+    limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
+
+    # Sort by points to enrich the most popular stories
+    by_points = sorted(
+        range(len(items)),
+        key=lambda i: items[i].get("engagement", {}).get("points", 0),
+        reverse=True,
+    )
+    to_enrich = by_points[:limit]
+
+    _log(f"Enriching top {len(to_enrich)} stories with comments")
+
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        futures = {
+            executor.submit(
+                _fetch_item_comments,
+                items[idx]["id"],
+            ): idx
+            for idx in to_enrich
+        }
+
+        for future in as_completed(futures):
+            idx = futures[future]
+            try:
+                result = future.result(timeout=15)
+                items[idx]["top_comments"] = result["comments"]
+                items[idx]["comment_insights"] = result["comment_insights"]
+            except (KeyError, TypeError, OSError) as exc:
+                _log(f"Comment enrichment failed for story {items[idx].get('id', '?')}: {type(exc).__name__}: {exc}")
+                items[idx]["top_comments"] = []
+                items[idx]["comment_insights"] = []
+
+    return items