luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/bluesky.py

@@ -23,6 +23,7 @@ from typing import Any, Dict, List, Optional
 from . import http, log
 
 BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
+BSKY_REFRESH_URL = "https://bsky.social/xrpc/com.atproto.server.refreshSession"
 _DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
 
 
@@ -90,8 +91,10 @@ DEPTH_CONFIG = {
 
 # Module-level token cache (valid for the lifetime of a single research run)
 _cached_token: Optional[str] = None
+_cached_refresh_token: Optional[str] = None
 _token_created_at: float = 0.0
 _session_error: Optional[str] = None
+_refresh_error_status: Optional[int] = None
 _TOKEN_MAX_AGE_SECONDS = 5400  # 90 minutes (conservative, tokens last ~2 hours)
 
 
@@ -109,7 +112,7 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
     Returns:
         Access JWT string, or None on failure. Sets _session_error on failure.
     """
-    global _cached_token, _token_created_at, _session_error
+    global _cached_token, _cached_refresh_token, _token_created_at, _session_error
     if _cached_token and (time.monotonic() - _token_created_at < _TOKEN_MAX_AGE_SECONDS):
         return _cached_token
     if _cached_token:
@@ -127,6 +130,7 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
         token = response.get("accessJwt")
         if token:
             _cached_token = token
+            _cached_refresh_token = response.get("refreshJwt")
             _token_created_at = time.monotonic()
             _session_error = None
             _log("Session created successfully")
@@ -149,11 +153,61 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
         return None
 
 
+def _refresh_session(refresh_jwt: Optional[str]) -> Optional[str]:
+    """Refresh an access token using the cached AT Protocol refresh token.
+
+    Returns the new access token on success.  ``_refresh_error_status`` records
+    an HTTP status so the caller can distinguish an invalid refresh token
+    (which should fall back to createSession) from a transient failure.
+    """
+    global _cached_token, _cached_refresh_token, _token_created_at
+    global _session_error, _refresh_error_status
+
+    _refresh_error_status = None
+    if not refresh_jwt:
+        _session_error = "No Bluesky refresh token is available"
+        return None
+
+    try:
+        response = http.request(
+            "POST",
+            BSKY_REFRESH_URL,
+            headers={"Authorization": f"Bearer {refresh_jwt}"},
+            timeout=15,
+            retries=0,
+        )
+        token = response.get("accessJwt")
+        if not token:
+            _session_error = "No accessJwt in refresh session response"
+            return None
+        _cached_token = token
+        _cached_refresh_token = response.get("refreshJwt") or refresh_jwt
+        _token_created_at = time.monotonic()
+        _session_error = None
+        return token
+    except http.HTTPError as e:
+        _refresh_error_status = e.status_code
+        _session_error = (
+            "Bluesky refresh token is invalid (401/400)"
+            if e.status_code in (400, 401)
+            else f"Bluesky session refresh failed (HTTP {e.status_code})"
+        )
+        _log(f"Session refresh failed: {_session_error}")
+        return None
+    except Exception as e:
+        _session_error = f"Bluesky session refresh failed: {type(e).__name__}"
+        _log(f"Session refresh failed: {_session_error}")
+        return None
+
+
 def _reset_session_cache() -> None:
-    global _cached_token, _token_created_at, _session_error
+    global _cached_token, _cached_refresh_token, _token_created_at
+    global _session_error, _refresh_error_status
     _cached_token = None
+    _cached_refresh_token = None
     _token_created_at = 0.0
     _session_error = None
+    _refresh_error_status = None
 
 
 def _extract_core_subject(topic: str) -> str:
@@ -231,11 +285,7 @@ def search_bluesky(
     }
     url = f"{_resolve_search_url(config)}?{urlencode(params)}"
 
-    def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
-        token = _create_session(handle, app_password)
-        if not token:
-            error_msg = _session_error or "Bluesky session creation failed (unknown error)"
-            return None, error_msg
+    def _search_with_token(token: str) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
         try:
             response = http.request(
                 "GET", url,
@@ -246,7 +296,6 @@ def search_bluesky(
         except http.HTTPError as e:
             _log(f"Search failed: {e}")
             if e.status_code == 401:
-                _reset_session_cache()
                 return None, "refresh"
             if e.status_code == 403 and e.body and "cloudflare" in e.body.lower():
                 return None, "Bluesky search blocked by Cloudflare (403). This is a network-level block - try a different network or VPN."
@@ -255,10 +304,35 @@ def search_bluesky(
             _log(f"Search failed: {e}")
             return None, f"Bluesky search failed: {type(e).__name__}: {e}"
 
+    def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
+        token = _create_session(handle, app_password)
+        if not token:
+            error_msg = _session_error or "Bluesky session creation failed (unknown error)"
+            return None, error_msg
+        return _search_with_token(token)
+
     response, error_msg = _auth_and_search()
     if error_msg == "refresh":
-        _log("Session expired; recreating token and retrying once")
-        response, error_msg = _auth_and_search()
+        _log("Session expired; refreshing access token and retrying once")
+        refreshed_token = _refresh_session(_cached_refresh_token)
+        if refreshed_token:
+            response, error_msg = _search_with_token(refreshed_token)
+        elif not _cached_refresh_token or _refresh_error_status in (400, 401):
+            _log("Refresh token missing or rejected; recreating session once")
+            _reset_session_cache()
+            response, error_msg = _auth_and_search()
+            if error_msg == "refresh":
+                error_msg = (
+                    "Bluesky session remained unauthorized after refresh and re-authentication; "
+                    "check credentials or app password"
+                )
+        else:
+            response, error_msg = None, _session_error
+        if error_msg == "refresh":
+            error_msg = (
+                "Bluesky session remained unauthorized after token refresh; "
+                "check credentials or app password"
+            )
     if error_msg:
         return {"posts": [], "error": error_msg}
     if response is None:

package/skills/trends/scripts/lib/bounded_get.py

@@ -0,0 +1,156 @@
+"""Owned GET transport for operation deadlines, including DNS and body reads."""
+
+from __future__ import annotations
+
+import base64
+from email.message import Message
+import io
+import json
+import socket
+import ssl
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from lib import subproc
+
+# Reserve TERM and KILL/reap time inside the caller's operation budget.
+CLEANUP_GRACE_SECONDS = 0.1
+
+
+class GetTimeout(TimeoutError):
+    def __init__(self, message, *, started):
+        super().__init__(message)
+        self.started = started
+
+
+class GetLaunchError(OSError):
+    """GET worker launch failed before the worker received a request."""
+
+
+def _error_record(exc):
+    if isinstance(exc, urllib.error.URLError):
+        reason = exc.reason
+        return {"kind": "URLError", "reason": _error_record(reason) if isinstance(reason, BaseException) else str(reason)}
+    return {"kind": type(exc).__name__, "message": str(exc), "errno": getattr(exc, "errno", None)}
+
+
+def _error_from_record(record):
+    if record["kind"] == "URLError":
+        reason = record["reason"]
+        return urllib.error.URLError(_error_from_record(reason) if isinstance(reason, dict) else reason)
+    kinds = {
+        "gaierror": socket.gaierror,
+        "TimeoutError": TimeoutError,
+        "ConnectionResetError": ConnectionResetError,
+        "ConnectionRefusedError": ConnectionRefusedError,
+        "OSError": OSError,
+        "SSLError": ssl.SSLError,
+        "SSLCertVerificationError": ssl.SSLCertVerificationError,
+    }
+    kind = kinds.get(record["kind"], OSError)
+    if record.get("errno") is not None:
+        return kind(record["errno"], record["message"])
+    return kind(record["message"])
+
+
+def get(
+    req: urllib.request.Request,
+    *,
+    timeout: float,
+    deadline_monotonic: float,
+    cancel: threading.Event | None = None,
+    read_body: bool = True,
+    read_error_body: bool = True,
+):
+    if req.get_method() != "GET":
+        raise ValueError("bounded transport supports GET only")
+    payload = {
+        "url": req.full_url,
+        "headers": req.headers,
+        "unredirected_headers": req.unredirected_hdrs,
+        "origin_req_host": req.origin_req_host,
+        "timeout": timeout,
+        "read_body": read_body,
+        "read_error_body": read_error_body,
+    }
+    transport_deadline = deadline_monotonic - 2 * CLEANUP_GRACE_SECONDS
+    try:
+        result = subproc.run_with_timeout(
+            [sys.executable, "-I", str(Path(__file__).resolve())],
+            timeout=max(0, transport_deadline - time.monotonic()),
+            deadline_monotonic=transport_deadline,
+            input_text=json.dumps(payload),
+            cancel=cancel,
+            cleanup_grace=CLEANUP_GRACE_SECONDS,
+        )
+    except subproc.SubprocTimeout as exc:
+        raise GetTimeout("GET exceeded operation deadline or was cancelled", started=exc.started) from exc
+    except OSError as exc:
+        if not getattr(exc, "_last30days_subproc_launch_failed", False):
+            raise
+        raise GetLaunchError(*exc.args) from exc
+    if result.returncode != 0:
+        raise OSError("bounded GET worker failed")
+    record = json.loads(result.stdout)
+    if "error" in record:
+        raise _error_from_record(record["error"])
+    body = base64.b64decode(record["body"]) if record["body"] is not None else None
+    error = None
+    if record.get("http_error") is not None:
+        headers = Message()
+        for key, value in record["headers"]:
+            headers[key] = value
+        error = urllib.error.HTTPError(
+            record["url"], record["status"], record["http_error"],
+            headers, io.BytesIO(body or b""),
+        )
+    return record["status"], body, error
+
+
+def _main():
+    from lib import http
+
+    payload = json.load(sys.stdin)
+    req = urllib.request.Request(
+        payload["url"], headers=payload["headers"], method="GET",
+        origin_req_host=payload["origin_req_host"],
+    )
+    for key, value in payload["unredirected_headers"].items():
+        req.add_unredirected_header(key, value)
+    error_reason = None
+    try:
+        try:
+            response = http.open_request(req, payload["timeout"])
+        except urllib.error.HTTPError as exc:
+            response = exc
+            error_reason = str(exc.reason)
+        with response:
+            body = None
+            read_body = payload["read_error_body"] if error_reason is not None else payload["read_body"]
+            if read_body:
+                try:
+                    body = response.read()
+                except OSError:
+                    if error_reason is None:
+                        raise
+            result = {
+                "status": getattr(response, "status", 200) or 200,
+                "body": base64.b64encode(body).decode("ascii") if body is not None else None,
+                "headers": list(response.headers.items()),
+                "url": response.geturl(),
+                "http_error": error_reason,
+            }
+    except Exception as exc:
+        result = {"error": _error_record(exc)}
+    json.dump(result, sys.stdout)
+
+
+if __name__ == "__main__":
+    _main()

package/skills/trends/scripts/lib/brightdata.py

@@ -0,0 +1,291 @@
+"""Bright Data CLI adapter for last30days.
+
+Shells out to the ``brightdata`` CLI (``@brightdata/cli``) to run Bright
+Data Pipelines. The CLI owns authentication end to end -- ``brightdata
+login`` does a gh-style zero-click browser flow and stores credentials in
+a platform config directory -- so this module never handles a login, and
+never reads credential *contents*: the auth probe is presence-only.
+
+Activation gate: two-way, mirroring the digg CLI-gated precedent but with
+an auth dimension the digg source does not have.
+
+1. ``shutil.which("brightdata")`` must resolve on the **agent subprocess
+   PATH** (not merely exist on disk -- Hermes/OpenClaw gateways often drop
+   ``~/.local/bin``).
+2. A credential signal must be present: either ``BRIGHTDATA_API_KEY``
+   resolved through the normal config layering, or the CLI's own
+   credentials file in the platform config dir.
+
+The second check is deliberately offline. A stale token passes it and
+then 401s fast at call time; that path degrades to empty results with the
+CLI's own error line preserved in the envelope, which is the AE2 contract.
+
+Metering note (R13): no pricing logic lives here. One pipeline request
+costs one credit against the account's monthly free tier regardless of how
+many records come back, so caps in the calling adapter bound *records*
+(paid-tier cost), not credits. Credit and auth warnings from the CLI are
+passed through verbatim rather than interpreted.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence
+
+from . import log, subproc, usage
+
+
+CLI_BIN = "brightdata"
+
+# Env var carrying an explicit API key. Registered in env.py so `.env` file
+# and keychain users pass the gate the same way process-env users do; when
+# it resolves from a non-process-env layer we pass it in the child environment.
+API_KEY_ENV = "BRIGHTDATA_API_KEY"
+
+# Credentials filename written by `brightdata login`. Probed for existence
+# only -- never opened, parsed, or logged.
+_CREDENTIALS_FILENAME = "credentials.json"
+_CONFIG_DIRNAME = "brightdata-cli"
+
+# The CLI's own polling timeout sits below our subprocess timeout so the CLI
+# exits cleanly with its own error rather than being SIGTERM'd mid-poll. Its
+# timeout path throws with zero records (verified in its polling module --
+# never partial output), so a timed-out pull is a clean parseable failure.
+_CLI_TIMEOUT_MARGIN = 10
+
+
+def _log(msg: str) -> None:
+    log.source_log("BrightData", msg, tty_only=False)
+
+
+def _config_dir() -> Path:
+    """Platform config directory the Bright Data CLI stores credentials in.
+
+    Mirrors the CLI's own credentials module: APPDATA on Windows, the
+    Application Support tree on macOS, XDG_CONFIG_HOME (or ~/.config) on
+    everything else.
+    """
+    if sys.platform == "win32":
+        base = os.environ.get("APPDATA")
+        root = Path(base) if base else Path.home() / "AppData" / "Roaming"
+    elif sys.platform == "darwin":
+        root = Path.home() / "Library" / "Application Support"
+    else:
+        base = os.environ.get("XDG_CONFIG_HOME")
+        root = Path(base) if base else Path.home() / ".config"
+    return root / _CONFIG_DIRNAME
+
+
+def is_installed() -> bool:
+    """True when the brightdata binary resolves on the agent subprocess PATH."""
+    return shutil.which(CLI_BIN) is not None
+
+
+def _api_key(config: Optional[Dict[str, Any]]) -> str:
+    if not config:
+        return ""
+    return str(config.get(API_KEY_ENV) or "").strip()
+
+
+def has_credentials(config: Optional[Dict[str, Any]] = None) -> bool:
+    """True when some credential signal exists, without reading any secret.
+
+    Presence-only by design: an explicit API key resolved through config
+    layering, or the existence of the CLI's credentials file. The file is
+    never opened. This cannot distinguish a live token from an expired one
+    -- that is what the fast 401 at call time is for.
+    """
+    if _api_key(config):
+        return True
+    try:
+        return (_config_dir() / _CREDENTIALS_FILENAME).exists()
+    except OSError:
+        return False
+
+
+def is_available(config: Optional[Dict[str, Any]] = None) -> bool:
+    """The full activation gate: binary on PATH *and* a credential signal."""
+    return is_installed() and has_credentials(config)
+
+
+def gate_status(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
+    """Two-field probe for ``pipeline.diagnose`` (bird_installed precedent).
+
+    Network-free, so it is safe on the ``--diagnose`` / doctor path.
+    """
+    installed = is_installed()
+    return {
+        "brightdata_installed": installed,
+        "brightdata_authenticated": installed and has_credentials(config),
+    }
+
+
+def _build_args(
+    pipeline_type: str,
+    params: Sequence[str],
+    *,
+    cli_timeout: int,
+) -> List[str]:
+    """Assemble the CLI invocation.
+
+    The API key is deliberately **absent** here -- it travels in the child's
+    environment instead (see ``_child_env``). Process arguments are not a
+    secret channel: ``/proc/<pid>/cmdline`` is world-readable under the
+    default ``hidepid=0``, and a review pull lives for up to 180s, so a key
+    on the command line is readable by any other local user and is captured
+    verbatim by execve auditing, process accounting, and any monitoring
+    agent that snapshots ``ps``. Mirrors the ``bird_x`` cookie-injection
+    precedent.
+
+    Positional params are fenced behind ``--`` so a keyword that happens to
+    begin with a dash is parsed as a search term rather than as an option.
+    """
+    return [
+        CLI_BIN,
+        "pipelines",
+        pipeline_type,
+        "--json",
+        "--timeout",
+        str(cli_timeout),
+        "--",
+        *(str(p) for p in params),
+    ]
+
+
+def _child_env(api_key: str) -> Optional[Dict[str, str]]:
+    """Environment for the child process, carrying the key when we have one.
+
+    Returns None when there is nothing to inject, so the child simply
+    inherits the parent environment (the common case: the CLI owns its own
+    credentials file, or the key is already exported).
+    """
+    if not api_key:
+        return None
+    return {**os.environ, API_KEY_ENV: api_key}
+
+
+def _scrub(text: str, secret: str) -> str:
+    """Remove a secret from text before it is logged or returned.
+
+    Defense in depth for the passthrough paths: the stderr lines this
+    module deliberately surfaces are auth and quota failures, which are
+    exactly the messages a CLI is most likely to echo the rejected
+    credential back in.
+    """
+    if not secret or not text:
+        return text
+    return text.replace(secret, "***")
+
+
+def _extract_records(payload: Any) -> List[Dict[str, Any]]:
+    """Pull the record list out of a parsed CLI payload.
+
+    Verified live (2026-08-13): both amazon pipelines return a **bare JSON
+    array** of flat record dicts, not the ``{"results": [...]}`` envelope the
+    digg CLI uses. The dict branches below are defensive against CLI churn,
+    which is a live risk on a package this young.
+    """
+    if isinstance(payload, list):
+        return [r for r in payload if isinstance(r, dict)]
+    if isinstance(payload, dict):
+        for key in ("records", "results", "data"):
+            value = payload.get(key)
+            if isinstance(value, list):
+                return [r for r in value if isinstance(r, dict)]
+    return []
+
+
+def run_pipeline(
+    pipeline_type: str,
+    params: Sequence[str],
+    *,
+    timeout: int,
+    config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Run one Bright Data pipeline and return ``{"records", "error"}``.
+
+    Never raises. Every failure mode -- missing binary, spawn failure,
+    subprocess timeout, non-zero exit, unparseable stdout -- returns empty
+    records plus a one-line ``error`` string, so callers can record the
+    failure in ``errors_by_source`` without branching on exception types.
+
+    The CLI's first stderr line is preserved verbatim as the error (auth
+    401s and low-credit warnings are the cases that matter), and also
+    mirrored to ``source_log`` so the failure is visible in non-TTY hosts.
+
+    Args:
+        pipeline_type: pipeline name, e.g. ``amazon_product_search``.
+        params: positional pipeline params, passed through in order.
+        timeout: subprocess timeout in seconds. The CLI's own polling
+            timeout is set just below this so it can fail cleanly first.
+        config: resolved config dict, consulted only for the API key.
+
+    Returns:
+        ``{"records": [...]}`` on success, else ``{"records": [], "error": str}``.
+    """
+    if not is_installed():
+        return {"records": [], "error": f"{CLI_BIN} not on PATH"}
+
+    cli_timeout = max(5, int(timeout) - _CLI_TIMEOUT_MARGIN)
+    key = _api_key(config)
+    cmd = _build_args(pipeline_type, params, cli_timeout=cli_timeout)
+
+    try:
+        usage.begin("brightdata")
+        result = subproc.run_with_timeout(cmd, timeout=timeout, env=_child_env(key))
+    except subproc.SubprocTimeout as exc:
+        _log(f"Timeout: {exc}")
+        return {"records": [], "error": str(exc)}
+    except FileNotFoundError as exc:
+        _log(f"Binary missing: {exc}")
+        return {"records": [], "error": str(exc)}
+    except OSError as exc:
+        _log(f"Spawn failed: {exc}")
+        return {"records": [], "error": str(exc)}
+
+    stderr = _scrub(result.stderr or "", key)
+    _passthrough_warnings(stderr)
+
+    if result.returncode != 0:
+        lines = [ln.strip() for ln in stderr.strip().splitlines() if ln.strip()]
+        # The CLI narrates polling progress on stderr, so the *last* line is
+        # the actual failure; the first line is "Triggering pipeline...".
+        first = lines[-1] if lines else f"exit {result.returncode}"
+        _log(f"CLI exit {result.returncode}: {first}")
+        return {"records": [], "error": first}
+
+    stdout = result.stdout or ""
+    if not stdout.strip():
+        return {"records": []}
+    try:
+        payload = json.loads(stdout)
+    except json.JSONDecodeError as exc:
+        _log(f"JSON decode failed: {exc}")
+        return {"records": [], "error": f"json decode: {exc}"}
+
+    return {"records": _extract_records(payload)}
+
+
+# Substrings that mark a stderr line worth surfacing even on a successful
+# run -- credit exhaustion and auth trouble are the two the user must see.
+# Matched case-insensitively against the CLI's own wording, and echoed
+# verbatim rather than reworded (R13: no pricing logic, no interpretation).
+_WARNING_MARKERS = ("credit", "quota", "balance", "unauthor", "401", "expired", "login")
+
+
+def _passthrough_warnings(stderr: str) -> None:
+    """Echo credit/auth warning lines from the CLI verbatim.
+
+    Skips the routine polling narration so a normal run stays quiet.
+    """
+    for line in (stderr or "").splitlines():
+        text = line.strip()
+        if not text or text.lower().startswith(("status:", "triggering", "triggered", "data received")):
+            continue
+        lowered = text.lower()
+        if any(marker in lowered for marker in _WARNING_MARKERS):
+            _log(text)

package/skills/trends/scripts/lib/categories.py

@@ -23,6 +23,7 @@ before `ai_chat_model` so "gpt image 2" matches the image-gen category.
 
 from __future__ import annotations
 
+import re
 from typing import List, Optional, TypedDict
 
 
@@ -126,6 +127,8 @@ CATEGORY_PEERS: dict[str, _CategoryEntry] = {
     },
     "ai_agent_framework": {
         "patterns": [
+            "ai agent",
+            "ai agents",
             "agent framework",
             "agentic framework",
             "langchain",
@@ -264,7 +267,10 @@ def detect_category(topic: Optional[str]) -> Optional[str]:
     lowered = topic.lower()
     for category_id, entry in CATEGORY_PEERS.items():
         for pattern in entry["patterns"]:
-            if pattern in lowered:
+            # Word-boundary match: "ai agent" must not fire on "Dubai agents"
+            # or "Thai agents". Substring matching classified those as
+            # ai_agent_framework and routed discovery to LangChain subreddits.
+            if re.search(rf"(?<![a-z0-9]){re.escape(pattern)}(?![a-z0-9])", lowered):
                 return category_id
     return None