luckiest-co 1.0.46 → 1.0.48

package/skills/trends/scripts/lib/providers.py

@@ -2,11 +2,13 @@
 
 from __future__ import annotations
 
+import ipaddress
 import json
 import os
 import re
 import sys
 from typing import Any
+from urllib.parse import urlsplit, urlunsplit
 
 from . import env, http, schema
 
@@ -24,6 +26,93 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
 # constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
 # double-check that OpenRouter's slug still maps to the same upstream model.
 OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
+PROVIDER_BASE_URL_KEYS = frozenset({"OPENAI_BASE_URL", "XAI_BASE_URL", "OPENROUTER_BASE_URL"})
+
+
+def _is_loopback(host: str) -> bool:
+    """True for hosts that never leave the machine."""
+    if host == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def allowed_base_url_override(value: str) -> bool:
+    try:
+        parts = urlsplit(value)
+        host = parts.hostname
+    except ValueError:
+        return False
+    return bool(host) and (
+        parts.scheme == "https" or (parts.scheme == "http" and _is_loopback(host))
+    )
+
+
+def is_loopback_http_endpoint(url: str) -> bool:
+    parts = urlsplit(url)
+    return parts.scheme == "http" and _is_loopback(parts.hostname or "")
+
+
+def base_url_override(key: str, default: str) -> str:
+    """Resolve a provider base-URL override, refusing cleartext remote hosts.
+
+    Every provider resolves its endpoint through here so one guard covers both
+    ways a value arrives: the process environment, and
+    ``_propagate_config_to_environ`` pushing a `.env` value into os.environ.
+
+    A base-URL override redirects the request that carries the provider's bearer
+    token, so an `http://` override to a remote host would put the API key on
+    the wire in cleartext. That is refused and the built-in vendor endpoint is
+    used instead - failing closed protects the credential, and the warning makes
+    the drop visible instead of leaving the user to wonder why their gateway is
+    being bypassed. Loopback is the one legitimate `http://` case (a local
+    LiteLLM/Ollama gateway or an SSH tunnel never leaves the machine), so it is
+    allowed.
+    """
+    raw = (os.environ.get(key) or "").strip()
+    if not raw:
+        return default
+    if allowed_base_url_override(raw):
+        return raw
+    sys.stderr.write(
+        f"[last30days] WARNING: ignoring {key} - a provider endpoint override "
+        "must be https:// (http:// is allowed only on localhost), otherwise the API "
+        f"key would be sent in cleartext. Using {default} instead.\n"
+    )
+    sys.stderr.flush()
+    return default
+
+
+_ENDPOINT_PATHS = {
+    OPENAI_RESPONSES_URL: "/responses",
+    XAI_RESPONSES_URL: "/responses",
+    OPENROUTER_URL: "/chat/completions",
+}
+
+
+def resolve_endpoint(env_var: str, default_url: str) -> str:
+    """Resolve a ``*_BASE_URL`` override into a full endpoint URL.
+
+    By the convention every OpenAI-compatible provider documents, ``*_BASE_URL``
+    names the API root (``https://host/v1``) and the client appends the endpoint
+    path. This module historically required the full endpoint URL instead, so a
+    value copied from a provider's setup guide POSTed to the API root and failed.
+
+    Accept both forms: a host or versioned API root gets the endpoint path
+    appended, and a complete gateway route is used unchanged. Query strings
+    stay after the path in either form.
+    """
+    override = base_url_override(env_var, default_url)
+    if override == default_url:
+        return default_url
+    parts = urlsplit(override)
+    root_path = parts.path.rstrip("/")
+    last_segment = root_path.rsplit("/", 1)[-1]
+    if not root_path or re.fullmatch(r"v\d+(?:beta\d*)?", last_segment):
+        return urlunsplit(parts._replace(path=root_path + _ENDPOINT_PATHS[default_url]))
+    return override
 
 
 class ReasoningClient:
@@ -31,6 +120,95 @@ class ReasoningClient:
 
     name: str
 
+    def __init__(self) -> None:
+        self._usage_calls = 0
+        self._usage_prompt_tokens = 0
+        self._usage_completion_tokens = 0
+        self._usage_complete = True
+
+    @staticmethod
+    def _valid_token_count(value: Any) -> bool:
+        return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+    def record_usage(
+        self,
+        prompt_tokens: Any,
+        completion_tokens: Any,
+        total_tokens: Any = None,
+    ) -> None:
+        self._usage_calls += 1
+        if not self._valid_token_count(prompt_tokens):
+            self._usage_complete = False
+            return
+        if total_tokens is not None:
+            if (
+                not self._valid_token_count(total_tokens)
+                or total_tokens < prompt_tokens
+                or (
+                    completion_tokens is not None
+                    and (
+                        not self._valid_token_count(completion_tokens)
+                        or total_tokens < prompt_tokens + completion_tokens
+                    )
+                )
+            ):
+                self._usage_complete = False
+                return
+            # Reported totals can include reasoning tokens absent from completion counts.
+            completion_tokens = total_tokens - prompt_tokens
+        elif not self._valid_token_count(completion_tokens):
+            self._usage_complete = False
+            return
+        self._usage_prompt_tokens += prompt_tokens
+        self._usage_completion_tokens += completion_tokens
+
+    @property
+    def total_usage(self) -> dict[str, int] | None:
+        if not self._usage_calls or not self._usage_complete:
+            return None
+        return {
+            "calls": self._usage_calls,
+            "promptTokens": self._usage_prompt_tokens,
+            "completionTokens": self._usage_completion_tokens,
+            "totalTokens": self._usage_prompt_tokens + self._usage_completion_tokens,
+        }
+
+    def _mark_usage_incomplete(self) -> None:
+        self._usage_complete = False
+
+    def _post(self, url: str, payload: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
+        try:
+            return http.post(url, payload, on_retry=self._mark_usage_incomplete, **kwargs)
+        except Exception:
+            self._mark_usage_incomplete()
+            raise
+
+    def _record_response_usage(
+        self,
+        response: dict[str, Any],
+        *,
+        metadata_key: str,
+        prompt_key: str,
+        completion_key: str,
+        total_key: str,
+        prompt_fallback_key: str | None = None,
+        completion_fallback_key: str | None = None,
+    ) -> None:
+        usage = response.get(metadata_key)
+        if not isinstance(usage, dict):
+            usage = {}
+        prompt_tokens = usage.get(prompt_key)
+        completion_tokens = usage.get(completion_key)
+        if prompt_tokens is None and prompt_fallback_key:
+            prompt_tokens = usage.get(prompt_fallback_key)
+        if completion_tokens is None and completion_fallback_key:
+            completion_tokens = usage.get(completion_fallback_key)
+        self.record_usage(
+            prompt_tokens,
+            completion_tokens,
+            usage.get(total_key),
+        )
+
     def generate_text(
         self,
         model: str,
@@ -56,6 +234,7 @@ class GeminiClient(ReasoningClient):
     name = "gemini"
 
     def __init__(self, api_key: str):
+        super().__init__()
         self.api_key = api_key
 
     def _generate_content(
@@ -74,7 +253,7 @@ class GeminiClient(ReasoningClient):
             body["generationConfig"]["responseMimeType"] = response_mime_type
         if tools:
             body["tools"] = tools
-        return http.post(
+        return self._post(
             GEMINI_URL.format(model=model, api_key=self.api_key),
             body,
             headers={"Content-Type": "application/json"},
@@ -95,12 +274,20 @@ class GeminiClient(ReasoningClient):
             tools=tools,
             response_mime_type=response_mime_type,
         )
+        self._record_response_usage(
+            payload,
+            metadata_key="usageMetadata",
+            prompt_key="promptTokenCount",
+            completion_key="candidatesTokenCount",
+            total_key="totalTokenCount",
+        )
         return extract_gemini_text(payload)
 
 class OpenAIClient(ReasoningClient):
     name = "openai"
 
     def __init__(self, token: str):
+        super().__init__()
         self.token = token
 
     def generate_text(
@@ -118,14 +305,23 @@ class OpenAIClient(ReasoningClient):
             "input": prompt,
             "temperature": 0,
         }
-        response = http.post(
-            os.environ.get("OPENAI_BASE_URL", OPENAI_RESPONSES_URL),
+        endpoint = resolve_endpoint("OPENAI_BASE_URL", OPENAI_RESPONSES_URL)
+        response = self._post(
+            endpoint,
             payload,
             headers={
                 "Authorization": f"Bearer {self.token}",
                 "Content-Type": "application/json",
             },
             timeout=90,
+            bypass_proxy=is_loopback_http_endpoint(endpoint),
+        )
+        self._record_response_usage(
+            response,
+            metadata_key="usage",
+            prompt_key="input_tokens",
+            completion_key="output_tokens",
+            total_key="total_tokens",
         )
         return extract_openai_text(response)
 
@@ -134,6 +330,7 @@ class XAIClient(ReasoningClient):
     name = "xai"
 
     def __init__(self, api_key: str):
+        super().__init__()
         self.api_key = api_key
 
     def generate_text(
@@ -149,14 +346,25 @@ class XAIClient(ReasoningClient):
             "model": model,
             "input": [{"role": "user", "content": prompt}],
         }
-        response = http.post(
-            os.environ.get("XAI_BASE_URL", XAI_RESPONSES_URL),
+        endpoint = resolve_endpoint("XAI_BASE_URL", XAI_RESPONSES_URL)
+        response = self._post(
+            endpoint,
             payload,
             headers={
                 "Authorization": f"Bearer {self.api_key}",
                 "Content-Type": "application/json",
             },
             timeout=90,
+            bypass_proxy=is_loopback_http_endpoint(endpoint),
+        )
+        self._record_response_usage(
+            response,
+            metadata_key="usage",
+            prompt_key="input_tokens",
+            completion_key="output_tokens",
+            total_key="total_tokens",
+            prompt_fallback_key="prompt_tokens",
+            completion_fallback_key="completion_tokens",
         )
         return extract_openai_text(response)
 
@@ -165,6 +373,7 @@ class OpenRouterClient(ReasoningClient):
     name = "openrouter"
 
     def __init__(self, api_key: str):
+        super().__init__()
         self.api_key = api_key
 
     def generate_text(
@@ -181,14 +390,23 @@ class OpenRouterClient(ReasoningClient):
             "messages": [{"role": "user", "content": prompt}],
             "temperature": 0,
         }
-        response = http.post(
-            OPENROUTER_URL,
+        endpoint = resolve_endpoint("OPENROUTER_BASE_URL", OPENROUTER_URL)
+        response = self._post(
+            endpoint,
             payload,
             headers={
                 "Authorization": f"Bearer {self.api_key}",
                 "Content-Type": "application/json",
             },
             timeout=90,
+            bypass_proxy=is_loopback_http_endpoint(endpoint),
+        )
+        self._record_response_usage(
+            response,
+            metadata_key="usage",
+            prompt_key="prompt_tokens",
+            completion_key="completion_tokens",
+            total_key="total_tokens",
         )
         return extract_openai_text(response)
 
@@ -315,9 +533,12 @@ def resolve_runtime(config: dict[str, Any], depth: str) -> tuple[schema.Provider
 
 
 def _resolve_x_backend(config: dict[str, Any]) -> str | None:
-    preferred = (config.get("LAST30DAYS_X_BACKEND") or "").lower()
-    if preferred in {"xai", "bird"}:
-        return preferred
+    """Resolve the X backend for runtime fetch.
+
+    Delegates to env.get_x_source which handles:
+    - Any known pin (X_BACKEND_KNOWN) exclusively: returns pin if available, None otherwise
+    - Unpinned: walks auto-chain (X_BACKEND_ORDER) only, never auto-selects opt-in backends
+    """
     return env.get_x_source(config)
 
 

package/skills/trends/scripts/lib/quality_nudge.py

@@ -1,13 +1,21 @@
 """Post-research quality score and upgrade nudge.
 
-Computes a quality score based on 5 core sources and builds
+Computes a quality score based on the non-blocking core sources and builds
 a nudge message describing what the user missed and how to fix it.
+
+Fix text comes from ``lib.prescriptions`` (the single remediation
+vocabulary shared with the doctor command, KTD 7); only the trigger
+logic and the message framing live here.
 """
 
-from typing import List
+from typing import List, Optional
+
+from . import env, health, http, prescriptions, x_envelope
 
 
-# The 5 core sources
+# Sources whose absence can justify a post-run quality repair. X remains a
+# supported source, but it is optional: declining cookie access must not turn a
+# successful multi-source run into a setup prompt or a lower quality grade.
 CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit"]
 
 # Labels for display
@@ -22,6 +30,8 @@ SOURCE_LABELS = {
 
 def _is_x_active(config: dict, research_results: dict) -> bool:
     """Check if X source is active (has credentials AND didn't error)."""
+    if "x" in (research_results.get("active_sources") or []):
+        return not bool(research_results.get("x_error"))
     has_creds = _has_x_credentials(config)
     if not has_creds:
         return False
@@ -32,12 +42,60 @@ def _is_x_active(config: dict, research_results: dict) -> bool:
 
 
 def _has_x_credentials(config: dict) -> bool:
-    """Return True when any X/Twitter source credential is configured."""
-    return bool(
+    """Return True when any X/Twitter source credential is configured.
+
+    ``X_BEARER_TOKEN`` counts where the xapi backend can actually run: an
+    official-only host (it heads the auto chain there) or an explicit xapi
+    pin. An ambient bearer on any other host is not a configured X source,
+    so nothing changes there.
+    """
+    if (
         config.get("AUTH_TOKEN")
         or config.get("XAI_API_KEY")
         or config.get("XQUIK_API_KEY")
-    )
+    ):
+        return True
+    if config.get("X_BEARER_TOKEN"):
+        return "xapi" in env.x_auto_chain(config) or env.x_backend_pin(config) == "xapi"
+    return False
+
+
+def _x_error_prescription(config: dict, research_results: dict) -> prescriptions.Prescription:
+    """The fix for a configured X that errored, routed through the X policy.
+
+    The pipeline stamps the failed backend into ``x_error`` or
+    ``x_degraded_error``, sometimes behind a simplified-query retry label.
+    Use that runtime provenance before the host policy.
+    """
+    message = str(research_results.get("x_error") or research_results.get("x_degraded_error") or "")
+    if message.startswith(x_envelope.DETAIL_NOT_PASSED):
+        # The model declared the X connector lane and passed no envelope:
+        # the fix is the connector, on any host.
+        return prescriptions.for_x(config, "connector_missing")
+    backend_error = message.removeprefix("Simplified-query retry failed: ")
+    backend_error = backend_error.removeprefix("All X backends failed — ")
+    if backend_error.startswith("X served via "):
+        served_backend, separator, origin_error = backend_error.removeprefix("X served via ").partition(" after ")
+        if separator:
+            backend_error = origin_error.split(f"; {served_backend}", 1)[0]
+    if backend_error.startswith("xai:"):
+        state = http.classify_failure(message=backend_error)
+        if state == health.PAYMENT_REQUIRED:
+            failure = "xai_payment_required"
+        elif state == health.RATE_LIMITED:
+            failure = "xai_rate_limited"
+        elif state == health.TIMEOUT:
+            failure = "xai_timeout"
+        elif state == health.AUTH_FAILED or "model" in backend_error.lower():
+            failure = "xai_error"
+        else:
+            failure = "xai_unavailable"
+        return prescriptions.for_x(config, failure)
+    failure = "cookies_expired"
+    if env.x_policy(config).hint_namespace == "official":
+        if http.classify_failure(message=message) == health.PAYMENT_REQUIRED:
+            failure = "payment_required"
+    return prescriptions.for_x(config, failure)
 
 
 def _has_ytdlp() -> bool:
@@ -143,7 +201,7 @@ def _is_instagram_silent_failure(config: dict, research_results: dict) -> bool:
 
 
 def compute_quality_score(config: dict, research_results: dict) -> dict:
-    """Compute research quality score based on 5 core sources.
+    """Compute research quality score from the non-blocking core sources.
 
     Args:
         config: Configuration dict from env.get_config()
@@ -153,13 +211,14 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
             degraded-YouTube detection (transcript-fetch ratio below threshold,
             or fallback/provider data returned without local yt-dlp).
             Optional key ``instagram_items_count`` enables silent-failure
-            detection for the bonus Instagram source.
+            detection for the bonus Instagram source. ``x_degraded_error``
+            carries a failed xAI lane when another backend served X items.
 
     Returns:
         {
-            "score_pct": 40-100,
+            "score_pct": 0-100,
             "core_active": ["hn", "polymarket", ...],
-            "core_missing": ["x", "youtube"],
+            "core_missing": ["youtube"],
             "core_errored": [],          # configured but errored at top level
             "core_degraded": [],         # configured and returned items but quality below threshold
             "bonus_errored": [],         # bonus sources (Instagram, etc.) configured but silent
@@ -177,14 +236,22 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
     core_active.append("polymarket")
     core_active.append("reddit")
 
-    # X
-    has_x_creds = _has_x_credentials(config)
+    # X splits three ways. Active counts normally. Configured-but-errored is a
+    # real outage: it still docks the score and surfaces a repair, never an
+    # "optional omission". Only unconfigured/declined X leaves the denominator.
+    optional_omitted: List[str] = []
+    x_configured = _has_x_credentials(config) or (
+        "x" in (research_results.get("active_sources") or [])
+    )
     if _is_x_active(config, research_results):
         core_active.append("x")
-    else:
+        if research_results.get("x_degraded_error"):
+            core_degraded.append("x")
+    elif x_configured and research_results.get("x_error"):
         core_missing.append("x")
-        if has_x_creds and research_results.get("x_error"):
-            core_errored.append("x")
+        core_errored.append("x")
+    else:
+        optional_omitted.append("x")
 
     # YouTube
     has_ytdlp = _has_ytdlp()
@@ -219,10 +286,12 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
     if _is_instagram_silent_failure(config, research_results):
         bonus_errored.append("instagram")
 
-    score_pct = int(len(core_active) / 5 * 100)
+    scored_source_count = len(CORE_SOURCES) - len(optional_omitted)
+    score_pct = int(len(core_active) / scored_source_count * 100)
 
     has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
     active_sources = research_results.get("active_sources") or []
+    x_fix = _x_error_prescription(config, research_results) if "x" in core_errored or "x" in core_degraded else None
     nudge_text = _build_nudge_text(
         core_missing,
         core_errored,
@@ -232,6 +301,8 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
         active_sources=active_sources,
         bonus_errored=bonus_errored,
         has_ytdlp=has_ytdlp,
+        core_total=scored_source_count,
+        x_fix=x_fix,
     ) if (core_missing or core_degraded or bonus_errored) else None
 
     return {
@@ -254,6 +325,8 @@ def _build_nudge_text(
     active_sources: list = None,
     bonus_errored: List[str] = None,
     has_ytdlp: bool = False,
+    core_total: int | None = None,
+    x_fix: Optional[prescriptions.Prescription] = None,
 ) -> str:
     """Build human-readable nudge text describing what was missed or degraded.
 
@@ -274,8 +347,9 @@ def _build_nudge_text(
         else:
             missed_parts.append(label)
 
-    active_count = 5 - len(core_missing)
-    lines.append(f"Research quality: {active_count}/5 core sources.")
+    effective_total = core_total if core_total is not None else len(CORE_SOURCES)
+    active_count = effective_total - len(core_missing)
+    lines.append(f"Research quality: {active_count}/{effective_total} core sources.")
     if missed_parts:
         lines.append(f"Missing: {', '.join(missed_parts)}.")
     if core_degraded:
@@ -289,29 +363,30 @@ def _build_nudge_text(
     # Free suggestions
     free_suggestions: List[str] = []
 
-    if "x" in core_missing:
-        if "x" in core_errored:
-            free_suggestions.append(
-                "X/Twitter errored - log into x.com in your browser, then re-run."
-            )
-        else:
-            free_suggestions.append(
-                "X/Twitter: real-time posts with likes and reposts - the fastest "
-                "signal for breaking topics. Three options: log into x.com in your "
-                "browser and re-run (cookies detected automatically), or add "
-                "XAI_API_KEY to your .env (get key at api.x.ai), or add "
-                "XQUIK_API_KEY to your .env (get key at xquik.com)."
-            )
+    # A configured X that errored is the only X entry that can reach
+    # core_missing: unconfigured/declined X is an optional omission and never
+    # lands here. Surface the repair instead of hiding the outage.
+    if "x" in core_missing and "x" in core_errored:
+        if x_fix is None:
+            x_fix = prescriptions.get("x", "cookies_expired")
+        free_suggestions.append(f"X/Twitter errored - {x_fix.fix_nl}.")
+
+    if "x" in core_degraded:
+        if x_fix is None:
+            x_fix = prescriptions.get("x", "cookies_expired")
+        free_suggestions.append(f"X/Twitter used a backup - {x_fix.fix_nl}.")
 
     if "youtube" in core_missing:
         if "youtube" in core_errored:
+            yt_fix = prescriptions.get("youtube", "ytdlp_stale")
             free_suggestions.append(
-                "YouTube errored - update yt-dlp: brew upgrade yt-dlp"
+                f"YouTube errored - update yt-dlp: {yt_fix.fix_cli}"
             )
         else:
+            yt_fix = prescriptions.get("youtube", "ytdlp_missing")
             free_suggestions.append(
                 "YouTube: video transcripts with key moments - often the deepest "
-                "explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
+                f"explanations on any topic. Install yt-dlp: {yt_fix.fix_cli} (free)"
             )
 
     if "youtube" in core_degraded:
@@ -319,12 +394,18 @@ def _build_nudge_text(
         transcripts = int(research_results.get("youtube_transcripts_count") or 0)
         captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)
         if not has_ytdlp and _youtube_returned_data(research_results):
+            install = prescriptions.get("youtube", "ytdlp_missing")
+            # Tolerant lookup: alt_cli makes no arity promise, so an entry
+            # gaining/losing a platform alternate must degrade the wording,
+            # never crash the nudge path.
+            scoop_install = install.alt_cli[0] if len(install.alt_cli) > 0 else install.fix_cli
+            pip_install = install.alt_cli[1] if len(install.alt_cli) > 1 else scoop_install
             free_suggestions.append(
                 f"YouTube returned {videos} videos and {transcripts} transcripts "
                 "through a fallback/provider path, but local yt-dlp is not "
                 "installed. Install yt-dlp to enable the free local YouTube lane "
-                "and reduce reliance on fallback providers: brew install yt-dlp "
-                "(macOS), scoop install yt-dlp (Windows), or pip install -U yt-dlp."
+                f"and reduce reliance on fallback providers: {install.fix_cli} "
+                f"(macOS), {scoop_install} (Windows), or {pip_install}."
             )
         else:
             captions_note = ""
@@ -333,13 +414,17 @@ def _build_nudge_text(
                     f" ({captions_disabled} of those had captions disabled by the "
                     "uploader, which is a separate cause and not fixable on your end)"
                 )
+            update = prescriptions.get("youtube", "ytdlp_stale")
+            # Same tolerant lookup as the install branch above.
+            scoop_update = update.alt_cli[0] if len(update.alt_cli) > 0 else update.fix_cli
+            pip_update = update.alt_cli[1] if len(update.alt_cli) > 1 else scoop_update
             free_suggestions.append(
                 f"YouTube returned {videos} videos but only {transcripts} transcripts "
                 f"captured{captions_note}. The most common remaining cause is a stale "
                 "yt-dlp binary - YouTube's caption format changes frequently and old "
                 "binaries silently fail every transcript. Update via your package "
-                "manager: scoop update yt-dlp (Windows), brew upgrade yt-dlp (macOS), "
-                "or pip install -U yt-dlp."
+                f"manager: {scoop_update} (Windows), {update.fix_cli} (macOS), "
+                f"or {pip_update}."
             )
 
     if "instagram" in bonus_errored: