luckiest-co 1.0.47 → 1.0.49

package/skills/trends/scripts/lib/trustpilot.py

@@ -15,12 +15,24 @@ Default-on safety (three gates):
      company/brand -- a domain-like token, or a short (<=2-word) capitalized
      proper noun. Generic phrases ("AI coding agents", "agent memory") and
      longer multi-word phrases never call the CLI, so Trustpilot stays quiet --
-     and never harvests Chrome -- on non-company topics.
+     and never harvests Chrome -- on non-company topics. An explicit resolved
+     domain (``--trustpilot-domain`` or an auto-resolve hint) bypasses this
+     gate: an explicit domain is proof of brand intent.
   2. Browser opt-out. Automated contexts (cron, CI, the eval harness) can set
      ``LAST30DAYS_TRUSTPILOT_NO_BROWSER`` to disable the source entirely, so a
      headless run never spawns the cookie harvest.
   3. Graceful degradation. Any CLI failure (no Chrome, expired cookie that
      cannot re-harvest, timeout) degrades to empty results, never an error.
+
+Domain resolution: Trustpilot review pages are keyed by domain
+(``www.thriftbooks.com``), not company name -- ``info ThriftBooks`` 404s.
+Priority chain: user flag (verbatim) > auto-resolve hint (retries via search
+on a miss) > domain token in the topic > CLI ``search`` name->domain lookup
+(cached per topic) > cleaned topic (legacy behavior).
+
+Session pre-flight: ``ensure_session_ready`` performs one serialized
+``auth status`` / ``auth login`` before the parallel fan-out so concurrent
+streams and vs-mode sub-runs never race their own Chrome harvests.
 """
 
 from __future__ import annotations
@@ -29,6 +41,8 @@ import json
 import os
 import re
 import shutil
+import threading
+import time
 from typing import Any, Dict, List, Optional
 
 from . import dates, log, subproc
@@ -39,8 +53,16 @@ CLI_BIN = "trustpilot-pp-cli"
 
 SEARCH_TIMEOUT = 75  # generous: a cold run may harvest a WAF cookie (~10s).
 
+AUTH_STATUS_TIMEOUT = 20  # auth status is a local SQLite read; fast.
+
 NO_BROWSER_ENV = "LAST30DAYS_TRUSTPILOT_NO_BROWSER"
 
+# Among name-matching search hits, the top hit must have this many times the
+# runner-up's review volume to win automatically. Lookalike/squatter pages have
+# tiny volume (ThriftBooks: 2.8M vs 130); comparable volume means genuine
+# ambiguity, where falling back beats silently picking the wrong company.
+DOMAIN_DOMINANCE_FACTOR = 50
+
 # Domain-like token, e.g. "chownow.com", "nothing.tech".
 _DOMAIN_RE = re.compile(r"\b[a-z0-9][a-z0-9-]*\.(com|io|co|net|org|app|ai|dev|gg|tech|shop|store)\b")
 
@@ -132,6 +154,192 @@ def _build_info_args(identifier: str) -> List[str]:
     return [CLI_BIN, "info", identifier, "--agent"]
 
 
+def _normalize_name(text: str) -> str:
+    """Case/whitespace/punctuation-insensitive brand-name key."""
+    return re.sub(r"[^a-z0-9]", "", (text or "").lower())
+
+
+# name->domain results, keyed by normalized topic. Per-topic (NOT a single
+# process-wide slot): vs-mode resolves several entities in one process, and a
+# single slot would serve entity A's domain to entity B.
+_domain_cache: Dict[str, Optional[str]] = {}
+_domain_cache_lock = threading.Lock()
+
+_warmup_lock = threading.Lock()
+_warmup_at: Optional[float] = None  # time.monotonic() of the last warm-up
+
+# Warm-up freshness window, matching the CLI's ~4-minute safe replay bound for
+# WAF tokens. A boolean-forever flag would leave long-lived host processes
+# running stale (and never retrying a failed login); the TTL re-checks cheaply
+# via `auth status` once the window lapses.
+WARMUP_TTL_SECONDS = 240
+
+
+def _reset_state_for_tests() -> None:
+    """Clear module-level caches/flags (tests only)."""
+    global _warmup_at
+    with _domain_cache_lock:
+        _domain_cache.clear()
+    _warmup_at = None
+
+
+def _warmup_fresh() -> bool:
+    return _warmup_at is not None and (time.monotonic() - _warmup_at) < WARMUP_TTL_SECONDS
+
+
+def _select_search_hit(topic: str, hits: List[Any]) -> Optional[str]:
+    """Pick the canonical domain from search hits, or None when ambiguous.
+
+    Name-match is mandatory: review volume must never override a name
+    mismatch, or the engine attributes another company's reviews to the topic.
+    Among name-matching hits the winner must dominate on review volume
+    (DOMAIN_DOMINANCE_FACTOR); comparable volume is genuine ambiguity and
+    falls back to legacy behavior.
+    """
+    want = _normalize_name(topic)
+    if not want:
+        return None
+    matching: List[tuple[int, str]] = []
+    for hit in hits:
+        if not isinstance(hit, dict):
+            continue
+        domain = str(hit.get("domain") or hit.get("identifyingName") or "").strip()
+        name = str(hit.get("displayName") or hit.get("name") or "").strip()
+        if not domain or _normalize_name(name) != want:
+            continue
+        try:
+            count = int(hit.get("numberOfReviews") or 0)
+        except (TypeError, ValueError):
+            count = 0
+        matching.append((count, domain))
+    if not matching:
+        top = next(
+            (str(h.get("domain") or "").strip() for h in hits
+             if isinstance(h, dict) and h.get("domain")),
+            "",
+        )
+        if top:
+            _log(
+                f"no name-matching search hit; top candidate was '{top}' - "
+                f"pass --trustpilot-domain to target it"
+            )
+        return None
+    matching.sort(reverse=True)
+    if len(matching) == 1:
+        return matching[0][1]
+    top_count, top_domain = matching[0]
+    runner_count, runner_domain = matching[1]
+    if top_count >= max(1, runner_count) * DOMAIN_DOMINANCE_FACTOR:
+        return top_domain
+    _log(
+        f"ambiguous search hits ('{top_domain}' vs '{runner_domain}'); "
+        f"falling back - pass --trustpilot-domain to disambiguate"
+    )
+    return None
+
+
+def _search_domain(topic: str) -> Optional[str]:
+    """Resolve a company name to its Trustpilot domain via the CLI's search.
+
+    Cached per normalized topic (thread-safe), so repeat lookups cost one
+    subprocess while vs-mode entities still resolve independently. Only
+    definitive results are cached: a transient CLI failure (timeout, spawn
+    error, malformed JSON) returns None WITHOUT caching, so one flaky search
+    does not suppress resolution for this topic for the rest of the process.
+    """
+    key = _normalize_name(topic)
+    if not key:
+        return None
+    with _domain_cache_lock:
+        if key in _domain_cache:
+            return _domain_cache[key]
+    data = _run_cli(
+        [CLI_BIN, "search", topic.strip(), "--limit", "5", "--agent"],
+        timeout=SEARCH_TIMEOUT,
+    )
+    if not isinstance(data, dict) or "error" in data:
+        return None  # transient failure: retry on the next lookup
+    hits = data.get("hits")
+    if not isinstance(hits, list):
+        # Degenerate payload (e.g. empty stdout parses to {}): not a
+        # definitive no-match -- do not cache, retry on the next lookup.
+        return None
+    domain = _select_search_hit(topic, hits)
+    if domain:
+        _log(f"resolved '{topic}' -> '{domain}' via search")
+    with _domain_cache_lock:
+        _domain_cache[key] = domain
+    return domain
+
+
+def _is_session_fresh(status: Dict[str, Any]) -> bool:
+    """Read the freshness signal from an ``auth status --agent`` payload."""
+    if not isinstance(status, dict) or "error" in status:
+        return False
+    containers: List[Dict[str, Any]] = [status]
+    session = status.get("session")
+    if isinstance(session, dict):
+        containers.append(session)
+    for container in containers:
+        for key in ("isFresh", "fresh"):
+            if key in container:
+                return bool(container[key])
+    return False
+
+
+def ensure_session_ready(
+    topic: str,
+    config: Optional[Dict[str, Any]] = None,
+    has_domain: bool = False,
+) -> None:
+    """Warm the CLI's WAF session, serialized, at the first Trustpilot fetch.
+
+    Called from ``search_trustpilot`` (never from the pipeline's fan-out
+    setup), so it only ever delays the one capped Trustpilot stream, never
+    the other sources -- and never fires for runs whose plan fetches no
+    Trustpilot at all. The module lock serializes concurrent streams (vs-mode
+    fans out up to 6 entity sub-runs) so they never race their own Chrome
+    harvests. Freshness is a monotonic TTL (WARMUP_TTL_SECONDS, matching the
+    CLI's ~4-minute token bound), not a boolean-forever flag: long-lived host
+    processes re-check via ``auth status`` after the window lapses, which
+    also retries a previously failed login. ``auth login`` fires only when
+    ``auth status`` reports the session missing or stale -- login always
+    harvests (~10s Chrome), it has no freshness no-op. Logs only structured
+    status strings; the raw CLI payload carries live WAF-token prefixes and
+    must never be logged. Never raises.
+    """
+    global _warmup_at
+    if _warmup_fresh():
+        return
+    if not _is_available():
+        return
+    if not _harvest_allowed(config):
+        return
+    if not has_domain and not is_brand_shaped(topic):
+        return
+    with _warmup_lock:
+        if _warmup_fresh():
+            return
+        status = _run_cli(
+            [CLI_BIN, "auth", "status", "--agent"], timeout=AUTH_STATUS_TIMEOUT
+        )
+        if _is_session_fresh(status):
+            _log("warm-up: fresh")
+            _warmup_at = time.monotonic()
+            return
+        # Missing session exits non-zero (an error dict here): that is the
+        # "login needed" signal, not a warm-up failure.
+        login = _run_cli([CLI_BIN, "auth", "login", "--agent"], timeout=SEARCH_TIMEOUT)
+        if isinstance(login, dict) and "error" in login:
+            _log("warm-up failed: auth login did not complete")
+        else:
+            _log("warm-up: harvested")
+        # Stamp even on failure: a broken Chrome will not fix itself within
+        # the TTL, per-call CLI auto-harvest remains the fallback, and the
+        # TTL lapse retries the warm-up later.
+        _warmup_at = time.monotonic()
+
+
 def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
     """Invoke trustpilot-pp-cli and parse the JSON object. Never raises."""
     if not _is_available():
@@ -171,23 +379,66 @@ def search_trustpilot(
     to_date: str,
     depth: str = "default",
     config: Optional[Dict[str, Any]] = None,
+    explicit_domain: Optional[str] = None,
+    domain_is_hint: bool = False,
 ) -> Dict[str, Any]:
     """Look up a company's Trustpilot sentiment, gated on a brand-shaped topic.
 
+    ``explicit_domain`` is used verbatim as the CLI identifier and bypasses
+    the brand-shape gate (an explicit domain is proof of brand intent). A
+    user-set domain is verbatim-final; a resolved hint
+    (``domain_is_hint=True``) retries via the CLI search when the lookup
+    misses, since auto-resolve can guess a plausible-but-wrong domain (the
+    official site is not always Trustpilot's canonical identifyingName).
+
+    Without an explicit domain, a bare company name resolves via the CLI's
+    ``search`` (cached per topic) before falling back to the cleaned topic.
+
     Returns ``{"results": [info_dict]}`` for a resolved company, or
-    ``{"results": []}`` when the topic is not brand-shaped, the browser opt-out
-    is set, or the CLI fails.
+    ``{"results": []}`` when the topic is not brand-shaped, the browser
+    opt-out is set, or the CLI fails.
     """
-    if not is_brand_shaped(topic):
+    explicit_domain = (explicit_domain or "").strip() or None
+    user_domain = bool(explicit_domain) and not domain_is_hint
+    # Only a USER-set domain proves brand intent and bypasses the brand-shape
+    # gate. An auto-resolved hint must not widen activation beyond
+    # brand-shaped topics, or a generic topic that happens to yield a hint
+    # would trigger a Chrome harvest -- violating the module's documented
+    # "never harvests on non-company topics" contract.
+    if not user_domain and not is_brand_shaped(topic):
         return {"results": []}
     if not _is_available():
         return {"results": [], "error": f"{CLI_BIN} not on PATH"}
     if not _harvest_allowed(config):
         _log("skipped: browser opt-out set")
         return {"results": []}
-    identifier = _company_identifier(topic)
+    # Serialized session check at first source touch (all gates above have
+    # passed, so this never fires for topics the source would not fetch).
+    ensure_session_ready(topic, config=config, has_domain=bool(explicit_domain))
+    # Retry-budget timer starts AFTER the warm-up: a slow Chrome harvest must
+    # not consume the hint-retry budget when the info call itself was fast.
+    started = time.monotonic()
+    if explicit_domain:
+        identifier = explicit_domain
+    else:
+        identifier = _company_identifier(topic)
+        if not _DOMAIN_RE.search(topic.lower()):
+            # No domain token in the topic: Trustpilot pages are keyed by
+            # domain, so resolve name -> domain before the info lookup.
+            identifier = _search_domain(topic) or identifier
     _log(f"info '{identifier}'")
     data = _run_cli(_build_info_args(identifier), timeout=SEARCH_TIMEOUT)
+    if ("error" in data or not data) and explicit_domain and domain_is_hint:
+        # The auto-resolved hint missed. Only user-set flags are
+        # verbatim-final; a hint falls through to the search resolution.
+        # Skip the retry chain when the first lookup already consumed a full
+        # single-call budget (hung CLI) -- one stream must not chain three
+        # sequential SEARCH_TIMEOUT-bound subprocesses.
+        if time.monotonic() - started < SEARCH_TIMEOUT:
+            resolved = _search_domain(topic)
+            if resolved and resolved != identifier:
+                _log(f"hint '{identifier}' missed; retrying via search as '{resolved}'")
+                data = _run_cli(_build_info_args(resolved), timeout=SEARCH_TIMEOUT)
     if "error" in data or not data:
         return {"results": []}
     return {"results": [data]}

package/skills/trends/scripts/lib/truthsocial.py

@@ -89,7 +89,15 @@ def search_truthsocial(
     try:
         response = http.request(
             "GET", url,
-            headers={"Authorization": f"Bearer {token}"},
+            headers={
+                "Authorization": f"Bearer {token}",
+                # Cloudflare 403s the skill's default User-Agent regardless of token validity (#909).
+                # Reuse http.BROWSER_USER_AGENT, as the keyless Reddit path does.
+                "User-Agent": http.BROWSER_USER_AGENT,
+                "Accept": "application/json, text/plain, */*",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Referer": "https://truthsocial.com/",
+            },
             timeout=30,
         )
     except http.HTTPError as e:

package/skills/trends/scripts/lib/ui.py

@@ -121,6 +121,7 @@ SOURCE_COMPLETION_ORDER = [
     "tiktok",
     "instagram",
     "hackernews",
+    "devto",
     "bluesky",
     "truthsocial",
     "polymarket",
@@ -130,6 +131,8 @@ SOURCE_COMPLETION_ORDER = [
     "arxiv",
     "techmeme",
     "trustpilot",
+    "amazon",
+    "meta_ads",
 ]
 
 SOURCE_COMPLETION_META = {
@@ -139,6 +142,7 @@ SOURCE_COMPLETION_META = {
     "tiktok": ("TikTok", "video", "videos", Colors.PURPLE),
     "instagram": ("Instagram", "reel", "reels", Colors.PURPLE),
     "hackernews": ("HN", "story", "stories", Colors.YELLOW),
+    "devto": ("dev.to", "article", "articles", Colors.GREEN),
     "bluesky": ("Bluesky", "post", "posts", Colors.BLUE),
     "truthsocial": ("Truth Social", "post", "posts", Colors.CYAN),
     "polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
@@ -148,6 +152,8 @@ SOURCE_COMPLETION_META = {
     "arxiv": ("arXiv", "paper", "papers", Colors.RED),
     "techmeme": ("Techmeme", "headline", "headlines", Colors.CYAN),
     "trustpilot": ("Trustpilot", "review", "reviews", Colors.GREEN),
+    "amazon": ("Amazon", "product", "products", Colors.YELLOW),
+    "meta_ads": ("Meta Ads", "creative", "creatives", Colors.BLUE),
 }
 
 

package/skills/trends/scripts/lib/usage.py

@@ -0,0 +1,132 @@
+"""Provider-reported charges for a scoped watchlist subprocess."""
+
+from __future__ import annotations
+
+import math
+import os
+import sqlite3
+import uuid
+from contextlib import contextmanager
+from pathlib import Path
+from urllib.parse import urlsplit
+
+JOURNAL_ENV = "LAST30DAYS_USAGE_JOURNAL"
+
+_PAID_HOSTS = {
+    "openrouter.ai": "openrouter",
+    "api.perplexity.ai": "perplexity",
+    "api.openai.com": "openai",
+    "api.x.ai": "xai",
+    "generativelanguage.googleapis.com": "gemini",
+    "api.scrapecreators.com": "scrapecreators",
+    "api.search.brave.com": "brave",
+    "api.tavily.com": "tavily",
+    "api.exa.ai": "exa",
+    "google.serper.dev": "serper",
+    "api.parallel.ai": "parallel",
+    "api.x.com": "x",
+    "api.twitter.com": "x",
+    "xquik.com": "xquik",
+    "api.groq.com": "groq",
+}
+
+
+@contextmanager
+def _connection(path):
+    conn = sqlite3.connect(path)
+    try:
+        with conn:
+            yield conn
+    finally:
+        conn.close()
+
+
+def create_journal(path: Path) -> None:
+    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+    os.close(fd)
+    with _connection(path) as conn:
+        conn.execute(
+            "CREATE TABLE attempts (id TEXT PRIMARY KEY, provider TEXT NOT NULL, "
+            "cost REAL, unknown INTEGER NOT NULL DEFAULT 1, "
+            "prompt_tokens INTEGER DEFAULT 0, completion_tokens INTEGER DEFAULT 0)"
+        )
+
+
+def begin(provider: str) -> tuple[str, str, str] | None:
+    path = os.environ.get(JOURNAL_ENV)
+    if not path:
+        return None
+    attempt_id = uuid.uuid4().hex
+    with _connection(path) as conn:
+        conn.execute("INSERT INTO attempts (id, provider) VALUES (?, ?)", (attempt_id, provider))
+    return path, attempt_id, provider
+
+
+def begin_http(url: str, method: str) -> tuple[str, str, str] | None:
+    if not os.environ.get(JOURNAL_ENV):
+        return None
+    parts = urlsplit(url)
+    provider = _PAID_HOSTS.get(parts.hostname)
+    if provider == "perplexity" and method == "GET":
+        return None  # Retrieving an existing background operation does not start another charge.
+    if provider is None:
+        for name in ("OPENAI", "XAI", "OPENROUTER"):
+            override = os.environ.get(f"{name}_BASE_URL")
+            if override and urlsplit(override).netloc == parts.netloc:
+                provider = name.lower()
+                break
+    return begin(provider) if provider else None
+
+
+def cancel(attempt: tuple[str, str, str] | None) -> None:
+    """Remove an attempt when the transport provably never started."""
+    if attempt is not None:
+        path, attempt_id, _ = attempt
+        with _connection(path) as conn:
+            conn.execute("DELETE FROM attempts WHERE id = ?", (attempt_id,))
+
+
+def finish(attempt: tuple[str, str, str] | None, payload: object) -> None:
+    if attempt is None or not isinstance(payload, dict):
+        return
+    path, attempt_id, provider = attempt
+    usage = payload.get("usage")
+    if not isinstance(usage, dict):
+        return
+    cost = usage.get("cost")
+    if provider == "perplexity":
+        if not isinstance(cost, dict) or cost.get("currency") != "USD":
+            return
+        cost = cost.get("total_cost")
+    elif provider != "openrouter":
+        return
+    if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0:
+        return
+    # A background response may expose a running subtotal rather than its final bill.
+    unknown = int(payload.get("status") in {"queued", "in_progress"})
+    prompt = usage.get("prompt_tokens", usage.get("input_tokens", 0))
+    completion = usage.get("completion_tokens", usage.get("output_tokens", 0))
+    prompt = prompt if type(prompt) is int and prompt >= 0 else 0
+    completion = completion if type(completion) is int and completion >= 0 else 0
+    with _connection(path) as conn:
+        conn.execute(
+            "UPDATE attempts SET cost = ?, unknown = ?, prompt_tokens = ?, completion_tokens = ? WHERE id = ?",
+            (cost, unknown, prompt, completion, attempt_id),
+        )
+
+
+def read_journal(path: Path) -> dict:
+    try:
+        with _connection(path) as conn:
+            row = conn.execute(
+                "SELECT COALESCE(SUM(cost), 0), COALESCE(SUM(unknown), 0), "
+                "COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0) FROM attempts"
+            ).fetchone()
+    except sqlite3.Error:
+        return {"token_cost": 0.0, "cost_unknown": 1, "prompt_tokens": 0, "completion_tokens": 0}
+    return {
+        "token_cost": row[0],
+        "cost_unknown": int(row[1] > 0),
+        "prompt_tokens": row[2],
+        "completion_tokens": row[3],
+    }

package/skills/trends/scripts/lib/web_search_keyless.py

@@ -9,10 +9,12 @@ It must never run on a host that has native search (the model does it better
 there) or preempt a configured paid backend. The pipeline/grounding layer owns
 that gating; this module just performs the search when asked.
 
-Two vendor-neutral rungs, both stdlib-only via :mod:`http`:
+Three vendor-neutral rungs, all stdlib-only via :mod:`http`:
   1. DuckDuckGo HTML endpoint (no key, no instance to maintain).
-  2. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),
-     tried when the primary yields nothing.
+  2. Startpage HTML results, tried when DuckDuckGo yields nothing — notably
+     when DuckDuckGo anomaly-blocks a datacenter IP with a 202 challenge page.
+  3. A configurable SearXNG instance returning JSON (``LAST30DAYS_SEARXNG_URL``),
+     tried when the HTML rungs yield nothing.
 
 Never raises. Returns results in the same dict shape as the paid backends in
 :mod:`grounding` so they flow through normalize/score/dedupe unchanged. On total
@@ -37,6 +39,10 @@ _DDG_HTML_URL = "https://html.duckduckgo.com/html/"
 _KEYLESS_RELEVANCE = 0.6
 
 _TAG_RE = re.compile(r"<[^>]+>")
+# Strip <style>/<script> blocks *including their contents* before dropping tags,
+# so inline CSS/JS text (e.g. Startpage's emotion styles) never leaks into a
+# title or snippet.
+_STYLE_SCRIPT_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
 _RESULT_A_RE = re.compile(
     r'class="result__a"[^>]*href="(?P<href>[^"]+)"[^>]*>(?P<title>.*?)</a>',
     re.IGNORECASE | re.DOTALL,
@@ -46,6 +52,21 @@ _SNIPPET_RE = re.compile(
     re.IGNORECASE | re.DOTALL,
 )
 
+_STARTPAGE_HTML_URL = "https://www.startpage.com/sp/search"
+# Startpage marks each organic hit with an <a class="result-title result-link …"
+# href="<target>">…<h2 …>title</h2></a>, and the description in a following
+# <p class="…description…">. Class names carry hashed emotion suffixes, so match
+# on the stable "result-title" / "description" substrings.
+_SP_RESULT_RE = re.compile(
+    r'<a\b[^>]*class="[^"]*result-title[^"]*"[^>]*href="(?P<href>https?://[^"]+)"[^>]*>(?P<inner>.*?)</a>',
+    re.IGNORECASE | re.DOTALL,
+)
+_SP_H2_RE = re.compile(r"<h2\b[^>]*>(?P<title>.*?)</h2>", re.IGNORECASE | re.DOTALL)
+_SP_DESC_RE = re.compile(
+    r'<p\b[^>]*class="[^"]*description[^"]*"[^>]*>(?P<snippet>.*?)</p>',
+    re.IGNORECASE | re.DOTALL,
+)
+
 
 def _domain(url: str) -> str:
     # Normalize identically to grounding._domain (strip + lowercase) so keyless
@@ -57,7 +78,8 @@ def _domain(url: str) -> str:
 
 
 def _strip_html(fragment: str) -> str:
-    return html.unescape(_TAG_RE.sub("", fragment or "")).strip()
+    without_blocks = _STYLE_SCRIPT_RE.sub("", fragment or "")
+    return html.unescape(_TAG_RE.sub("", without_blocks)).strip()
 
 
 def _unwrap_ddg_redirect(href: str) -> str:
@@ -81,6 +103,11 @@ def keyless_search(
     """Run keyless web search; returns (items, artifact). Never raises."""
     items = _search_ddg(query, count)
     used = "ddg"
+    if not items:
+        # DuckDuckGo anomaly-blocks datacenter IPs (202 challenge page); fall
+        # back to Startpage, which still serves organic results there.
+        items = _search_startpage(query, count)
+        used = "startpage"
     if not items:
         searxng_url = (config.get("LAST30DAYS_SEARXNG_URL") or "").strip()
         if searxng_url:
@@ -123,6 +150,38 @@ def _search_ddg(query: str, count: int) -> list[dict]:
     return items
 
 
+def _search_startpage(query: str, count: int) -> list[dict]:
+    """Keyless rung 2: Startpage's HTML results page. Unlike DuckDuckGo's HTML
+    endpoint (which anomaly-blocks datacenter IPs with a 202 challenge page),
+    Startpage returns organic results to a plain browser-UA GET, making it the
+    working floor on hosts DuckDuckGo refuses. Never raises."""
+    url = f"{_STARTPAGE_HTML_URL}?{urlencode({'query': query})}"
+    text = http.get_text(url, accept="text/html", retries=2)
+    if not text:
+        return []
+    items: list[dict] = []
+    result_matches = list(_SP_RESULT_RE.finditer(text))
+    desc_matches = list(_SP_DESC_RE.finditer(text))
+    for match in result_matches:
+        if len(items) >= count:
+            break
+        target = html.unescape(match.group("href"))
+        if not target.startswith("http"):
+            continue
+        h2 = _SP_H2_RE.search(match.group("inner"))
+        title = _strip_html(h2.group("title") if h2 else match.group("inner"))
+        if not title:
+            continue
+        # First description block that appears after this result's title anchor.
+        snippet = ""
+        for desc in desc_matches:
+            if desc.start() > match.end():
+                snippet = _strip_html(desc.group("snippet"))
+                break
+        items.append(_to_item(len(items), title, target, snippet))
+    return items
+
+
 def _search_searxng(query: str, count: int, instance_url: str) -> list[dict]:
     base = instance_url.rstrip("/")
     url = f"{base}/search?{urlencode({'q': query, 'format': 'json'})}"