@tiens.nguyen/gonext-local-worker 1.0.274 → 1.0.275

package/gonext_agent_chat.py

@@ -364,73 +364,206 @@ def _get_json(url, timeout=15):
         return None
 
 
-def _web_search_impl(query, max_results=5):
-    """Look up factual info via free no-key JSON APIs (DuckDuckGo + Wikipedia).
-
-    Returns a short SUMMARY (for a direct-fact question) followed by a numbered list
-    of the top matching PAGES (title — snippet — URL), so the model can fetch_url() a
-    real page when a one-line summary can't carry the data it needs (a schedule, a
-    table, a fixtures list). Returning only a single Wikipedia intro used to trap weak
-    models in a re-search loop — they never saw a candidate URL to open (task #75).
-    Never fabricates — returns an honest 'no results' when nothing is found.
-    """
+# --- web_search: parallel keyless multi-backend (task #103) ------------------------------
+# The old impl called DuckDuckGo's INSTANT-ANSWER API + Wikipedia sequentially. The IA API
+# isn't a real web SERP (only curated abstracts for known entities), so general queries came
+# back thin/empty and slow (2-3 sequential 15-25s fetches). This runs several KEYLESS
+# backends CONCURRENTLY with short per-backend timeouts and an overall deadline, then merges
+# + dedupes + ranks by cross-engine agreement. Backends: DuckDuckGo HTML (real results),
+# a self-hosted SearXNG metasearch (aggregates Google/Bing/Yahoo/… server-side, keyless — set
+# GONEXT_SEARXNG_URL or cfg.searxngUrl), and Wikipedia (encyclopedic). No API keys.
+_WEB_SEARCH_TTL = 300  # seconds — short cache so repeated queries in a turn don't re-hit net
+_WEB_SEARCH_CACHE: dict = {}  # (query_lower, searxng_url) -> (ts, result_text)
+_WEB_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+           "(KHTML, like Gecko) Chrome/121.0 Safari/537.36")
+
+
+def _get_text(url, timeout=6, ua=None, headers=None):
+    """GET a URL and return the decoded body text ("" on failure). For scraping SERP HTML."""
+    h = {"User-Agent": ua or _WEB_UA,
+         "Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
+         "Accept-Language": "en-US,en;q=0.9"}
+    if headers:
+        h.update(headers)
+    req = urllib.request.Request(url, method="GET", headers=h)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
+            return resp.read(600_000).decode("utf-8", errors="replace")
+    except Exception as e:  # noqa: BLE001
+        _log(f"web_search GET failed {url[:80]}: {e}")
+        return ""
+
+
+def _search_strip(s):
     import html as _html
-    from urllib.parse import quote
-    q = (query or "").strip()
-    if not q:
-        return "web_search: empty query."
+    return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
 
-    def _strip(s):
-        return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
 
-    summary = ""
-    summary_src = ""
-    results = []  # (title, snippet, url)
-    seen = set()
+def _norm_url(u):
+    """Normalize a URL for dedupe (lower host/scheme, drop fragment + trailing slash)."""
+    from urllib.parse import urlsplit, urlunsplit
+    u = (u or "").strip()
+    if not u:
+        return ""
+    try:
+        s = urlsplit(u)
+        return urlunsplit((s.scheme.lower(), s.netloc.lower(), s.path.rstrip("/"), "", ""))
+    except Exception:  # noqa: BLE001
+        return u.rstrip("/")
 
-    def _add(title, snippet, url):
-        url = (url or "").strip()
-        if not url or url in seen or len(results) >= max_results:
-            return
-        seen.add(url)
-        results.append((_strip(title) or url, _strip(snippet), url))
 
-    # 1) DuckDuckGo Instant Answer — a direct abstract + related topics (as pages).
-    ddg = _get_json(
-        f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1"
-    )
-    if isinstance(ddg, dict):
-        abstract = (ddg.get("AbstractText") or "").strip()
-        if abstract:
-            summary = abstract[:1200]
-            summary_src = (ddg.get("AbstractURL") or "").strip()
-        for topic in ddg.get("RelatedTopics") or []:
-            if isinstance(topic, dict) and topic.get("Text") and topic.get("FirstURL"):
-                _add(topic["Text"][:80], topic["Text"], topic["FirstURL"])
-
-    # 2) Wikipedia full-text search — SEVERAL candidate pages with snippets, so the
-    #    model can pick the specific one (e.g. a "…knockout stage" fixtures page).
+def _bk_ddg_html(q, timeout=6):
+    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless)."""
+    from urllib.parse import quote, unquote
+    html_body = _get_text(f"https://html.duckduckgo.com/html/?q={quote(q)}", timeout=timeout)
+    if not html_body:
+        return []
+    out = []
+    anchors = re.findall(r'<a\b[^>]*class="result__a"[^>]*>.*?</a>', html_body, re.S)
+    snippets = re.findall(r'<a\b[^>]*class="result__snippet"[^>]*>(.*?)</a>', html_body, re.S)
+    for i, a in enumerate(anchors):
+        hm = re.search(r'href="([^"]*)"', a)
+        if not hm:
+            continue
+        href = hm.group(1)
+        um = re.search(r'[?&]uddg=([^&]+)', href)
+        real = unquote(um.group(1)) if um else href
+        if real.startswith("//"):
+            real = "https:" + real
+        if not real.startswith("http"):
+            continue
+        title = _search_strip(a)
+        snip = _search_strip(snippets[i]) if i < len(snippets) else ""
+        if title:
+            out.append({"title": title[:120], "snippet": snip, "url": real})
+    return out
+
+
+def _bk_searxng(q, base, timeout=6):
+    """Self-hosted SearXNG JSON API → aggregated results [{title,snippet,url}] (keyless)."""
+    from urllib.parse import quote
+    base = (base or "").strip().rstrip("/")
+    if not base:
+        return []
+    data = _get_json(f"{base}/search?q={quote(q)}&format=json&safesearch=0", timeout=timeout)
+    out = []
+    if isinstance(data, dict):
+        for r in (data.get("results") or []):
+            url = (r.get("url") or "").strip()
+            if not url:
+                continue
+            out.append({
+                "title": _search_strip(r.get("title") or url)[:120],
+                "snippet": _search_strip(r.get("content") or ""),
+                "url": url,
+            })
+    return out
+
+
+def _bk_wikipedia(q, timeout=6, n=5):
+    """Wikipedia full-text search → encyclopedic candidate pages [{title,snippet,url}]."""
+    from urllib.parse import quote
     search = _get_json(
         "https://en.wikipedia.org/w/api.php?action=query&list=search"
-        f"&srsearch={quote(q)}&srlimit={max_results}&format=json"
+        f"&srsearch={quote(q)}&srlimit={n}&format=json",
+        timeout=timeout,
     )
+    out = []
     try:
         hits = search["query"]["search"]
     except Exception:  # noqa: BLE001
         hits = []
     for hit in hits:
         slug = quote(hit.get("title", "").replace(" ", "_"))
-        _add(hit.get("title", ""), hit.get("snippet", ""),
-             f"https://en.wikipedia.org/wiki/{slug}")
-    # If no direct abstract, use the best-matching page's summary as the answer.
-    if not summary and hits:
-        slug = quote(hits[0]["title"].replace(" ", "_"))
-        s = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
-        if isinstance(s, dict):
-            summary = (s.get("extract") or "").strip()[:1200]
-            summary_src = f"https://en.wikipedia.org/wiki/{slug}"
-
-    if not summary and not results:
+        out.append({
+            "title": hit.get("title", ""),
+            "snippet": _search_strip(hit.get("snippet", "")),
+            "url": f"https://en.wikipedia.org/wiki/{slug}",
+        })
+    return out
+
+
+def _bk_ddg_ia_summary(q, timeout=6):
+    """DuckDuckGo Instant Answer — a direct abstract for a well-known entity (summary only)."""
+    from urllib.parse import quote
+    ddg = _get_json(
+        f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1",
+        timeout=timeout,
+    )
+    if isinstance(ddg, dict):
+        abstract = (ddg.get("AbstractText") or "").strip()
+        if abstract:
+            return (abstract[:1200], (ddg.get("AbstractURL") or "").strip())
+    return ("", "")
+
+
+def _web_search_impl(query, max_results=5, searxng_url=""):
+    """Parallel keyless web search (task #103): run DuckDuckGo-HTML + SearXNG (if configured)
+    + Wikipedia CONCURRENTLY with a short deadline, merge/dedupe/rank by cross-engine
+    agreement, and return a short SUMMARY (when available) + a numbered list of the top
+    PAGES (title — snippet — URL) so the model can fetch_url() the best one. Never fabricates.
+    """
+    import os
+    import time as _t
+    from concurrent.futures import ThreadPoolExecutor
+
+    q = (query or "").strip()
+    if not q:
+        return "web_search: empty query."
+    searxng = (searxng_url or os.environ.get("GONEXT_SEARXNG_URL") or "").strip()
+
+    # Short TTL cache (identical query within a turn).
+    ckey = (q.lower(), searxng)
+    hit = _WEB_SEARCH_CACHE.get(ckey)
+    if hit and (_t.time() - hit[0]) < _WEB_SEARCH_TTL:
+        return hit[1]
+
+    per_timeout = 6   # per-backend socket timeout
+    overall = 9       # overall wall-clock deadline across all backends
+
+    # Result-producing backends (run in parallel). Order = tie-break priority in the merge.
+    tasks = [("ddg", lambda: _bk_ddg_html(q, per_timeout))]
+    if searxng:
+        tasks.append(("searxng", lambda: _bk_searxng(q, searxng, per_timeout)))
+    tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
+    tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))
+
+    results_by = {}
+    summary, summary_src = "", ""
+    with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
+        futs = {ex.submit(fn): name for name, fn in tasks}
+        end = _t.time() + overall
+        for fut, name in futs.items():
+            try:
+                r = fut.result(timeout=max(0.0, end - _t.time()))
+            except Exception:  # noqa: BLE001 — a slow/failed backend is skipped
+                continue
+            if name == "summary":
+                summary, summary_src = r
+            else:
+                results_by[name] = r
+
+    # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
+    merged = {}
+    order = 0
+    for name in ("searxng", "ddg", "wikipedia"):
+        for r in results_by.get(name, []):
+            nu = _norm_url(r.get("url"))
+            if not nu:
+                continue
+            if nu in merged:
+                merged[nu]["hits"] += 1
+                if not merged[nu]["snippet"] and r.get("snippet"):
+                    merged[nu]["snippet"] = r["snippet"]
+            else:
+                merged[nu] = {
+                    "title": r.get("title") or r["url"], "snippet": r.get("snippet", ""),
+                    "url": r["url"], "hits": 1, "order": order,
+                }
+                order += 1
+    ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]
+
+    if not summary and not ranked:
         return (
             f"No results found for '{q}'. Tell the user you couldn't find this — "
             "do NOT invent an answer or a URL."
@@ -439,15 +572,17 @@ def _web_search_impl(query, max_results=5):
     parts = []
     if summary:
         parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
-    if results:
+    if ranked:
         lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
-        for i, (title, snippet, url) in enumerate(results, 1):
-            line = f"{i}. {title}"
-            if snippet:
-                line += f" — {snippet[:160]}"
-            lines.append(line + f"\n   {url}")
+        for i, e in enumerate(ranked, 1):
+            line = f"{i}. {e['title']}"
+            if e["snippet"]:
+                line += f" — {e['snippet'][:160]}"
+            lines.append(line + f"\n   {e['url']}")
         parts.append("\n".join(lines))
-    return "\n\n".join(parts)
+    out = "\n\n".join(parts)
+    _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
+    return out
 
 
 class _AgentConfigError(RuntimeError):
@@ -3893,7 +4028,9 @@ def run_agent_chat(cfg):
         if blocked:
             return blocked
         _emit({"type": "step", "text": f"Searching the web → {query[:80]}"})
-        result = _web_search_impl(query)
+        # SearXNG URL (task #103): from Settings (cfg.searxngUrl) if the API sends it, else
+        # the GONEXT_SEARXNG_URL env fallback inside _web_search_impl. Keyless either way.
+        result = _web_search_impl(query, searxng_url=(cfg.get("searxngUrl") or "").strip())
         _log(f"web_search {query[:60]!r} → {result[:80]}")
         _last_obs["text"] = result
         if result and not result.startswith("Error"):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.274",
+  "version": "1.0.275",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",