@tiens.nguyen/gonext-local-worker 1.0.274 → 1.0.276

package/gonext-local-worker.mjs

@@ -1893,6 +1893,9 @@ async function runAgentChatJob(job) {
       agentModelId: payload?.agentModelId ?? "",
       codingBaseURL: payload?.codingBaseURL ?? "",
       codingModelId: payload?.codingModelId ?? "",
+      // Optional search model (task #105): synthesizes web_search results. Empty = no synthesis.
+      searchBaseURL: payload?.searchBaseURL ?? "",
+      searchModelId: payload?.searchModelId ?? "",
       tools: payload?.tools ?? ["http_request"],
       // 0 = "not explicitly set": python defaults to 5, but may auto-raise to 12 when
       // workspaces are registered. Passing a literal 5 here would look user-set and

package/gonext_agent_chat.py

@@ -364,90 +364,354 @@ def _get_json(url, timeout=15):
         return None
 
 
-def _web_search_impl(query, max_results=5):
-    """Look up factual info via free no-key JSON APIs (DuckDuckGo + Wikipedia).
-
-    Returns a short SUMMARY (for a direct-fact question) followed by a numbered list
-    of the top matching PAGES (title — snippet — URL), so the model can fetch_url() a
-    real page when a one-line summary can't carry the data it needs (a schedule, a
-    table, a fixtures list). Returning only a single Wikipedia intro used to trap weak
-    models in a re-search loop — they never saw a candidate URL to open (task #75).
-    Never fabricates — returns an honest 'no results' when nothing is found.
-    """
+# --- web_search: parallel keyless multi-backend (task #103) ------------------------------
+# The old impl called DuckDuckGo's INSTANT-ANSWER API + Wikipedia sequentially. The IA API
+# isn't a real web SERP (only curated abstracts for known entities), so general queries came
+# back thin/empty and slow (2-3 sequential 15-25s fetches). This runs several KEYLESS
+# backends CONCURRENTLY with short per-backend timeouts and an overall deadline, then merges
+# + dedupes + ranks by cross-engine agreement. Backends: DuckDuckGo HTML (real results),
+# a self-hosted SearXNG metasearch (aggregates Google/Bing/Yahoo/… server-side, keyless — set
+# GONEXT_SEARXNG_URL or cfg.searxngUrl), and Wikipedia (encyclopedic). No API keys.
+_WEB_SEARCH_TTL = 300  # seconds — short cache so repeated queries in a turn don't re-hit net
+_WEB_SEARCH_CACHE: dict = {}  # (query_lower, searxng_url) -> (ts, result_text)
+_WEB_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+           "(KHTML, like Gecko) Chrome/121.0 Safari/537.36")
+
+
+def _get_text(url, timeout=6, ua=None, headers=None):
+    """GET a URL and return the decoded body text ("" on failure). For scraping SERP HTML."""
+    h = {"User-Agent": ua or _WEB_UA,
+         "Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
+         "Accept-Language": "en-US,en;q=0.9"}
+    if headers:
+        h.update(headers)
+    req = urllib.request.Request(url, method="GET", headers=h)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
+            return resp.read(600_000).decode("utf-8", errors="replace")
+    except Exception as e:  # noqa: BLE001
+        _log(f"web_search GET failed {url[:80]}: {e}")
+        return ""
+
+
+def _search_strip(s):
     import html as _html
-    from urllib.parse import quote
-    q = (query or "").strip()
-    if not q:
-        return "web_search: empty query."
+    return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
 
-    def _strip(s):
-        return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
 
-    summary = ""
-    summary_src = ""
-    results = []  # (title, snippet, url)
-    seen = set()
+def _norm_url(u):
+    """Normalize a URL for dedupe (lower host/scheme, drop fragment + trailing slash)."""
+    from urllib.parse import urlsplit, urlunsplit
+    u = (u or "").strip()
+    if not u:
+        return ""
+    try:
+        s = urlsplit(u)
+        return urlunsplit((s.scheme.lower(), s.netloc.lower(), s.path.rstrip("/"), "", ""))
+    except Exception:  # noqa: BLE001
+        return u.rstrip("/")
 
-    def _add(title, snippet, url):
-        url = (url or "").strip()
-        if not url or url in seen or len(results) >= max_results:
-            return
-        seen.add(url)
-        results.append((_strip(title) or url, _strip(snippet), url))
 
-    # 1) DuckDuckGo Instant Answer — a direct abstract + related topics (as pages).
-    ddg = _get_json(
-        f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1"
-    )
-    if isinstance(ddg, dict):
-        abstract = (ddg.get("AbstractText") or "").strip()
-        if abstract:
-            summary = abstract[:1200]
-            summary_src = (ddg.get("AbstractURL") or "").strip()
-        for topic in ddg.get("RelatedTopics") or []:
-            if isinstance(topic, dict) and topic.get("Text") and topic.get("FirstURL"):
-                _add(topic["Text"][:80], topic["Text"], topic["FirstURL"])
-
-    # 2) Wikipedia full-text search — SEVERAL candidate pages with snippets, so the
-    #    model can pick the specific one (e.g. a "…knockout stage" fixtures page).
+def _bk_ddg_html(q, timeout=6):
+    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless)."""
+    from urllib.parse import quote, unquote
+    html_body = _get_text(f"https://html.duckduckgo.com/html/?q={quote(q)}", timeout=timeout)
+    if not html_body:
+        return []
+    out = []
+    anchors = re.findall(r'<a\b[^>]*class="result__a"[^>]*>.*?</a>', html_body, re.S)
+    snippets = re.findall(r'<a\b[^>]*class="result__snippet"[^>]*>(.*?)</a>', html_body, re.S)
+    for i, a in enumerate(anchors):
+        hm = re.search(r'href="([^"]*)"', a)
+        if not hm:
+            continue
+        href = hm.group(1)
+        um = re.search(r'[?&]uddg=([^&]+)', href)
+        real = unquote(um.group(1)) if um else href
+        if real.startswith("//"):
+            real = "https:" + real
+        if not real.startswith("http"):
+            continue
+        title = _search_strip(a)
+        snip = _search_strip(snippets[i]) if i < len(snippets) else ""
+        if title:
+            out.append({"title": title[:120], "snippet": snip, "url": real})
+    return out
+
+
+def _bk_searxng(q, base, timeout=6):
+    """Self-hosted SearXNG JSON API (keyless) → {"results":[{title,snippet,url}], "answers":[str]}.
+    `answers` = SearXNG's aggregated zero-click ANSWERS + INFOBOXES (the 'answer box' / knowledge
+    panel a human sees on a SERP) — surfaced first so one search often needs no page read (#104)."""
+    from urllib.parse import quote
+    base = (base or "").strip().rstrip("/")
+    if not base:
+        return {"results": [], "answers": []}
+    data = _get_json(f"{base}/search?q={quote(q)}&format=json&safesearch=0", timeout=timeout)
+    results, answers = [], []
+    if isinstance(data, dict):
+        for r in (data.get("results") or []):
+            url = (r.get("url") or "").strip()
+            if not url:
+                continue
+            results.append({
+                "title": _search_strip(r.get("title") or url)[:120],
+                "snippet": _search_strip(r.get("content") or ""),
+                "url": url,
+            })
+        for a in (data.get("answers") or []):
+            txt = _search_strip(a if isinstance(a, str) else (a.get("answer") or a.get("content") or ""))
+            if txt:
+                answers.append(txt[:600])
+        for ib in (data.get("infoboxes") or []):
+            if isinstance(ib, dict):
+                txt = _search_strip(ib.get("content") or "")
+                if txt:
+                    answers.append(txt[:600])
+    return {"results": results, "answers": answers}
+
+
+def _bk_wikipedia(q, timeout=6, n=5):
+    """Wikipedia full-text search → encyclopedic candidate pages [{title,snippet,url}]."""
+    from urllib.parse import quote
     search = _get_json(
         "https://en.wikipedia.org/w/api.php?action=query&list=search"
-        f"&srsearch={quote(q)}&srlimit={max_results}&format=json"
+        f"&srsearch={quote(q)}&srlimit={n}&format=json",
+        timeout=timeout,
     )
+    out = []
     try:
         hits = search["query"]["search"]
     except Exception:  # noqa: BLE001
         hits = []
     for hit in hits:
         slug = quote(hit.get("title", "").replace(" ", "_"))
-        _add(hit.get("title", ""), hit.get("snippet", ""),
-             f"https://en.wikipedia.org/wiki/{slug}")
-    # If no direct abstract, use the best-matching page's summary as the answer.
-    if not summary and hits:
-        slug = quote(hits[0]["title"].replace(" ", "_"))
-        s = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
-        if isinstance(s, dict):
-            summary = (s.get("extract") or "").strip()[:1200]
-            summary_src = f"https://en.wikipedia.org/wiki/{slug}"
-
-    if not summary and not results:
+        out.append({
+            "title": hit.get("title", ""),
+            "snippet": _search_strip(hit.get("snippet", "")),
+            "url": f"https://en.wikipedia.org/wiki/{slug}",
+        })
+    return out
+
+
+def _bk_ddg_ia_summary(q, timeout=6):
+    """DuckDuckGo Instant Answer — a direct abstract for a well-known entity (summary only)."""
+    from urllib.parse import quote
+    ddg = _get_json(
+        f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1",
+        timeout=timeout,
+    )
+    if isinstance(ddg, dict):
+        abstract = (ddg.get("AbstractText") or "").strip()
+        if abstract:
+            return (abstract[:1200], (ddg.get("AbstractURL") or "").strip())
+    return ("", "")
+
+
+def _condense_for_query(text, query, max_chars=1600):
+    """Keep the QUERY-RELEVANT paragraphs of a page (task #104): split into paragraphs,
+    score each by distinct query-term overlap, keep the highest-scoring ones up to max_chars,
+    then restore original order. Boilerplate/nav is already gone (via _html_to_text); this
+    picks the part that actually answers the query so the read stays small + on-signal."""
+    terms = {t for t in re.findall(r"[a-z0-9]{3,}", (query or "").lower())}
+    paras = [p.strip() for p in re.split(r"\n\s*\n|\n", text or "") if len(p.strip()) > 20]
+    if not paras:
+        return (text or "")[:max_chars].strip()
+    if not terms:
+        return "\n".join(paras)[:max_chars].strip()
+    scored = [(sum(1 for t in terms if t in p.lower()), i, p) for i, p in enumerate(paras)]
+    chosen, total = [], 0
+    for score, i, p in sorted(scored, key=lambda x: (-x[0], x[1])):
+        if score == 0 and chosen:  # stop once we're into zero-signal paragraphs
+            break
+        if chosen and total + len(p) > max_chars:
+            break
+        chosen.append((i, p))
+        total += len(p) + 1
+    chosen.sort()
+    return "\n".join(p for _, p in chosen)[:max_chars].strip()
+
+
+def _read_pages(urls, query, per_page_chars=1600, per_timeout=5):
+    """Fetch several result pages IN PARALLEL and return {url: condensed_readable_text}.
+    The 'read' half of search-and-read (task #104) — reuses _fetch_page_impl + _html_to_text.
+    Skips non-HTML/failed pages (the snippet stays as the fallback)."""
+    from concurrent.futures import ThreadPoolExecutor
+    import time as _t
+
+    def _read_one(u):
+        status, ctype, raw = _fetch_page_impl(u, timeout=per_timeout, max_bytes=300000)
+        if not status or status >= 400:
+            return u, ""
+        if ctype and ("html" not in ctype and "text" not in ctype):
+            return u, ""  # skip pdf/json/binary — the model can fetch_url those explicitly
+        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
+        return u, _condense_for_query(text, query, per_page_chars)
+
+    out = {}
+    if not urls:
+        return out
+    with ThreadPoolExecutor(max_workers=len(urls)) as ex:
+        futs = {ex.submit(_read_one, u): u for u in urls}
+        end = _t.time() + per_timeout + 3
+        for fut in list(futs):
+            try:
+                uu, txt = fut.result(timeout=max(0.0, end - _t.time()))
+                if txt:
+                    out[uu] = txt
+            except Exception:  # noqa: BLE001
+                continue
+    return out
+
+
+def _synthesize_search(query, context, base_url, model_id, api_key="local"):
+    """Search model (task #105): ask a configured OpenAI-compatible model to write a concise,
+    cited answer from the already-fetched search results — so the (slow) code model doesn't
+    have to read/reason over raw pages. Returns "" on any failure (caller falls back)."""
+    from openai import OpenAI
+    client = OpenAI(base_url=base_url, api_key=api_key or "local", max_retries=0, timeout=45)
+    resp = _chat_create(
+        client,
+        model=model_id or "default_model",
+        messages=[
+            {"role": "system", "content": (
+                "You are a research assistant. Using ONLY the search results provided, write a "
+                "concise, factual answer to the user's query. Cite sources inline as [n] using "
+                "their numbers. If the results do not contain the answer, say so plainly — never "
+                "invent facts, dates, names, or URLs."
+            )},
+            {"role": "user", "content": f"Query: {query}\n\nSearch results:\n{context}\n\nAnswer:"},
+        ],
+        max_tokens=700,
+        temperature=0.2,
+    )
+    return (resp.choices[0].message.content or "").strip()
+
+
+def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", search_model_id=""):
+    """Parallel keyless web search (task #103): run DuckDuckGo-HTML + SearXNG (if configured)
+    + Wikipedia CONCURRENTLY with a short deadline, merge/dedupe/rank by cross-engine
+    agreement, and return a short SUMMARY (when available) + a numbered list of the top
+    PAGES (title — snippet — URL) so the model can fetch_url() the best one. Never fabricates.
+    """
+    import os
+    import time as _t
+    from concurrent.futures import ThreadPoolExecutor
+
+    q = (query or "").strip()
+    if not q:
+        return "web_search: empty query."
+    searxng = (searxng_url or os.environ.get("GONEXT_SEARXNG_URL") or "").strip()
+
+    # Short TTL cache (identical query within a turn). Key includes the search model so a
+    # synthesized answer isn't served when the model config changed.
+    ckey = (q.lower(), searxng, search_base_url, search_model_id)
+    hit = _WEB_SEARCH_CACHE.get(ckey)
+    if hit and (_t.time() - hit[0]) < _WEB_SEARCH_TTL:
+        return hit[1]
+
+    per_timeout = 6   # per-backend socket timeout
+    overall = 9       # overall wall-clock deadline across all backends
+
+    # Result-producing backends (run in parallel). Order = tie-break priority in the merge.
+    tasks = [("ddg", lambda: _bk_ddg_html(q, per_timeout))]
+    if searxng:
+        tasks.append(("searxng", lambda: _bk_searxng(q, searxng, per_timeout)))
+    tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
+    tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))
+
+    results_by = {}
+    answers = []                 # zero-click 'answer box' text (SearXNG + DDG-IA)
+    summary, summary_src = "", ""
+    with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
+        futs = {ex.submit(fn): name for name, fn in tasks}
+        end = _t.time() + overall
+        for fut, name in futs.items():
+            try:
+                r = fut.result(timeout=max(0.0, end - _t.time()))
+            except Exception:  # noqa: BLE001 — a slow/failed backend is skipped
+                continue
+            if name == "summary":
+                summary, summary_src = r
+            elif name == "searxng":
+                results_by["searxng"] = r.get("results", [])
+                answers.extend(r.get("answers", []))
+            else:
+                results_by[name] = r
+    if summary:  # DDG instant answer is an 'answer box' too
+        answers.insert(0, summary + (f" (source: {summary_src})" if summary_src else ""))
+
+    # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
+    merged = {}
+    order = 0
+    for name in ("searxng", "ddg", "wikipedia"):
+        for r in results_by.get(name, []):
+            nu = _norm_url(r.get("url"))
+            if not nu:
+                continue
+            if nu in merged:
+                merged[nu]["hits"] += 1
+                if not merged[nu]["snippet"] and r.get("snippet"):
+                    merged[nu]["snippet"] = r["snippet"]
+            else:
+                merged[nu] = {
+                    "title": r.get("title") or r["url"], "snippet": r.get("snippet", ""),
+                    "url": r["url"], "hits": 1, "order": order,
+                }
+                order += 1
+    ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]
+
+    if not answers and not ranked:
         return (
             f"No results found for '{q}'. Tell the user you couldn't find this — "
             "do NOT invent an answer or a URL."
         )
 
+    # Search-AND-read (task #104): READ the top pages here (in parallel) so the model gets
+    # answer-ready content from ONE call and rarely needs a follow-up fetch_url — the big win
+    # when the reasoning model is slow (each avoided round-trip saves a full model step).
+    read_enabled = (os.environ.get("GONEXT_SEARCH_READ", "1").strip() != "0")
+    try:
+        read_k = max(0, int(os.environ.get("GONEXT_SEARCH_READ_K", "3")))
+    except ValueError:
+        read_k = 3
+    try:
+        page_chars = max(400, int(os.environ.get("GONEXT_SEARCH_PAGE_CHARS", "1600")))
+    except ValueError:
+        page_chars = 1600
+    page_texts = {}
+    if read_enabled and read_k and ranked:
+        page_texts = _read_pages([e["url"] for e in ranked[:read_k]], q,
+                                  per_page_chars=page_chars, per_timeout=5)
+
     parts = []
-    if summary:
-        parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
-    if results:
-        lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
-        for i, (title, snippet, url) in enumerate(results, 1):
-            line = f"{i}. {title}"
-            if snippet:
-                line += f" — {snippet[:160]}"
-            lines.append(line + f"\n   {url}")
-        parts.append("\n".join(lines))
-    return "\n\n".join(parts)
+    if answers:
+        parts.append("Answer:\n" + "\n".join(f"• {a}" for a in answers[:3]))
+    if ranked:
+        header = ("Sources — already READ for you (you usually do NOT need fetch_url; "
+                  "call it only to read a specific page in full):")
+        blocks = [header]
+        for i, e in enumerate(ranked, 1):
+            head = f"[{i}] {e['title']}\n    {e['url']}"
+            body = page_texts.get(e["url"]) or (e["snippet"][:200] if e["snippet"] else "")
+            blocks.append(head + (f"\n{body}" if body else ""))
+        parts.append("\n\n".join(blocks))
+    out = "\n\n".join(parts)
+
+    # Search model (task #105): if configured, have it write the answer FROM the results we
+    # just read — so the code model gets a clean cited answer, not raw pages. Best-effort:
+    # any failure falls back to the read output above.
+    if search_base_url and search_model_id and (answers or ranked):
+        try:
+            synth = _synthesize_search(q, out, search_base_url, search_model_id)
+            if synth:
+                srcs = "\n".join(f"[{i}] {e['url']}" for i, e in enumerate(ranked, 1))
+                out = f"Answer:\n{synth}" + (f"\n\nSources:\n{srcs}" if srcs else "")
+        except Exception as e:  # noqa: BLE001
+            _log(f"search-model synthesis failed ({type(e).__name__}: {e}); using read output")
+
+    _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
+    return out
 
 
 class _AgentConfigError(RuntimeError):
@@ -3874,13 +4138,13 @@ def run_agent_chat(cfg):
 
     @tool
     def web_search(query: str) -> str:
-        """Search for factual or encyclopedic information using free no-key sources.
+        """Search the web AND read the top results in one call — returns an 'Answer:' (when found) plus the READ CONTENT of the best pages, not just links. Because the pages are already read for you, you normally do NOT need to call fetch_url afterwards — answer or create_pdf directly from what this returns. Only call fetch_url when you must read one specific page in full.
 
-        Use this INSTEAD of guessing a URL when the user asks to 'find' something or asks a
-        general-knowledge question. Returns a short summary and a source URL.
+        Use this INSTEAD of guessing a URL whenever the user asks to 'find' something or asks a
+        factual / current-events question.
 
         Args:
-            query: What to look up, e.g. 'capital of France' or 'productivity day-to-day method'.
+            query: What to look up, e.g. 'FIFA World Cup 2026 final date' or 'express error handling best practices'.
         """
         dup, nudge = _dedup("web_search", query,
                             "Use the facts you have ALREADY gathered: if the task asks for a "
@@ -3893,7 +4157,13 @@ def run_agent_chat(cfg):
         if blocked:
             return blocked
         _emit({"type": "step", "text": f"Searching the web → {query[:80]}"})
-        result = _web_search_impl(query)
+        # SearXNG URL (task #103) + optional search model (task #105) from Settings.
+        result = _web_search_impl(
+            query,
+            searxng_url=(cfg.get("searxngUrl") or "").strip(),
+            search_base_url=(cfg.get("searchBaseURL") or "").strip(),
+            search_model_id=(cfg.get("searchModelId") or "").strip(),
+        )
         _log(f"web_search {query[:60]!r} → {result[:80]}")
         _last_obs["text"] = result
         if result and not result.startswith("Error"):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.274",
+  "version": "1.0.276",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",