@tiens.nguyen/gonext-local-worker 1.0.280 → 1.0.281

package/gonext_agent_chat.py

@@ -413,15 +413,23 @@ def _norm_url(u):
 
 
 def _bk_ddg_html(q, timeout=6):
-    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless)."""
+    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless).
+
+    Parses each result as a UNIT (task #104 fix): for every result__a anchor we take the
+    snippet that belongs to IT — the result__snippet appearing before the NEXT anchor —
+    instead of zipping two independent global lists. The old zip silently misaligned
+    snippets onto the wrong title/URL the moment any result lacked a snippet (ads, news,
+    or snippet-less rows), so the model read evidence attached to the wrong page.
+    """
     from urllib.parse import quote, unquote
     html_body = _get_text(f"https://html.duckduckgo.com/html/?q={quote(q)}", timeout=timeout)
     if not html_body:
         return []
     out = []
-    anchors = re.findall(r'<a\b[^>]*class="result__a"[^>]*>.*?</a>', html_body, re.S)
-    snippets = re.findall(r'<a\b[^>]*class="result__snippet"[^>]*>(.*?)</a>', html_body, re.S)
-    for i, a in enumerate(anchors):
+    anchors = list(re.finditer(
+        r'<a\b[^>]*class="[^"]*result__a[^"]*"[^>]*>.*?</a>', html_body, re.S))
+    for idx, am in enumerate(anchors):
+        a = am.group(0)
         hm = re.search(r'href="([^"]*)"', a)
         if not hm:
             continue
@@ -433,31 +441,52 @@ def _bk_ddg_html(q, timeout=6):
         if not real.startswith("http"):
             continue
         title = _search_strip(a)
-        snip = _search_strip(snippets[i]) if i < len(snippets) else ""
-        if title:
-            out.append({"title": title[:120], "snippet": snip, "url": real})
+        if not title:
+            continue
+        # Snippet = the result__snippet that lies BETWEEN this anchor and the next one
+        # (i.e. inside this result's block). Missing → "" with no shift onto the next row.
+        seg_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else len(html_body)
+        seg = html_body[am.end():seg_end]
+        sm = re.search(
+            r'<a\b[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>', seg, re.S)
+        snip = _search_strip(sm.group(1)) if sm else ""
+        out.append({"title": title[:120], "snippet": snip, "url": real})
     return out
 
 
 def _bk_searxng(q, base, timeout=6):
-    """Self-hosted SearXNG JSON API → aggregated results [{title,snippet,url}] (keyless)."""
+    """Self-hosted SearXNG JSON API → {results:[{title,snippet,url}], answers:[str]} (keyless).
+
+    'answers' + 'infoboxes' are SearXNG's aggregated SERP answer box / knowledge panel — a
+    DIRECT answer we can surface without reading any page (task #104). Returns the dict shape
+    so _web_search_impl can promote answers into the 'Answer:' block."""
     from urllib.parse import quote
     base = (base or "").strip().rstrip("/")
     if not base:
-        return []
+        return {"results": [], "answers": []}
     data = _get_json(f"{base}/search?q={quote(q)}&format=json&safesearch=0", timeout=timeout)
-    out = []
+    results, answers = [], []
     if isinstance(data, dict):
         for r in (data.get("results") or []):
             url = (r.get("url") or "").strip()
             if not url:
                 continue
-            out.append({
+            results.append({
                 "title": _search_strip(r.get("title") or url)[:120],
                 "snippet": _search_strip(r.get("content") or ""),
                 "url": url,
             })
-    return out
+        # 'answers' entries are plain strings on some versions, {answer: ...} dicts on others.
+        for a in (data.get("answers") or []):
+            txt = _search_strip(str((a.get("answer") if isinstance(a, dict) else a) or ""))
+            if txt:
+                answers.append(txt)
+        for ib in (data.get("infoboxes") or []):
+            if isinstance(ib, dict):
+                txt = _search_strip(str(ib.get("content") or ""))
+                if txt:
+                    answers.append(txt)
+    return {"results": results, "answers": answers}
 
 
 def _bk_wikipedia(q, timeout=6, n=5):
@@ -497,11 +526,92 @@ def _bk_ddg_ia_summary(q, timeout=6):
     return ("", "")
 
 
+def _condense_for_query(text, query, budget=1500):
+    """Keep the query-relevant + STRUCTURAL lines of a page's text, bounded to `budget`
+    chars, in original order (task #104). Structural lines — table rows (which _html_to_text
+    renders with ' | '), list items, or lines carrying a year / clock time — are ALWAYS kept
+    even at zero prose-score, so schedules / fixtures / specs survive instead of being
+    summarized away (the flatten-to-summary failure). Model-free: pure text selection."""
+    if not text:
+        return ""
+    terms = {w for w in re.findall(r"\w+", (query or "").lower()) if len(w) > 2}
+    lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+    picked = []
+    for ln in lines:
+        low = ln.lower()
+        score = sum(1 for t in terms if t in low)
+        structural = (
+            "|" in ln
+            or bool(re.match(r"^([-*•]|\d+[.)])\s", ln))
+            or bool(re.search(r"\b(19|20)\d{2}\b", ln))
+            or bool(re.search(r"\d{1,2}:\d{2}", ln))
+        )
+        if score > 0 or structural:
+            picked.append(ln)
+    if not picked:  # nothing matched — fall back to the page lead so we return SOMETHING
+        picked = lines[:20]
+    kept, total = [], 0
+    for ln in picked:
+        if total + len(ln) + 1 > budget:
+            continue
+        kept.append(ln)
+        total += len(ln) + 1
+    return "\n".join(kept)
+
+
+def _read_pages(ranked, query, per_page_chars=1500, per_timeout=5, overall=6):
+    """Read the given result pages IN PARALLEL → {url: condensed_text} (task #104).
+
+    Model-free: HTTP GET (_fetch_page_impl) + _html_to_text + _condense_for_query. Each URL
+    is SSRF-guarded (_rag_assert_safe_url) BEFORE fetching — these come from an untrusted
+    SERP, and this reads them automatically (not a URL the model explicitly chose), so an
+    entry pointing at an internal/link-local host must never be fetched. PDFs/binaries and
+    any per-page failure are skipped (the caller falls back to that result's snippet)."""
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    import time as _t
+    if not ranked:
+        return {}
+
+    def _read_one(url):
+        try:
+            _rag_assert_safe_url(url)
+        except Exception:  # noqa: BLE001 — SSRF/scheme refusal → skip this page
+            return url, ""
+        status, ctype, raw = _fetch_page_impl(url, timeout=per_timeout, max_bytes=300000)
+        if status is None or (isinstance(status, int) and status >= 400):
+            return url, ""
+        if "application/pdf" in ctype or raw[:5].lstrip().startswith(b"%PDF"):
+            return url, ""  # binary — snippet fallback (fetch_url can't read PDFs either)
+        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=per_page_chars * 3)
+        return url, _condense_for_query(text, query, per_page_chars)
+
+    out = {}
+    # Manual shutdown (not `with`): a slow page must not hold the whole search past `overall`
+    # via shutdown(wait=True). We collect what finishes within the deadline, then return.
+    ex = ThreadPoolExecutor(max_workers=len(ranked))
+    futs = {ex.submit(_read_one, e["url"]): e["url"] for e in ranked}
+    end = _t.time() + overall
+    try:
+        for fut in as_completed(futs, timeout=overall):
+            try:
+                url, txt = fut.result(timeout=max(0.0, end - _t.time()))
+            except Exception:  # noqa: BLE001
+                continue
+            if txt:
+                out[url] = txt
+    except Exception:  # noqa: BLE001 — overall deadline hit; return what finished
+        pass
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
+    return out
+
+
 def _web_search_impl(query, max_results=5, searxng_url=""):
-    """Parallel keyless web search (task #103): run DuckDuckGo-HTML + SearXNG (if configured)
-    + Wikipedia CONCURRENTLY with a short deadline, merge/dedupe/rank by cross-engine
-    agreement, and return a short SUMMARY (when available) + a numbered list of the top
-    PAGES (title — snippet — URL) so the model can fetch_url() the best one. Never fabricates.
+    """Keyless web search that READS for the model (tasks #103 + #104): run DuckDuckGo-HTML
+    + SearXNG (if configured) + Wikipedia CONCURRENTLY, merge/dedupe/rank by cross-engine
+    agreement, surface any SERP answer box, then (unless GONEXT_SEARCH_READ=0) READ the top
+    pages in parallel and return answer-ready condensed content — so the model usually does
+    NOT need a slow fetch_url round-trip. Model-free; never fabricates.
     """
     import os
     import time as _t
@@ -519,20 +629,31 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
         return hit[1]
 
     per_timeout = 6   # per-backend socket timeout
-    overall = 9       # overall wall-clock deadline across all backends
+    overall = 5       # overall deadline; a straggler is abandoned (shutdown is non-blocking)
 
     # Result-producing backends (run in parallel). Order = tie-break priority in the merge.
     tasks = [("ddg", lambda: _bk_ddg_html(q, per_timeout))]
     if searxng:
+        # SearXNG aggregates Wikipedia (and everything else) server-side, so the standalone
+        # Wikipedia backend is redundant here — and its full-text API is the slow straggler
+        # (~7s) that otherwise dominates the whole search. Only pay for it as a keyless
+        # fallback when SearXNG isn't configured.
         tasks.append(("searxng", lambda: _bk_searxng(q, searxng, per_timeout)))
-    tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
+    else:
+        tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
     tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))
 
     results_by = {}
+    answers = []           # #104: direct answer-box text (SearXNG answers/infoboxes)
     summary, summary_src = "", ""
-    with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
-        futs = {ex.submit(fn): name for name, fn in tasks}
-        end = _t.time() + overall
+    # NOT a `with` block: the context manager's shutdown(wait=True) would block on straggler
+    # threads PAST the deadline (a slow SearXNG/engine could add 10s+). We read results up to
+    # `overall`, then shutdown(wait=False) so the function returns on time; leftover backend
+    # threads finish harmlessly in the background.
+    ex = ThreadPoolExecutor(max_workers=len(tasks))
+    futs = {ex.submit(fn): name for name, fn in tasks}
+    end = _t.time() + overall
+    try:
         for fut, name in futs.items():
             try:
                 r = fut.result(timeout=max(0.0, end - _t.time()))
@@ -540,8 +661,13 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
                 continue
             if name == "summary":
                 summary, summary_src = r
+            elif name == "searxng":  # dict shape: {results, answers}
+                results_by[name] = r.get("results", [])
+                answers.extend(r.get("answers", []))
             else:
                 results_by[name] = r
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
 
     # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
     merged = {}
@@ -563,22 +689,43 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
                 order += 1
     ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]
 
-    if not summary and not ranked:
+    if not summary and not ranked and not answers:
         return (
             f"No results found for '{q}'. Tell the user you couldn't find this — "
             "do NOT invent an answer or a URL."
         )
 
+    # #104 read step: unless disabled, READ the top pages inline so the model gets
+    # answer-ready content in ONE call (no slow fetch_url round-trips). GONEXT_SEARCH_READ=0
+    # is the kill switch → link+snippet behaviour of #103.
+    def _env_int(name, default):
+        try:
+            return max(1, int(os.environ.get(name, "").strip()))
+        except (ValueError, TypeError):
+            return default
+    read_on = os.environ.get("GONEXT_SEARCH_READ", "1").strip().lower() not in ("0", "false", "no")
+    read_k = _env_int("GONEXT_SEARCH_READ_K", 3)
+    page_chars = _env_int("GONEXT_SEARCH_PAGE_CHARS", 1500)
+    page_texts = _read_pages(ranked[:read_k], q, page_chars) if (read_on and ranked) else {}
+
     parts = []
+    # Answer box(es) first — a direct answer often needs no page read at all.
+    answer_lines = [f"• {a[:600]}" for a in answers[:3]]
     if summary:
-        parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
+        answer_lines.append(f"• {summary[:600]}" + (f" (source: {summary_src})" if summary_src else ""))
+    if answer_lines:
+        parts.append("Answer:\n" + "\n".join(answer_lines))
+
     if ranked:
-        lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
+        if page_texts:
+            lines = ["Sources — already READ for you (you usually do NOT need fetch_url; "
+                     "call it only to read a specific page in full):"]
+        else:
+            lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
         for i, e in enumerate(ranked, 1):
-            line = f"{i}. {e['title']}"
-            if e["snippet"]:
-                line += f" — {e['snippet'][:160]}"
-            lines.append(line + f"\n   {e['url']}")
+            head = f"[{i}] {e['title']}\n    {e['url']}"
+            body = page_texts.get(e["url"]) or (e["snippet"][:200] if e["snippet"] else "")
+            lines.append(head + (f"\n{body}" if body else ""))
         parts.append("\n".join(lines))
     out = "\n\n".join(parts)
     _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
@@ -4009,13 +4156,16 @@ def run_agent_chat(cfg):
 
     @tool
     def web_search(query: str) -> str:
-        """Search for factual or encyclopedic information using free no-key sources.
+        """Search the web AND read the top pages for you, in one call.
 
         Use this INSTEAD of guessing a URL when the user asks to 'find' something or asks a
-        general-knowledge question. Returns a short summary and a source URL.
+        general-knowledge question. Returns any direct answer box plus the top sources with
+        their CONTENT ALREADY READ (HTML stripped, tables kept) — so you can usually answer
+        or call final_answer/create_pdf straight away WITHOUT calling fetch_url again. Only
+        call fetch_url when you specifically need a single page in full.
 
         Args:
-            query: What to look up, e.g. 'capital of France' or 'productivity day-to-day method'.
+            query: What to look up, e.g. 'capital of France' or 'world cup 2026 schedule'.
         """
         dup, nudge = _dedup("web_search", query,
                             "Use the facts you have ALREADY gathered: if the task asks for a "

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.280",
+  "version": "1.0.281",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",