@tiens.nguyen/gonext-local-worker 1.0.275 → 1.0.277

package/gonext-local-worker.mjs

@@ -1893,6 +1893,9 @@ async function runAgentChatJob(job) {
       agentModelId: payload?.agentModelId ?? "",
       codingBaseURL: payload?.codingBaseURL ?? "",
       codingModelId: payload?.codingModelId ?? "",
+      // Optional search model (task #105): synthesizes web_search results. Empty = no synthesis.
+      searchBaseURL: payload?.searchBaseURL ?? "",
+      searchModelId: payload?.searchModelId ?? "",
       tools: payload?.tools ?? ["http_request"],
       // 0 = "not explicitly set": python defaults to 5, but may auto-raise to 12 when
       // workspaces are registered. Passing a literal 5 here would look user-set and

package/gonext_agent_chat.py

@@ -440,24 +440,35 @@ def _bk_ddg_html(q, timeout=6):
 
 
 def _bk_searxng(q, base, timeout=6):
-    """Self-hosted SearXNG JSON API → aggregated results [{title,snippet,url}] (keyless)."""
+    """Self-hosted SearXNG JSON API (keyless) → {"results":[{title,snippet,url}], "answers":[str]}.
+    `answers` = SearXNG's aggregated zero-click ANSWERS + INFOBOXES (the 'answer box' / knowledge
+    panel a human sees on a SERP) — surfaced first so one search often needs no page read (#104)."""
     from urllib.parse import quote
     base = (base or "").strip().rstrip("/")
     if not base:
-        return []
+        return {"results": [], "answers": []}
     data = _get_json(f"{base}/search?q={quote(q)}&format=json&safesearch=0", timeout=timeout)
-    out = []
+    results, answers = [], []
     if isinstance(data, dict):
         for r in (data.get("results") or []):
             url = (r.get("url") or "").strip()
             if not url:
                 continue
-            out.append({
+            results.append({
                 "title": _search_strip(r.get("title") or url)[:120],
                 "snippet": _search_strip(r.get("content") or ""),
                 "url": url,
             })
-    return out
+        for a in (data.get("answers") or []):
+            txt = _search_strip(a if isinstance(a, str) else (a.get("answer") or a.get("content") or ""))
+            if txt:
+                answers.append(txt[:600])
+        for ib in (data.get("infoboxes") or []):
+            if isinstance(ib, dict):
+                txt = _search_strip(ib.get("content") or "")
+                if txt:
+                    answers.append(txt[:600])
+    return {"results": results, "answers": answers}
 
 
 def _bk_wikipedia(q, timeout=6, n=5):
@@ -497,7 +508,87 @@ def _bk_ddg_ia_summary(q, timeout=6):
     return ("", "")
 
 
-def _web_search_impl(query, max_results=5, searxng_url=""):
+def _condense_for_query(text, query, max_chars=1600):
+    """Keep the QUERY-RELEVANT paragraphs of a page (task #104): split into paragraphs,
+    score each by distinct query-term overlap, keep the highest-scoring ones up to max_chars,
+    then restore original order. Boilerplate/nav is already gone (via _html_to_text); this
+    picks the part that actually answers the query so the read stays small + on-signal."""
+    terms = {t for t in re.findall(r"[a-z0-9]{3,}", (query or "").lower())}
+    paras = [p.strip() for p in re.split(r"\n\s*\n|\n", text or "") if len(p.strip()) > 20]
+    if not paras:
+        return (text or "")[:max_chars].strip()
+    if not terms:
+        return "\n".join(paras)[:max_chars].strip()
+    scored = [(sum(1 for t in terms if t in p.lower()), i, p) for i, p in enumerate(paras)]
+    chosen, total = [], 0
+    for score, i, p in sorted(scored, key=lambda x: (-x[0], x[1])):
+        if score == 0 and chosen:  # stop once we're into zero-signal paragraphs
+            break
+        if chosen and total + len(p) > max_chars:
+            break
+        chosen.append((i, p))
+        total += len(p) + 1
+    chosen.sort()
+    return "\n".join(p for _, p in chosen)[:max_chars].strip()
+
+
+def _read_pages(urls, query, per_page_chars=1600, per_timeout=5):
+    """Fetch several result pages IN PARALLEL and return {url: condensed_readable_text}.
+    The 'read' half of search-and-read (task #104) — reuses _fetch_page_impl + _html_to_text.
+    Skips non-HTML/failed pages (the snippet stays as the fallback)."""
+    from concurrent.futures import ThreadPoolExecutor
+    import time as _t
+
+    def _read_one(u):
+        status, ctype, raw = _fetch_page_impl(u, timeout=per_timeout, max_bytes=300000)
+        if not status or status >= 400:
+            return u, ""
+        if ctype and ("html" not in ctype and "text" not in ctype):
+            return u, ""  # skip pdf/json/binary — the model can fetch_url those explicitly
+        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
+        return u, _condense_for_query(text, query, per_page_chars)
+
+    out = {}
+    if not urls:
+        return out
+    with ThreadPoolExecutor(max_workers=len(urls)) as ex:
+        futs = {ex.submit(_read_one, u): u for u in urls}
+        end = _t.time() + per_timeout + 3
+        for fut in list(futs):
+            try:
+                uu, txt = fut.result(timeout=max(0.0, end - _t.time()))
+                if txt:
+                    out[uu] = txt
+            except Exception:  # noqa: BLE001
+                continue
+    return out
+
+
+def _synthesize_search(query, context, base_url, model_id, api_key="local"):
+    """Search model (task #105): ask a configured OpenAI-compatible model to write a concise,
+    cited answer from the already-fetched search results — so the (slow) code model doesn't
+    have to read/reason over raw pages. Returns "" on any failure (caller falls back)."""
+    from openai import OpenAI
+    client = OpenAI(base_url=base_url, api_key=api_key or "local", max_retries=0, timeout=45)
+    resp = _chat_create(
+        client,
+        model=model_id or "default_model",
+        messages=[
+            {"role": "system", "content": (
+                "You are a research assistant. Using ONLY the search results provided, write a "
+                "concise, factual answer to the user's query. Cite sources inline as [n] using "
+                "their numbers. If the results do not contain the answer, say so plainly — never "
+                "invent facts, dates, names, or URLs."
+            )},
+            {"role": "user", "content": f"Query: {query}\n\nSearch results:\n{context}\n\nAnswer:"},
+        ],
+        max_tokens=700,
+        temperature=0.2,
+    )
+    return (resp.choices[0].message.content or "").strip()
+
+
+def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", search_model_id=""):
     """Parallel keyless web search (task #103): run DuckDuckGo-HTML + SearXNG (if configured)
     + Wikipedia CONCURRENTLY with a short deadline, merge/dedupe/rank by cross-engine
     agreement, and return a short SUMMARY (when available) + a numbered list of the top
@@ -512,8 +603,9 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
         return "web_search: empty query."
     searxng = (searxng_url or os.environ.get("GONEXT_SEARXNG_URL") or "").strip()
 
-    # Short TTL cache (identical query within a turn).
-    ckey = (q.lower(), searxng)
+    # Short TTL cache (identical query within a turn). Key includes the search model so a
+    # synthesized answer isn't served when the model config changed.
+    ckey = (q.lower(), searxng, search_base_url, search_model_id)
     hit = _WEB_SEARCH_CACHE.get(ckey)
     if hit and (_t.time() - hit[0]) < _WEB_SEARCH_TTL:
         return hit[1]
@@ -529,6 +621,7 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
     tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))
 
     results_by = {}
+    answers = []                 # zero-click 'answer box' text (SearXNG + DDG-IA)
     summary, summary_src = "", ""
     with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
         futs = {ex.submit(fn): name for name, fn in tasks}
@@ -540,8 +633,13 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
                 continue
             if name == "summary":
                 summary, summary_src = r
+            elif name == "searxng":
+                results_by["searxng"] = r.get("results", [])
+                answers.extend(r.get("answers", []))
             else:
                 results_by[name] = r
+    if summary:  # DDG instant answer is an 'answer box' too
+        answers.insert(0, summary + (f" (source: {summary_src})" if summary_src else ""))
 
     # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
     merged = {}
@@ -563,24 +661,55 @@ def _web_search_impl(query, max_results=5, searxng_url=""):
                 order += 1
     ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]
 
-    if not summary and not ranked:
+    if not answers and not ranked:
         return (
             f"No results found for '{q}'. Tell the user you couldn't find this — "
             "do NOT invent an answer or a URL."
         )
 
+    # Search-AND-read (task #104): READ the top pages here (in parallel) so the model gets
+    # answer-ready content from ONE call and rarely needs a follow-up fetch_url — the big win
+    # when the reasoning model is slow (each avoided round-trip saves a full model step).
+    read_enabled = (os.environ.get("GONEXT_SEARCH_READ", "1").strip() != "0")
+    try:
+        read_k = max(0, int(os.environ.get("GONEXT_SEARCH_READ_K", "3")))
+    except ValueError:
+        read_k = 3
+    try:
+        page_chars = max(400, int(os.environ.get("GONEXT_SEARCH_PAGE_CHARS", "1600")))
+    except ValueError:
+        page_chars = 1600
+    page_texts = {}
+    if read_enabled and read_k and ranked:
+        page_texts = _read_pages([e["url"] for e in ranked[:read_k]], q,
+                                  per_page_chars=page_chars, per_timeout=5)
+
     parts = []
-    if summary:
-        parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
+    if answers:
+        parts.append("Answer:\n" + "\n".join(f"• {a}" for a in answers[:3]))
     if ranked:
-        lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
+        header = ("Sources — already READ for you (you usually do NOT need fetch_url; "
+                  "call it only to read a specific page in full):")
+        blocks = [header]
         for i, e in enumerate(ranked, 1):
-            line = f"{i}. {e['title']}"
-            if e["snippet"]:
-                line += f" — {e['snippet'][:160]}"
-            lines.append(line + f"\n   {e['url']}")
-        parts.append("\n".join(lines))
+            head = f"[{i}] {e['title']}\n    {e['url']}"
+            body = page_texts.get(e["url"]) or (e["snippet"][:200] if e["snippet"] else "")
+            blocks.append(head + (f"\n{body}" if body else ""))
+        parts.append("\n\n".join(blocks))
     out = "\n\n".join(parts)
+
+    # Search model (task #105): if configured, have it write the answer FROM the results we
+    # just read — so the code model gets a clean cited answer, not raw pages. Best-effort:
+    # any failure falls back to the read output above.
+    if search_base_url and search_model_id and (answers or ranked):
+        try:
+            synth = _synthesize_search(q, out, search_base_url, search_model_id)
+            if synth:
+                srcs = "\n".join(f"[{i}] {e['url']}" for i, e in enumerate(ranked, 1))
+                out = f"Answer:\n{synth}" + (f"\n\nSources:\n{srcs}" if srcs else "")
+        except Exception as e:  # noqa: BLE001
+            _log(f"search-model synthesis failed ({type(e).__name__}: {e}); using read output")
+
     _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
     return out
 
@@ -4009,13 +4138,13 @@ def run_agent_chat(cfg):
 
     @tool
     def web_search(query: str) -> str:
-        """Search for factual or encyclopedic information using free no-key sources.
+        """Search the web AND read the top results in one call — returns an 'Answer:' (when found) plus the READ CONTENT of the best pages, not just links. Because the pages are already read for you, you normally do NOT need to call fetch_url afterwards — answer or create_pdf directly from what this returns. Only call fetch_url when you must read one specific page in full.
 
-        Use this INSTEAD of guessing a URL when the user asks to 'find' something or asks a
-        general-knowledge question. Returns a short summary and a source URL.
+        Use this INSTEAD of guessing a URL whenever the user asks to 'find' something or asks a
+        factual / current-events question.
 
         Args:
-            query: What to look up, e.g. 'capital of France' or 'productivity day-to-day method'.
+            query: What to look up, e.g. 'FIFA World Cup 2026 final date' or 'express error handling best practices'.
         """
         dup, nudge = _dedup("web_search", query,
                             "Use the facts you have ALREADY gathered: if the task asks for a "
@@ -4028,9 +4157,13 @@ def run_agent_chat(cfg):
         if blocked:
             return blocked
         _emit({"type": "step", "text": f"Searching the web → {query[:80]}"})
-        # SearXNG URL (task #103): from Settings (cfg.searxngUrl) if the API sends it, else
-        # the GONEXT_SEARXNG_URL env fallback inside _web_search_impl. Keyless either way.
-        result = _web_search_impl(query, searxng_url=(cfg.get("searxngUrl") or "").strip())
+        # SearXNG URL (task #103) + optional search model (task #105) from Settings.
+        result = _web_search_impl(
+            query,
+            searxng_url=(cfg.get("searxngUrl") or "").strip(),
+            search_base_url=(cfg.get("searchBaseURL") or "").strip(),
+            search_model_id=(cfg.get("searchModelId") or "").strip(),
+        )
         _log(f"web_search {query[:60]!r} → {result[:80]}")
         _last_obs["text"] = result
         if result and not result.startswith("Error"):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.275",
+  "version": "1.0.277",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",