@tiens.nguyen/gonext-local-worker 1.0.278 → 1.0.281

package/gonext-local-worker.mjs

@@ -1893,9 +1893,6 @@ async function runAgentChatJob(job) {
       agentModelId: payload?.agentModelId ?? "",
       codingBaseURL: payload?.codingBaseURL ?? "",
       codingModelId: payload?.codingModelId ?? "",
-      // Optional search model (task #105): synthesizes web_search results. Empty = no synthesis.
-      searchBaseURL: payload?.searchBaseURL ?? "",
-      searchModelId: payload?.searchModelId ?? "",
       tools: payload?.tools ?? ["http_request"],
       // 0 = "not explicitly set": python defaults to 5, but may auto-raise to 12 when
       // workspaces are registered. Passing a literal 5 here would look user-set and

package/gonext_agent_chat.py

@@ -413,15 +413,23 @@ def _norm_url(u):
 
 
 def _bk_ddg_html(q, timeout=6):
-    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless)."""
+    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless).
+
+    Parses each result as a UNIT (task #104 fix): for every result__a anchor we take the
+    snippet that belongs to IT — the result__snippet appearing before the NEXT anchor —
+    instead of zipping two independent global lists. The old zip silently misaligned
+    snippets onto the wrong title/URL the moment any result lacked a snippet (ads, news,
+    or snippet-less rows), so the model read evidence attached to the wrong page.
+    """
     from urllib.parse import quote, unquote
     html_body = _get_text(f"https://html.duckduckgo.com/html/?q={quote(q)}", timeout=timeout)
     if not html_body:
         return []
     out = []
-    anchors = re.findall(r'<a\b[^>]*class="result__a"[^>]*>.*?</a>', html_body, re.S)
-    snippets = re.findall(r'<a\b[^>]*class="result__snippet"[^>]*>(.*?)</a>', html_body, re.S)
-    for i, a in enumerate(anchors):
+    anchors = list(re.finditer(
+        r'<a\b[^>]*class="[^"]*result__a[^"]*"[^>]*>.*?</a>', html_body, re.S))
+    for idx, am in enumerate(anchors):
+        a = am.group(0)
         hm = re.search(r'href="([^"]*)"', a)
         if not hm:
             continue
@@ -433,16 +441,25 @@ def _bk_ddg_html(q, timeout=6):
         if not real.startswith("http"):
             continue
         title = _search_strip(a)
-        snip = _search_strip(snippets[i]) if i < len(snippets) else ""
-        if title:
-            out.append({"title": title[:120], "snippet": snip, "url": real})
+        if not title:
+            continue
+        # Snippet = the result__snippet that lies BETWEEN this anchor and the next one
+        # (i.e. inside this result's block). Missing → "" with no shift onto the next row.
+        seg_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else len(html_body)
+        seg = html_body[am.end():seg_end]
+        sm = re.search(
+            r'<a\b[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>', seg, re.S)
+        snip = _search_strip(sm.group(1)) if sm else ""
+        out.append({"title": title[:120], "snippet": snip, "url": real})
     return out
 
 
 def _bk_searxng(q, base, timeout=6):
-    """Self-hosted SearXNG JSON API (keyless) → {"results":[{title,snippet,url}], "answers":[str]}.
-    `answers` = SearXNG's aggregated zero-click ANSWERS + INFOBOXES (the 'answer box' / knowledge
-    panel a human sees on a SERP) — surfaced first so one search often needs no page read (#104)."""
+    """Self-hosted SearXNG JSON API → {results:[{title,snippet,url}], answers:[str]} (keyless).
+
+    'answers' + 'infoboxes' are SearXNG's aggregated SERP answer box / knowledge panel — a
+    DIRECT answer we can surface without reading any page (task #104). Returns the dict shape
+    so _web_search_impl can promote answers into the 'Answer:' block."""
     from urllib.parse import quote
     base = (base or "").strip().rstrip("/")
     if not base:
@@ -459,15 +476,16 @@ def _bk_searxng(q, base, timeout=6):
                 "snippet": _search_strip(r.get("content") or ""),
                 "url": url,
             })
+        # 'answers' entries are plain strings on some versions, {answer: ...} dicts on others.
         for a in (data.get("answers") or []):
-            txt = _search_strip(a if isinstance(a, str) else (a.get("answer") or a.get("content") or ""))
+            txt = _search_strip(str((a.get("answer") if isinstance(a, dict) else a) or ""))
             if txt:
-                answers.append(txt[:600])
+                answers.append(txt)
         for ib in (data.get("infoboxes") or []):
             if isinstance(ib, dict):
-                txt = _search_strip(ib.get("content") or "")
+                txt = _search_strip(str(ib.get("content") or ""))
                 if txt:
-                    answers.append(txt[:600])
+                    answers.append(txt)
     return {"results": results, "answers": answers}
 
 
@@ -508,91 +526,92 @@ def _bk_ddg_ia_summary(q, timeout=6):
     return ("", "")
 
 
-def _condense_for_query(text, query, max_chars=1600):
-    """Keep the QUERY-RELEVANT paragraphs of a page (task #104): split into paragraphs,
-    score each by distinct query-term overlap, keep the highest-scoring ones up to max_chars,
-    then restore original order. Boilerplate/nav is already gone (via _html_to_text); this
-    picks the part that actually answers the query so the read stays small + on-signal."""
-    terms = {t for t in re.findall(r"[a-z0-9]{3,}", (query or "").lower())}
-    paras = [p.strip() for p in re.split(r"\n\s*\n|\n", text or "") if len(p.strip()) > 20]
-    if not paras:
-        return (text or "")[:max_chars].strip()
-    if not terms:
-        return "\n".join(paras)[:max_chars].strip()
-    scored = [(sum(1 for t in terms if t in p.lower()), i, p) for i, p in enumerate(paras)]
-    chosen, total = [], 0
-    for score, i, p in sorted(scored, key=lambda x: (-x[0], x[1])):
-        if score == 0 and chosen:  # stop once we're into zero-signal paragraphs
-            break
-        if chosen and total + len(p) > max_chars:
-            break
-        chosen.append((i, p))
-        total += len(p) + 1
-    chosen.sort()
-    return "\n".join(p for _, p in chosen)[:max_chars].strip()
+def _condense_for_query(text, query, budget=1500):
+    """Keep the query-relevant + STRUCTURAL lines of a page's text, bounded to `budget`
+    chars, in original order (task #104). Structural lines — table rows (which _html_to_text
+    renders with ' | '), list items, or lines carrying a year / clock time — are ALWAYS kept
+    even at zero prose-score, so schedules / fixtures / specs survive instead of being
+    summarized away (the flatten-to-summary failure). Model-free: pure text selection."""
+    if not text:
+        return ""
+    terms = {w for w in re.findall(r"\w+", (query or "").lower()) if len(w) > 2}
+    lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+    picked = []
+    for ln in lines:
+        low = ln.lower()
+        score = sum(1 for t in terms if t in low)
+        structural = (
+            "|" in ln
+            or bool(re.match(r"^([-*•]|\d+[.)])\s", ln))
+            or bool(re.search(r"\b(19|20)\d{2}\b", ln))
+            or bool(re.search(r"\d{1,2}:\d{2}", ln))
+        )
+        if score > 0 or structural:
+            picked.append(ln)
+    if not picked:  # nothing matched — fall back to the page lead so we return SOMETHING
+        picked = lines[:20]
+    kept, total = [], 0
+    for ln in picked:
+        if total + len(ln) + 1 > budget:
+            continue
+        kept.append(ln)
+        total += len(ln) + 1
+    return "\n".join(kept)
 
 
-def _read_pages(urls, query, per_page_chars=1600, per_timeout=5):
-    """Fetch several result pages IN PARALLEL and return {url: condensed_readable_text}.
-    The 'read' half of search-and-read (task #104) — reuses _fetch_page_impl + _html_to_text.
-    Skips non-HTML/failed pages (the snippet stays as the fallback)."""
-    from concurrent.futures import ThreadPoolExecutor
+def _read_pages(ranked, query, per_page_chars=1500, per_timeout=5, overall=6):
+    """Read the given result pages IN PARALLEL → {url: condensed_text} (task #104).
+
+    Model-free: HTTP GET (_fetch_page_impl) + _html_to_text + _condense_for_query. Each URL
+    is SSRF-guarded (_rag_assert_safe_url) BEFORE fetching — these come from an untrusted
+    SERP, and this reads them automatically (not a URL the model explicitly chose), so an
+    entry pointing at an internal/link-local host must never be fetched. PDFs/binaries and
+    any per-page failure are skipped (the caller falls back to that result's snippet)."""
+    from concurrent.futures import ThreadPoolExecutor, as_completed
     import time as _t
+    if not ranked:
+        return {}
 
-    def _read_one(u):
-        status, ctype, raw = _fetch_page_impl(u, timeout=per_timeout, max_bytes=300000)
-        if not status or status >= 400:
-            return u, ""
-        if ctype and ("html" not in ctype and "text" not in ctype):
-            return u, ""  # skip pdf/json/binary — the model can fetch_url those explicitly
-        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
-        return u, _condense_for_query(text, query, per_page_chars)
+    def _read_one(url):
+        try:
+            _rag_assert_safe_url(url)
+        except Exception:  # noqa: BLE001 — SSRF/scheme refusal → skip this page
+            return url, ""
+        status, ctype, raw = _fetch_page_impl(url, timeout=per_timeout, max_bytes=300000)
+        if status is None or (isinstance(status, int) and status >= 400):
+            return url, ""
+        if "application/pdf" in ctype or raw[:5].lstrip().startswith(b"%PDF"):
+            return url, ""  # binary — snippet fallback (fetch_url can't read PDFs either)
+        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=per_page_chars * 3)
+        return url, _condense_for_query(text, query, per_page_chars)
 
     out = {}
-    if not urls:
-        return out
-    with ThreadPoolExecutor(max_workers=len(urls)) as ex:
-        futs = {ex.submit(_read_one, u): u for u in urls}
-        end = _t.time() + per_timeout + 3
-        for fut in list(futs):
+    # Manual shutdown (not `with`): a slow page must not hold the whole search past `overall`
+    # via shutdown(wait=True). We collect what finishes within the deadline, then return.
+    ex = ThreadPoolExecutor(max_workers=len(ranked))
+    futs = {ex.submit(_read_one, e["url"]): e["url"] for e in ranked}
+    end = _t.time() + overall
+    try:
+        for fut in as_completed(futs, timeout=overall):
             try:
-                uu, txt = fut.result(timeout=max(0.0, end - _t.time()))
-                if txt:
-                    out[uu] = txt
+                url, txt = fut.result(timeout=max(0.0, end - _t.time()))
             except Exception:  # noqa: BLE001
                 continue
+            if txt:
+                out[url] = txt
+    except Exception:  # noqa: BLE001 — overall deadline hit; return what finished
+        pass
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
     return out
 
 
-def _synthesize_search(query, context, base_url, model_id, api_key="local"):
-    """Search model (task #105): ask a configured OpenAI-compatible model to write a concise,
-    cited answer from the already-fetched search results — so the (slow) code model doesn't
-    have to read/reason over raw pages. Returns "" on any failure (caller falls back)."""
-    from openai import OpenAI
-    client = OpenAI(base_url=base_url, api_key=api_key or "local", max_retries=0, timeout=45)
-    resp = _chat_create(
-        client,
-        model=model_id or "default_model",
-        messages=[
-            {"role": "system", "content": (
-                "You are a research assistant. Using ONLY the search results provided, write a "
-                "concise, factual answer to the user's query. Cite sources inline as [n] using "
-                "their numbers. If the results do not contain the answer, say so plainly — never "
-                "invent facts, dates, names, or URLs."
-            )},
-            {"role": "user", "content": f"Query: {query}\n\nSearch results:\n{context}\n\nAnswer:"},
-        ],
-        max_tokens=700,
-        temperature=0.2,
-    )
-    return (resp.choices[0].message.content or "").strip()
-
-
-def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", search_model_id=""):
-    """Parallel keyless web search (task #103): run DuckDuckGo-HTML + SearXNG (if configured)
-    + Wikipedia CONCURRENTLY with a short deadline, merge/dedupe/rank by cross-engine
-    agreement, and return a short SUMMARY (when available) + a numbered list of the top
-    PAGES (title — snippet — URL) so the model can fetch_url() the best one. Never fabricates.
+def _web_search_impl(query, max_results=5, searxng_url=""):
+    """Keyless web search that READS for the model (tasks #103 + #104): run DuckDuckGo-HTML
+    + SearXNG (if configured) + Wikipedia CONCURRENTLY, merge/dedupe/rank by cross-engine
+    agreement, surface any SERP answer box, then (unless GONEXT_SEARCH_READ=0) READ the top
+    pages in parallel and return answer-ready condensed content — so the model usually does
+    NOT need a slow fetch_url round-trip. Model-free; never fabricates.
     """
     import os
     import time as _t
@@ -603,29 +622,38 @@ def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", s
         return "web_search: empty query."
     searxng = (searxng_url or os.environ.get("GONEXT_SEARXNG_URL") or "").strip()
 
-    # Short TTL cache (identical query within a turn). Key includes the search model so a
-    # synthesized answer isn't served when the model config changed.
-    ckey = (q.lower(), searxng, search_base_url, search_model_id)
+    # Short TTL cache (identical query within a turn).
+    ckey = (q.lower(), searxng)
     hit = _WEB_SEARCH_CACHE.get(ckey)
     if hit and (_t.time() - hit[0]) < _WEB_SEARCH_TTL:
         return hit[1]
 
     per_timeout = 6   # per-backend socket timeout
-    overall = 9       # overall wall-clock deadline across all backends
+    overall = 5       # overall deadline; a straggler is abandoned (shutdown is non-blocking)
 
     # Result-producing backends (run in parallel). Order = tie-break priority in the merge.
     tasks = [("ddg", lambda: _bk_ddg_html(q, per_timeout))]
     if searxng:
+        # SearXNG aggregates Wikipedia (and everything else) server-side, so the standalone
+        # Wikipedia backend is redundant here — and its full-text API is the slow straggler
+        # (~7s) that otherwise dominates the whole search. Only pay for it as a keyless
+        # fallback when SearXNG isn't configured.
         tasks.append(("searxng", lambda: _bk_searxng(q, searxng, per_timeout)))
-    tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
+    else:
+        tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
     tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))
 
     results_by = {}
-    answers = []                 # zero-click 'answer box' text (SearXNG + DDG-IA)
+    answers = []           # #104: direct answer-box text (SearXNG answers/infoboxes)
     summary, summary_src = "", ""
-    with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
-        futs = {ex.submit(fn): name for name, fn in tasks}
-        end = _t.time() + overall
+    # NOT a `with` block: the context manager's shutdown(wait=True) would block on straggler
+    # threads PAST the deadline (a slow SearXNG/engine could add 10s+). We read results up to
+    # `overall`, then shutdown(wait=False) so the function returns on time; leftover backend
+    # threads finish harmlessly in the background.
+    ex = ThreadPoolExecutor(max_workers=len(tasks))
+    futs = {ex.submit(fn): name for name, fn in tasks}
+    end = _t.time() + overall
+    try:
         for fut, name in futs.items():
             try:
                 r = fut.result(timeout=max(0.0, end - _t.time()))
@@ -633,13 +661,13 @@ def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", s
                 continue
             if name == "summary":
                 summary, summary_src = r
-            elif name == "searxng":
-                results_by["searxng"] = r.get("results", [])
+            elif name == "searxng":  # dict shape: {results, answers}
+                results_by[name] = r.get("results", [])
                 answers.extend(r.get("answers", []))
             else:
                 results_by[name] = r
-    if summary:  # DDG instant answer is an 'answer box' too
-        answers.insert(0, summary + (f" (source: {summary_src})" if summary_src else ""))
+    finally:
+        ex.shutdown(wait=False, cancel_futures=True)
 
     # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
     merged = {}
@@ -661,55 +689,45 @@ def _web_search_impl(query, max_results=5, searxng_url="", search_base_url="", s
                 order += 1
     ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]
 
-    if not answers and not ranked:
+    if not summary and not ranked and not answers:
         return (
             f"No results found for '{q}'. Tell the user you couldn't find this — "
             "do NOT invent an answer or a URL."
         )
 
-    # Search-AND-read (task #104): READ the top pages here (in parallel) so the model gets
-    # answer-ready content from ONE call and rarely needs a follow-up fetch_url — the big win
-    # when the reasoning model is slow (each avoided round-trip saves a full model step).
-    read_enabled = (os.environ.get("GONEXT_SEARCH_READ", "1").strip() != "0")
-    try:
-        read_k = max(0, int(os.environ.get("GONEXT_SEARCH_READ_K", "3")))
-    except ValueError:
-        read_k = 3
-    try:
-        page_chars = max(400, int(os.environ.get("GONEXT_SEARCH_PAGE_CHARS", "1600")))
-    except ValueError:
-        page_chars = 1600
-    page_texts = {}
-    if read_enabled and read_k and ranked:
-        page_texts = _read_pages([e["url"] for e in ranked[:read_k]], q,
-                                  per_page_chars=page_chars, per_timeout=5)
+    # #104 read step: unless disabled, READ the top pages inline so the model gets
+    # answer-ready content in ONE call (no slow fetch_url round-trips). GONEXT_SEARCH_READ=0
+    # is the kill switch → link+snippet behaviour of #103.
+    def _env_int(name, default):
+        try:
+            return max(1, int(os.environ.get(name, "").strip()))
+        except (ValueError, TypeError):
+            return default
+    read_on = os.environ.get("GONEXT_SEARCH_READ", "1").strip().lower() not in ("0", "false", "no")
+    read_k = _env_int("GONEXT_SEARCH_READ_K", 3)
+    page_chars = _env_int("GONEXT_SEARCH_PAGE_CHARS", 1500)
+    page_texts = _read_pages(ranked[:read_k], q, page_chars) if (read_on and ranked) else {}
 
     parts = []
-    if answers:
-        parts.append("Answer:\n" + "\n".join(f"• {a}" for a in answers[:3]))
+    # Answer box(es) first — a direct answer often needs no page read at all.
+    answer_lines = [f"• {a[:600]}" for a in answers[:3]]
+    if summary:
+        answer_lines.append(f"• {summary[:600]}" + (f" (source: {summary_src})" if summary_src else ""))
+    if answer_lines:
+        parts.append("Answer:\n" + "\n".join(answer_lines))
+
     if ranked:
-        header = ("Sources — already READ for you (you usually do NOT need fetch_url; "
-                  "call it only to read a specific page in full):")
-        blocks = [header]
+        if page_texts:
+            lines = ["Sources — already READ for you (you usually do NOT need fetch_url; "
+                     "call it only to read a specific page in full):"]
+        else:
+            lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
         for i, e in enumerate(ranked, 1):
             head = f"[{i}] {e['title']}\n    {e['url']}"
             body = page_texts.get(e["url"]) or (e["snippet"][:200] if e["snippet"] else "")
-            blocks.append(head + (f"\n{body}" if body else ""))
-        parts.append("\n\n".join(blocks))
+            lines.append(head + (f"\n{body}" if body else ""))
+        parts.append("\n".join(lines))
     out = "\n\n".join(parts)
-
-    # Search model (task #105): if configured, have it write the answer FROM the results we
-    # just read — so the code model gets a clean cited answer, not raw pages. Best-effort:
-    # any failure falls back to the read output above.
-    if search_base_url and search_model_id and (answers or ranked):
-        try:
-            synth = _synthesize_search(q, out, search_base_url, search_model_id)
-            if synth:
-                srcs = "\n".join(f"[{i}] {e['url']}" for i, e in enumerate(ranked, 1))
-                out = f"Answer:\n{synth}" + (f"\n\nSources:\n{srcs}" if srcs else "")
-        except Exception as e:  # noqa: BLE001
-            _log(f"search-model synthesis failed ({type(e).__name__}: {e}); using read output")
-
     _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
     return out
 
@@ -3073,15 +3091,6 @@ def run_agent_chat(cfg):
         f"start model={agent_model_id!r} base={agent_base_url!r} "
         f"codeModel={coding_model_id!r} codeBase={coding_base_url!r} maxSteps={max_steps}"
     )
-    # Search model (task #105): who synthesizes web_search answers. Logged so a run shows at a
-    # glance whether it's wired.
-    _search_base = (cfg.get("searchBaseURL") or "").strip()
-    _search_model = (cfg.get("searchModelId") or "").strip()
-    _log(
-        f"search model: {_search_model!r} @ {_search_base}"
-        if _search_base
-        else "search model: none (web_search returns read page content, not a synthesized answer)"
-    )
 
     # Build the task from the conversation history. We include the FULL conversation
     # (both user AND assistant turns) so the agent remembers what it already did —
@@ -4147,13 +4156,16 @@ def run_agent_chat(cfg):
 
     @tool
     def web_search(query: str) -> str:
-        """Search the web AND read the top results in one call — returns an 'Answer:' (when found) plus the READ CONTENT of the best pages, not just links. Because the pages are already read for you, you normally do NOT need to call fetch_url afterwards — answer or create_pdf directly from what this returns. Only call fetch_url when you must read one specific page in full.
+        """Search the web AND read the top pages for you, in one call.
 
-        Use this INSTEAD of guessing a URL whenever the user asks to 'find' something or asks a
-        factual / current-events question.
+        Use this INSTEAD of guessing a URL when the user asks to 'find' something or asks a
+        general-knowledge question. Returns any direct answer box plus the top sources with
+        their CONTENT ALREADY READ (HTML stripped, tables kept) — so you can usually answer
+        or call final_answer/create_pdf straight away WITHOUT calling fetch_url again. Only
+        call fetch_url when you specifically need a single page in full.
 
         Args:
-            query: What to look up, e.g. 'FIFA World Cup 2026 final date' or 'express error handling best practices'.
+            query: What to look up, e.g. 'capital of France' or 'world cup 2026 schedule'.
         """
         dup, nudge = _dedup("web_search", query,
                             "Use the facts you have ALREADY gathered: if the task asks for a "
@@ -4166,13 +4178,9 @@ def run_agent_chat(cfg):
         if blocked:
             return blocked
         _emit({"type": "step", "text": f"Searching the web → {query[:80]}"})
-        # SearXNG URL (task #103) + optional search model (task #105) from Settings.
-        result = _web_search_impl(
-            query,
-            searxng_url=(cfg.get("searxngUrl") or "").strip(),
-            search_base_url=(cfg.get("searchBaseURL") or "").strip(),
-            search_model_id=(cfg.get("searchModelId") or "").strip(),
-        )
+        # SearXNG URL (task #103): from Settings (cfg.searxngUrl) if the API sends it, else
+        # the GONEXT_SEARXNG_URL env fallback inside _web_search_impl. Keyless either way.
+        result = _web_search_impl(query, searxng_url=(cfg.get("searxngUrl") or "").strip())
         _log(f"web_search {query[:60]!r} → {result[:80]}")
         _last_obs["text"] = result
         if result and not result.startswith("Error"):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.278",
+  "version": "1.0.281",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",