@tiens.nguyen/gonext-local-worker 1.0.238 → 1.0.239

package/gonext_agent_chat.py

@@ -131,6 +131,12 @@ def _html_to_text(html_text, limit=3000):
     # Main menu / Donate / Create account…" before any article text, which both starves
     # the model of real content and bloats the per-step context (OOM risk on local MLX).
     text = re.sub(r"(?is)<(nav|header|footer|aside|form|button|menu)\b.*?</\1>", " ", text)
+    # Preserve TABLE structure before stripping tags: a cell boundary becomes " | "
+    # and a row becomes a newline, so a schedule/fixtures table survives as readable
+    # pipe-delimited rows instead of collapsing into a wall of words — a "make a table"
+    # task (task #75) is unanswerable if the source table is flattened on the way in.
+    text = re.sub(r"(?i)</(td|th)>", " | ", text)
+    text = re.sub(r"(?i)</tr>", "\n", text)
     # Turn block-ending tags into newlines so document structure survives stripping.
     text = re.sub(r"(?i)<(br|/p|/div|/li|/tr|/h[1-6]|/section|/article)\s*>", "\n", text)
     # Remove all remaining tags.
@@ -358,60 +364,90 @@ def _get_json(url, timeout=15):
         return None
 
 
-def _web_search_impl(query):
+def _web_search_impl(query, max_results=5):
     """Look up factual info via free no-key JSON APIs (DuckDuckGo + Wikipedia).
 
-    Returns a short text summary with a source URL, or a 'no results' message.
-    Tries DuckDuckGo Instant Answer first, then falls back to a Wikipedia search +
-    REST summary. Never fabricates — callers should surface 'no results' honestly.
+    Returns a short SUMMARY (for a direct-fact question) followed by a numbered list
+    of the top matching PAGES (title — snippet — URL), so the model can fetch_url() a
+    real page when a one-line summary can't carry the data it needs (a schedule, a
+    table, a fixtures list). Returning only a single Wikipedia intro used to trap weak
+    models in a re-search loop — they never saw a candidate URL to open (task #75).
+    Never fabricates — returns an honest 'no results' when nothing is found.
     """
+    import html as _html
     from urllib.parse import quote
     q = (query or "").strip()
     if not q:
         return "web_search: empty query."
 
-    # 1) DuckDuckGo Instant Answer API.
+    def _strip(s):
+        return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
+
+    summary = ""
+    summary_src = ""
+    results = []  # (title, snippet, url)
+    seen = set()
+
+    def _add(title, snippet, url):
+        url = (url or "").strip()
+        if not url or url in seen or len(results) >= max_results:
+            return
+        seen.add(url)
+        results.append((_strip(title) or url, _strip(snippet), url))
+
+    # 1) DuckDuckGo Instant Answer — a direct abstract + related topics (as pages).
     ddg = _get_json(
         f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1"
     )
     if isinstance(ddg, dict):
         abstract = (ddg.get("AbstractText") or "").strip()
         if abstract:
-            src = (ddg.get("AbstractURL") or "").strip()
-            return f"{abstract[:1500]}\nSource: {src}" if src else abstract[:1500]
-        # No abstract — use the first related topic that has text.
+            summary = abstract[:1200]
+            summary_src = (ddg.get("AbstractURL") or "").strip()
         for topic in ddg.get("RelatedTopics") or []:
-            if isinstance(topic, dict) and topic.get("Text"):
-                src = (topic.get("FirstURL") or "").strip()
-                text = topic["Text"][:1500]
-                return f"{text}\nSource: {src}" if src else text
+            if isinstance(topic, dict) and topic.get("Text") and topic.get("FirstURL"):
+                _add(topic["Text"][:80], topic["Text"], topic["FirstURL"])
 
-    # 2) Wikipedia: find the best-matching title, then fetch its summary extract.
+    # 2) Wikipedia full-text search — SEVERAL candidate pages with snippets, so the
+    #    model can pick the specific one (e.g. a "…knockout stage" fixtures page).
     search = _get_json(
         "https://en.wikipedia.org/w/api.php?action=query&list=search"
-        f"&srsearch={quote(q)}&format=json&srlimit=1"
+        f"&srsearch={quote(q)}&srlimit={max_results}&format=json"
     )
-    title = ""
     try:
-        title = search["query"]["search"][0]["title"]
+        hits = search["query"]["search"]
     except Exception:  # noqa: BLE001
-        title = ""
-    if title:
-        slug = quote(title.replace(" ", "_"))
-        summary = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
-        if isinstance(summary, dict):
-            extract = (summary.get("extract") or "").strip()
-            if extract:
-                src = (
-                    (summary.get("content_urls") or {}).get("desktop", {}).get("page", "")
-                    or f"https://en.wikipedia.org/wiki/{slug}"
-                )
-                return f"{extract[:1500]}\nSource: {src}"
+        hits = []
+    for hit in hits:
+        slug = quote(hit.get("title", "").replace(" ", "_"))
+        _add(hit.get("title", ""), hit.get("snippet", ""),
+             f"https://en.wikipedia.org/wiki/{slug}")
+    # If no direct abstract, use the best-matching page's summary as the answer.
+    if not summary and hits:
+        slug = quote(hits[0]["title"].replace(" ", "_"))
+        s = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
+        if isinstance(s, dict):
+            summary = (s.get("extract") or "").strip()[:1200]
+            summary_src = f"https://en.wikipedia.org/wiki/{slug}"
+
+    if not summary and not results:
+        return (
+            f"No results found for '{q}'. Tell the user you couldn't find this — "
+            "do NOT invent an answer or a URL."
+        )
 
-    return (
-        f"No results found for '{q}'. Tell the user you couldn't find this — "
-        "do NOT invent an answer or a URL."
-    )
+    parts = []
+    if summary:
+        parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
+    if results:
+        lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
+        for i, (title, snippet, url) in enumerate(results, 1):
+            line = f"{i}. {title}"
+            if snippet:
+                line += f" — {snippet[:160]}"
+            lines.append(line + f"\n   {url}")
+        parts.append("\n".join(lines))
+    return "\n\n".join(parts)
 
 
 class _AgentConfigError(RuntimeError):
@@ -3510,7 +3546,12 @@ def run_agent_chat(cfg):
             _log(f"fetch_url {url} → PDF, cannot read")
             _last_obs["text"] = msg
             return msg
-        text = _html_to_text(raw.decode("utf-8", errors="replace")) or "(page had no readable text)"
+        # 8000, not the 3000 default: a research page's real content (fixture tables,
+        # data) sits well past the lead, so the old cap returned only the intro/nav and
+        # starved the model (task #75). Older large observations are trimmed by the #63
+        # step_callback, so the extra size only costs context for the latest 2 steps.
+        text = (_html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
+                or "(page had no readable text)")
         out = f"{url}\n{text}"
         _log(f"fetch_url {url} → {len(text)} chars")
         _last_obs["text"] = out
@@ -3654,6 +3695,24 @@ def run_agent_chat(cfg):
                    "text": f"PDF already created → {prev['title']}.pdf (reusing link)"})
             _last_obs["text"] = prev["msg"]
             return prev["msg"]
+
+        # Refuse a placeholder skeleton instead of shipping a confident-but-empty PDF
+        # (task #75): a table of 'TBD vs TBD' rows means the agent never found the real
+        # data. 3+ placeholder cells is the fingerprint of that failure and virtually
+        # never appears in a genuine user document. Steer the model to answer honestly
+        # rather than render a green "✅ ready" over an empty table.
+        _placeholders = len(re.findall(
+            r"\b(?:TBD|TBA|TBC|N/?A)\b|\?\?\?|—\s*(?:vs\.?)?\s*—", text or "", re.I))
+        if _placeholders >= 3:
+            msg = (
+                f"This document is mostly placeholders ({_placeholders} TBD/TBA cells) — "
+                "the real data was never found, so the PDF would be empty. Do NOT retry "
+                "create_pdf. Call final_answer to tell the user honestly that you could "
+                f"not find the actual {doc_title} data to fill the table."
+            )
+            _log(f"create_pdf refused: {_placeholders} placeholder cells")
+            _last_obs["text"] = msg
+            return msg
         # 1) Format the raw text into clean Markdown (extra model call, intentional).
         # CHAT model, not the coder — see the fast-path call site for why (task #74).
         _emit({"type": "step", "text": "Formatting document…"})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.238",
+  "version": "1.0.239",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",