@tiens.nguyen/gonext-local-worker 1.0.237 → 1.0.239

package/gonext_agent_chat.py

@@ -131,6 +131,12 @@ def _html_to_text(html_text, limit=3000):
     # Main menu / Donate / Create account…" before any article text, which both starves
     # the model of real content and bloats the per-step context (OOM risk on local MLX).
     text = re.sub(r"(?is)<(nav|header|footer|aside|form|button|menu)\b.*?</\1>", " ", text)
+    # Preserve TABLE structure before stripping tags: a cell boundary becomes " | "
+    # and a row becomes a newline, so a schedule/fixtures table survives as readable
+    # pipe-delimited rows instead of collapsing into a wall of words — a "make a table"
+    # task (task #75) is unanswerable if the source table is flattened on the way in.
+    text = re.sub(r"(?i)</(td|th)>", " | ", text)
+    text = re.sub(r"(?i)</tr>", "\n", text)
     # Turn block-ending tags into newlines so document structure survives stripping.
     text = re.sub(r"(?i)<(br|/p|/div|/li|/tr|/h[1-6]|/section|/article)\s*>", "\n", text)
     # Remove all remaining tags.
@@ -358,60 +364,90 @@ def _get_json(url, timeout=15):
         return None
 
 
-def _web_search_impl(query):
+def _web_search_impl(query, max_results=5):
     """Look up factual info via free no-key JSON APIs (DuckDuckGo + Wikipedia).
 
-    Returns a short text summary with a source URL, or a 'no results' message.
-    Tries DuckDuckGo Instant Answer first, then falls back to a Wikipedia search +
-    REST summary. Never fabricates — callers should surface 'no results' honestly.
+    Returns a short SUMMARY (for a direct-fact question) followed by a numbered list
+    of the top matching PAGES (title — snippet — URL), so the model can fetch_url() a
+    real page when a one-line summary can't carry the data it needs (a schedule, a
+    table, a fixtures list). Returning only a single Wikipedia intro used to trap weak
+    models in a re-search loop — they never saw a candidate URL to open (task #75).
+    Never fabricates — returns an honest 'no results' when nothing is found.
     """
+    import html as _html
     from urllib.parse import quote
     q = (query or "").strip()
     if not q:
         return "web_search: empty query."
 
-    # 1) DuckDuckGo Instant Answer API.
+    def _strip(s):
+        return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()
+
+    summary = ""
+    summary_src = ""
+    results = []  # (title, snippet, url)
+    seen = set()
+
+    def _add(title, snippet, url):
+        url = (url or "").strip()
+        if not url or url in seen or len(results) >= max_results:
+            return
+        seen.add(url)
+        results.append((_strip(title) or url, _strip(snippet), url))
+
+    # 1) DuckDuckGo Instant Answer — a direct abstract + related topics (as pages).
     ddg = _get_json(
         f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1"
     )
     if isinstance(ddg, dict):
         abstract = (ddg.get("AbstractText") or "").strip()
         if abstract:
-            src = (ddg.get("AbstractURL") or "").strip()
-            return f"{abstract[:1500]}\nSource: {src}" if src else abstract[:1500]
-        # No abstract — use the first related topic that has text.
+            summary = abstract[:1200]
+            summary_src = (ddg.get("AbstractURL") or "").strip()
         for topic in ddg.get("RelatedTopics") or []:
-            if isinstance(topic, dict) and topic.get("Text"):
-                src = (topic.get("FirstURL") or "").strip()
-                text = topic["Text"][:1500]
-                return f"{text}\nSource: {src}" if src else text
+            if isinstance(topic, dict) and topic.get("Text") and topic.get("FirstURL"):
+                _add(topic["Text"][:80], topic["Text"], topic["FirstURL"])
 
-    # 2) Wikipedia: find the best-matching title, then fetch its summary extract.
+    # 2) Wikipedia full-text search — SEVERAL candidate pages with snippets, so the
+    #    model can pick the specific one (e.g. a "…knockout stage" fixtures page).
     search = _get_json(
         "https://en.wikipedia.org/w/api.php?action=query&list=search"
-        f"&srsearch={quote(q)}&format=json&srlimit=1"
+        f"&srsearch={quote(q)}&srlimit={max_results}&format=json"
     )
-    title = ""
     try:
-        title = search["query"]["search"][0]["title"]
+        hits = search["query"]["search"]
     except Exception:  # noqa: BLE001
-        title = ""
-    if title:
-        slug = quote(title.replace(" ", "_"))
-        summary = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
-        if isinstance(summary, dict):
-            extract = (summary.get("extract") or "").strip()
-            if extract:
-                src = (
-                    (summary.get("content_urls") or {}).get("desktop", {}).get("page", "")
-                    or f"https://en.wikipedia.org/wiki/{slug}"
-                )
-                return f"{extract[:1500]}\nSource: {src}"
+        hits = []
+    for hit in hits:
+        slug = quote(hit.get("title", "").replace(" ", "_"))
+        _add(hit.get("title", ""), hit.get("snippet", ""),
+             f"https://en.wikipedia.org/wiki/{slug}")
+    # If no direct abstract, use the best-matching page's summary as the answer.
+    if not summary and hits:
+        slug = quote(hits[0]["title"].replace(" ", "_"))
+        s = _get_json("https://en.wikipedia.org/api/rest_v1/page/summary/" + slug)
+        if isinstance(s, dict):
+            summary = (s.get("extract") or "").strip()[:1200]
+            summary_src = f"https://en.wikipedia.org/wiki/{slug}"
+
+    if not summary and not results:
+        return (
+            f"No results found for '{q}'. Tell the user you couldn't find this — "
+            "do NOT invent an answer or a URL."
+        )
 
-    return (
-        f"No results found for '{q}'. Tell the user you couldn't find this — "
-        "do NOT invent an answer or a URL."
-    )
+    parts = []
+    if summary:
+        parts.append(summary + (f"\nSource: {summary_src}" if summary_src else ""))
+    if results:
+        lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
+        for i, (title, snippet, url) in enumerate(results, 1):
+            line = f"{i}. {title}"
+            if snippet:
+                line += f" — {snippet[:160]}"
+            lines.append(line + f"\n   {url}")
+        parts.append("\n".join(lines))
+    return "\n\n".join(parts)
 
 
 class _AgentConfigError(RuntimeError):
@@ -1392,8 +1428,14 @@ def _format_text_for_pdf(text: str, title: str, base_url: str, api_key: str,
         fallback = f"# {title}\n\n{fallback}"
     try:
         from openai import OpenAI
+        # timeout=20, NOT 60 (task #74): this formatting pass runs INSIDE a create_pdf
+        # tool call, which itself runs under the agent's python-executor wall clock. A
+        # 60s wait here could eat the whole step budget — the PDF then rendered+uploaded
+        # fine but the model was told the step timed out, so it re-created the PDF from
+        # scratch (3 duplicate PDFs, ~10 wasted minutes). Formatting is optional polish:
+        # give it 20s, then fall back to the raw text and leave room for render+upload.
         client = OpenAI(base_url=base_url, api_key=api_key or "local",
-                        max_retries=0, timeout=60)
+                        max_retries=0, timeout=20)
         system_content = (
             "You format raw text/data into clean Markdown for a PDF document. "
             "Add a single top-level '# Title', sensible headings, bullet lists, "
@@ -2961,8 +3003,15 @@ def run_agent_chat(cfg):
             )
 
             _emit({"type": "step", "text": "Formatting document…"})
+            # CHAT model, not the coder (task #74 review): formatting raw text into
+            # Markdown is a language task, and the chat model (local MLX) answers in
+            # seconds where the remote coder can take minutes to first token — inside
+            # the 20s format timeout that difference is "formatted" vs "always falls
+            # back to raw text". Same split as routing/plain replies/summarization.
             markdown_text = _format_text_for_pdf(
-                doc_text, doc_title, coding_base_url, agent_api_key, coding_model_id,
+                doc_text, doc_title,
+                agent_base_url or coding_base_url, agent_api_key,
+                agent_model_id or coding_model_id,
                 instruction=edit_instruction,
             )
 
@@ -3223,6 +3272,8 @@ def run_agent_chat(cfg):
         f"- RESEARCH BUDGET — you may make at most {research_budget} web_search/fetch_url calls "
         "TOTAL per task (rewording a query still counts). Reserve your remaining steps for "
         "create_pdf / final_answer.\n"
+        "- If an earlier Observation already says '✅ Your PDF … is ready', the PDF EXISTS — "
+        "do NOT call create_pdf again. Pass that download link to final_answer.\n"
         "- Do NOT put final_answer outside the code block.\n\n"
     )
     # Recover an interrupted prior turn's step trace, if one survives on disk (crash/
@@ -3495,7 +3546,12 @@ def run_agent_chat(cfg):
             _log(f"fetch_url {url} → PDF, cannot read")
             _last_obs["text"] = msg
             return msg
-        text = _html_to_text(raw.decode("utf-8", errors="replace")) or "(page had no readable text)"
+        # 8000, not the 3000 default: a research page's real content (fixture tables,
+        # data) sits well past the lead, so the old cap returned only the intro/nav and
+        # starved the model (task #75). Older large observations are trimmed by the #63
+        # step_callback, so the extra size only costs context for the latest 2 steps.
+        text = (_html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
+                or "(page had no readable text)")
         out = f"{url}\n{text}"
         _log(f"fetch_url {url} → {len(text)} chars")
         _last_obs["text"] = out
@@ -3595,6 +3651,22 @@ def run_agent_chat(cfg):
         _last_obs["text"] = result
         return result
 
+    # Task #74 retry guard. When a create_pdf call outlives the executor's wall clock,
+    # the tool still finishes (render + S3 upload succeed, "PDF ready" is emitted) but
+    # the model is told the step FAILED — so it calls create_pdf again for the same
+    # document. Remember the last success here; `timeout_pending` is flipped by
+    # step_callback whenever a step's result was an executor-timeout error, meaning
+    # whatever succeeded during that step was never reported to the model.
+    _pdf_state: dict = {"last": None, "timeout_pending": False}
+
+    def _pdf_sig(title: str, text: str) -> str:
+        return (title.strip().lower() + "||"
+                + re.sub(r"\s+", " ", (text or "").strip().lower())[:1500])
+
+    def _pdf_titles_similar(a: str, b: str) -> bool:
+        a, b = a.strip().lower(), b.strip().lower()
+        return bool(a and b) and (a == b or a in b or b in a)
+
     @tool
     def create_pdf(text: str, title: str = "") -> str:
         """Create a PDF document from text/data and return a download link.
@@ -3608,10 +3680,46 @@ def run_agent_chat(cfg):
             title: Optional document title shown at the top and used in the file name.
         """
         doc_title = (title or "").strip() or "Document"
+        # Retry guard (task #74): identical content is ALWAYS a retry; a merely similar
+        # title counts only when the previous success was swallowed by an executor
+        # timeout (the model rewords/shortens on retry, so the text rarely matches).
+        prev = _pdf_state.get("last")
+        if prev and (
+            prev["sig"] == _pdf_sig(doc_title, text)
+            or (_pdf_state.get("timeout_pending")
+                and _pdf_titles_similar(prev["title"], doc_title))
+        ):
+            _log(f"create_pdf retry detected (prev title={prev['title']!r}) "
+                 "→ returning the existing PDF instead of re-rendering")
+            _emit({"type": "step",
+                   "text": f"PDF already created → {prev['title']}.pdf (reusing link)"})
+            _last_obs["text"] = prev["msg"]
+            return prev["msg"]
+
+        # Refuse a placeholder skeleton instead of shipping a confident-but-empty PDF
+        # (task #75): a table of 'TBD vs TBD' rows means the agent never found the real
+        # data. 3+ placeholder cells is the fingerprint of that failure and virtually
+        # never appears in a genuine user document. Steer the model to answer honestly
+        # rather than render a green "✅ ready" over an empty table.
+        _placeholders = len(re.findall(
+            r"\b(?:TBD|TBA|TBC|N/?A)\b|\?\?\?|—\s*(?:vs\.?)?\s*—", text or "", re.I))
+        if _placeholders >= 3:
+            msg = (
+                f"This document is mostly placeholders ({_placeholders} TBD/TBA cells) — "
+                "the real data was never found, so the PDF would be empty. Do NOT retry "
+                "create_pdf. Call final_answer to tell the user honestly that you could "
+                f"not find the actual {doc_title} data to fill the table."
+            )
+            _log(f"create_pdf refused: {_placeholders} placeholder cells")
+            _last_obs["text"] = msg
+            return msg
         # 1) Format the raw text into clean Markdown (extra model call, intentional).
+        # CHAT model, not the coder — see the fast-path call site for why (task #74).
         _emit({"type": "step", "text": "Formatting document…"})
         markdown_text = _format_text_for_pdf(
-            text or "", doc_title, coding_base_url, agent_api_key, coding_model_id
+            text or "", doc_title,
+            agent_base_url or coding_base_url, agent_api_key,
+            agent_model_id or coding_model_id,
         )
 
         # 2) Render the Markdown to PDF bytes locally (pure-Python xhtml2pdf).
@@ -3655,6 +3763,9 @@ def run_agent_chat(cfg):
         )
         _emit({"type": "step", "text": f"PDF ready → {doc_title}.pdf"})
         _log(f"create_pdf ok title={doc_title!r} bytes={len(pdf_bytes)}")
+        _pdf_state["last"] = {
+            "sig": _pdf_sig(doc_title, text), "title": doc_title, "msg": out,
+        }
         _last_obs["text"] = out
         return out
 
@@ -3670,6 +3781,19 @@ def run_agent_chat(cfg):
     def step_callback(step_log):
         step_num = getattr(step_log, "step_number", "?")
 
+        # Task #74: an executor-timeout error means whatever the step's tool call
+        # achieved was NEVER reported to the model (a create_pdf may have finished —
+        # or may still be finishing — in the background). Flag it so a repeat
+        # create_pdf with a similar title is treated as the retry it is; any step
+        # that ends with a real (non-timeout) result clears the flag.
+        try:
+            _err = getattr(step_log, "error", None)
+            _pdf_state["timeout_pending"] = bool(
+                _err and "maximum execution time" in str(_err)
+            )
+        except Exception:  # noqa: BLE001
+            pass
+
         # (B) Trim OLD, LARGE tool observations out of the CodeAgent's step memory so
         # they aren't re-sent to the model on every subsequent step (the #63 O(n²) token
         # blow-up: an uncapped read_file_lines dump cost ~5-6k tokens PER remaining step).
@@ -5122,7 +5246,15 @@ def run_agent_chat(cfg):
         )
         if _AgentBase is CodeAgent:
             # Only the CodeAgent runs a Python executor; ToolCallingAgent takes neither.
-            agent_kwargs["executor_kwargs"] = {"timeout_seconds": 60}
+            # 660s, NOT 60 (task #74): the wall clock must exceed the LARGEST tool-internal
+            # timeout (run_command caps at 600s; create_pdf's format+render+upload can pass
+            # 60s), otherwise a tool that eventually SUCCEEDS gets reported to the model as
+            # 'Code execution exceeded the maximum execution time' — the model then retries
+            # work that already finished (the duplicate-PDF bug). Every tool enforces its
+            # own tighter timeout and returns a clean message; runaway pure-python is still
+            # caught by smolagents' op-count caps (MAX_OPERATIONS/MAX_WHILE_ITERATIONS),
+            # so this wall clock is a last-resort backstop, not the primary guard.
+            agent_kwargs["executor_kwargs"] = {"timeout_seconds": 660}
             agent_kwargs["additional_authorized_imports"] = [
                 "json", "base64", "urllib", "urllib.request", "urllib.error"
             ]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.237",
+  "version": "1.0.239",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",