@tiens.nguyen/gonext-local-worker 1.0.160 → 1.0.164

package/gonext_agent_chat.py

@@ -1156,17 +1156,39 @@ def _rag_workdir(source_key: str) -> str:
     return base
 
 
-def _rag_download(url: str, dest_path: str = "") -> str:
-    """Download a public URL to a local path with SSRF + size guards. Returns the path."""
+def _rag_download(url: str, dest_path: str = "") -> tuple:
+    """Download a public URL with SSRF + size guards — IDEMPOTENT per URL. Every URL
+    maps to a deterministic work dir (~/.gonext/rag-work/<urlHash>/); relative
+    dest_paths are contained there (never the process cwd, which is the user's home),
+    and a repeat call for an already-downloaded URL reuses the cached file instantly
+    (follow-up questions re-run the pipeline cheaply). Returns (path, cached)."""
     import os
     src = _rag_gdrive_direct((url or "").strip())
     _rag_assert_safe_url(src)
     key = _rag_source_key(url)
+    workdir = _rag_workdir(key)
+    dest_path = os.path.expanduser((dest_path or "").strip())
     if not dest_path:
         name = os.path.basename(src.split("?")[0]) or "download.bin"
         if not name.lower().endswith(".zip") and "zip" in src.lower():
             name += ".zip"
-        dest_path = os.path.join(_rag_workdir(key), name)
+        dest_path = os.path.join(workdir, name)
+    elif not os.path.isabs(dest_path):
+        # Contain relative paths in the per-URL work dir — models pass bare names like
+        # "project.zip" which would otherwise land in the worker's cwd (the home dir).
+        dest_path = os.path.join(workdir, dest_path)
+    # Cache hit 1: the exact target already exists.
+    if os.path.isfile(dest_path) and os.path.getsize(dest_path) > 0:
+        return dest_path, True
+    # Cache hit 2: this URL was downloaded before under a different name — the work dir
+    # is per-URL, so any archive already in it IS this URL's file.
+    try:
+        for fn in sorted(os.listdir(workdir)):
+            p = os.path.join(workdir, fn)
+            if os.path.isfile(p) and fn.lower().endswith(".zip") and os.path.getsize(p) > 0:
+                return p, True
+    except OSError:
+        pass
     os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
     req = urllib.request.Request(src, headers={"User-Agent": "gonext-rag/1.0"})
     total = 0
@@ -1184,15 +1206,48 @@ def _rag_download(url: str, dest_path: str = "") -> str:
                     f"Download exceeds the {_RAG_MAX_DOWNLOAD_BYTES // (1024*1024)} MB cap."
                 )
             out.write(block)
-    return dest_path
+    return dest_path, False
+
+
+def _rag_resolve_zip(zip_path: str) -> str:
+    """Resolve a (possibly relative) zip path. Bare names the model echoes from earlier
+    turns are searched in the rag work dirs so a fresh process still finds them."""
+    import glob as _glob
+    import os
+    zp = os.path.expanduser((zip_path or "").strip())
+    if os.path.isfile(zp):
+        return zp
+    if not os.path.isabs(zp):
+        hits = _glob.glob(os.path.join(
+            os.path.expanduser("~"), ".gonext", "rag-work", "*", os.path.basename(zp)))
+        if hits:
+            return hits[0]
+    return zp
 
 
 def _rag_unzip(zip_path: str, dest_dir: str = "") -> tuple:
-    """Extract a local zip with Zip-Slip + zip-bomb guards. Returns (dir, file_list)."""
+    """Extract a local zip with Zip-Slip + zip-bomb guards — IDEMPOTENT. If the target
+    dir already has extracted files, reuse it instead of re-extracting. Relative
+    dest_dirs are placed next to the zip (inside the per-URL work dir), never the cwd.
+    Returns (dir, file_list, cached)."""
     import os
     import zipfile
+    zip_path = _rag_resolve_zip(zip_path)
+    dest_dir = os.path.expanduser((dest_dir or "").strip())
     if not dest_dir:
         dest_dir = os.path.splitext(zip_path)[0] + "_unzipped"
+    elif not os.path.isabs(dest_dir):
+        dest_dir = os.path.join(os.path.dirname(zip_path) or ".", dest_dir)
+    # Cache hit: already extracted for this zip → return the existing listing.
+    if os.path.isdir(dest_dir):
+        existing = []
+        for dirpath, _dirnames, filenames in os.walk(dest_dir):
+            for fn in filenames:
+                existing.append(os.path.relpath(os.path.join(dirpath, fn), dest_dir))
+            if len(existing) > _RAG_MAX_FILES:
+                break
+        if existing:
+            return dest_dir, sorted(existing), True
     os.makedirs(dest_dir, exist_ok=True)
     dest_root = os.path.realpath(dest_dir)
     written = 0
@@ -1216,7 +1271,7 @@ def _rag_unzip(zip_path: str, dest_dir: str = "") -> tuple:
                 dst.write(src.read())
             written += 1
             names.append(info.filename)
-    return dest_dir, names
+    return dest_dir, names, False
 
 
 def _rag_iter_text_files(root: str):
@@ -1776,8 +1831,11 @@ def run_agent_chat(cfg):
         "THEN list_dir(dir) to find the README, THEN read_text_file(that README) THEN answer from it. "
         "This is the FASTEST path for a summary — do NOT index unless deep Q&A across many files is needed.\n"
         "- user wants deep Q&A across the WHOLE project -> download_file THEN unzip_file THEN "
-        "rag_index(dir, source_url=url) THEN rag_search(source_url=url, query=<question>) THEN answer. "
-        "For FOLLOW-UPs on the SAME url, just rag_search (do NOT re-download). "
+        "rag_index(dir, source_url=url) THEN rag_search(source_url=url, query=<question>) THEN answer.\n"
+        "- FOLLOW-UP question about a ZIP url from EARLIER in the conversation -> try "
+        "rag_search(source_url=url, query=<question>) FIRST. If it says no knowledge base exists, "
+        "run download_file(url) + unzip_file(zip) again — both are CACHED per url and return "
+        "instantly — then list_dir/read_text_file or rag_index as needed. "
         "If the user adds information, rag_add(source_url=url, text=...).\n"
     ) if _RAG_AVAILABLE else ""
     _pdf_read_tool_line = (
@@ -2325,6 +2383,26 @@ def run_agent_chat(cfg):
             in_tok = out_tok = reasoning_len = 0
             first_token_at = None
 
+            # Runaway guard: a degenerate model (seen live: gemma4:26b spewing
+            # "<channel|><thought>" forever) will stream garbage until the client times
+            # out — a multi-minute hang. Abort early if the output either (a) balloons
+            # past a hard char cap, or (b) collapses into a repeating short line. On abort
+            # we return "" so the empty-turn retry re-prompts with the CODE-NOW directive.
+            _MAX_STREAM_CHARS = 16000  # a single agent step never legitimately needs this
+            _stream_buf = []           # every streamed piece (content + reasoning)
+            _stream_chars = 0
+            _runaway = False
+
+            def _looks_runaway() -> bool:
+                tail = "".join(_stream_buf)[-1600:]
+                lines = [ln.strip() for ln in tail.splitlines() if ln.strip()]
+                # 12+ of the last lines are the same short string → degenerate repetition.
+                if len(lines) >= 12:
+                    last = lines[-12:]
+                    if len(set(last)) <= 2 and all(len(ln) <= 80 for ln in last):
+                        return True
+                return False
+
             def _mark_first():
                 # First streamed byte (content OR reasoning). The gap t0→here is pure
                 # prompt-eval time (server sent nothing before this); everything after is
@@ -2362,13 +2440,33 @@ def run_agent_chat(cfg):
                 if rpiece:
                     _mark_first()
                     reasoning_len += len(rpiece)
+                    _stream_buf.append(rpiece)
+                    _stream_chars += len(rpiece)
                     _emit({"type": "stream", "text": rpiece})
                 piece = getattr(delta, "content", None)
                 if piece:
                     _mark_first()
                     parts.append(piece)
+                    _stream_buf.append(piece)
+                    _stream_chars += len(piece)
                     _emit({"type": "stream", "text": piece})
+                # Check the runaway guard periodically (cheap) once enough has streamed.
+                if _stream_chars > 500 and (_stream_chars > _MAX_STREAM_CHARS or _looks_runaway()):
+                    _runaway = True
+                    reason = "char cap" if _stream_chars > _MAX_STREAM_CHARS else "repeating output"
+                    _log(f"streamed generate: ABORTING ({reason}) after {_stream_chars} chars "
+                         "— model degenerated; closing stream.")
+                    try:
+                        stream.close()
+                    except Exception:  # noqa: BLE001
+                        pass
+                    break
             content = "".join(parts)
+            if _runaway:
+                # Discard the garbage; "" makes _streamed_generate run the CODE-NOW retry.
+                _emit({"type": "step", "text": "Model output ran away — restarting this step…"})
+                _log(f"streamed generate: discarded {len(content)} runaway chars → empty content")
+                return "", role, in_tok, out_tok, reasoning_len
             _log(f"streamed generate assembled {len(content)} chars "
                  f"(in={in_tok} out={out_tok} tokens, reasoning={reasoning_len} chars)")
             return content, role, in_tok, out_tok, reasoning_len
@@ -2678,10 +2776,14 @@ def run_agent_chat(cfg):
                 url: public http(s) URL of the file to download.
                 dest_path: optional local save path; leave empty for an automatic work-dir path.
             """
-            _emit({"type": "step", "text": f"Downloading → {url[:80]}"})
             try:
                 import os
-                path = _rag_download(url, (dest_path or "").strip())
+                path, cached = _rag_download(url, (dest_path or "").strip())
+                if cached:
+                    _emit({"type": "step", "text": "Already downloaded — reusing cached file"})
+                    return (f"Already downloaded earlier — reusing {path} "
+                            f"({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}').")
+                _emit({"type": "step", "text": f"Downloading → {url[:80]}"})
                 return f"Downloaded to {path} ({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}') if it is a zip."
             except Exception as e:  # noqa: BLE001
                 return f"Error: download failed: {type(e).__name__}: {e}"
@@ -2694,11 +2796,15 @@ def run_agent_chat(cfg):
                 zip_path: local path to the .zip (from download_file).
                 dest_dir: optional target directory; leave empty for an automatic one.
             """
-            _emit({"type": "step", "text": "Unzipping…"})
             try:
-                d, names = _rag_unzip((zip_path or "").strip(), (dest_dir or "").strip())
+                d, names, cached = _rag_unzip((zip_path or "").strip(), (dest_dir or "").strip())
                 preview = ", ".join(names[:15])
                 more = f" …(+{len(names) - 15} more)" if len(names) > 15 else ""
+                if cached:
+                    _emit({"type": "step", "text": "Already unzipped — reusing extracted files"})
+                    return (f"Already unzipped earlier — {len(names)} files at {d}. "
+                            f"Files: {preview}{more}. Use list_dir/read_text_file or rag_index on it.")
+                _emit({"type": "step", "text": "Unzipping…"})
                 return f"Unzipped {len(names)} files to {d}. Files: {preview}{more}. Next: rag_index(path='{d}', source_url=<the original url>)."
             except Exception as e:  # noqa: BLE001
                 return f"Error: unzip failed: {type(e).__name__}: {e}"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.160",
+  "version": "1.0.164",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",