@tiens.nguyen/gu-cli 1.0.798 → 1.0.799

package/cli-token.mjs

@@ -170,11 +170,44 @@ export async function claimTrial({ hostedBase, device, fetchImpl = fetch, timeou
     return typeof data?.token === "string"
       ? { ok: true, token: data.token, endsAt: data.endsAt ?? null, created: data.created === true }
       : { ok: false, why: "bad-response" };
-  } catch {
-    return { ok: false, why: "unreachable" };
+  } catch (err) {
+    // #17: say WHY it failed (DNS, timeout, refused, TLS…) — a bare "can't be reached" hid a
+    // fresh Ubuntu server's real problem.
+    return { ok: false, why: "unreachable", detail: netErrorDetail(err, timeoutMs) };
   }
 }
 
+/** A short, human cause for a failed fetch (#17): "DNS lookup failed (ENOTFOUND)", "no answer in 8s"… */
+export function netErrorDetail(err, timeoutMs = 8000) {
+  const name = String(err?.name ?? "");
+  if (name === "TimeoutError" || name === "AbortError") return `no answer in ${Math.round(timeoutMs / 1000)}s`;
+  const code = String(err?.cause?.code ?? err?.code ?? "");
+  const known = {
+    ENOTFOUND: "DNS lookup failed",
+    EAI_AGAIN: "DNS lookup failed (temporary)",
+    ECONNREFUSED: "connection refused",
+    ECONNRESET: "connection reset",
+    ETIMEDOUT: "connection timed out",
+    EHOSTUNREACH: "host unreachable",
+    ENETUNREACH: "network unreachable",
+    CERT_HAS_EXPIRED: "TLS certificate expired",
+    UNABLE_TO_VERIFY_LEAF_SIGNATURE: "TLS certificate could not be verified",
+    UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "TLS certificate could not be verified (missing CA certificates?)",
+    SELF_SIGNED_CERT_IN_CHAIN: "TLS: self-signed certificate in chain (proxy?)",
+    DEPTH_ZERO_SELF_SIGNED_CERT: "TLS: self-signed certificate (proxy?)",
+    ERR_TLS_CERT_ALTNAME_INVALID: "TLS certificate does not match the host name",
+  };
+  if (code && known[code]) return `${known[code]} (${code})`;
+  if (code) return code;
+  const msg = String(err?.cause?.message ?? err?.message ?? err ?? "").trim();
+  return msg ? msg.slice(0, 160) : "network error";
+}
+
+/** " (at host)" for the messages above (#17), or "" when there is no base. */
+function trialHost(base) {
+  try { return base ? ` (at ${new URL(String(base)).host})` : ""; } catch { return ""; }
+}
+
 /** Where a person who thinks a lock is wrong can write (#315 P7) — the address the website shows. */
 export const SUPPORT_EMAIL = "support@getmarsic.cc";
 
@@ -256,7 +289,15 @@ export async function promptForToken({
       ? [`This is a free trial token: ${daysLeft(trial.endsAt)} days left on this computer, until ${shortDate(trial.endsAt)}.`, getOne, "Enter uses it — or delete it (one Backspace) and paste your own token."]
       : [
           ...(trial.why === "used" && trial.endedAt ? [`The free trial on this computer ended on ${shortDate(trial.endedAt)}.`] : []),
-          ...(trial.why === "unreachable" ? ["The gu service can't be reached right now — Enter on an empty line tries again."] : []),
+          ...(trial.why === "unreachable"
+            ? [`The gu service can't be reached right now${trialHost(hostedBase)}${trial.detail ? `: ${trial.detail}` : ""} — Enter on an empty line tries again.`]
+            : []),
+          ...(/^http-\d+$/.test(String(trial.why ?? ""))
+            ? [`The gu service answered ${trial.why.replace("http-", "HTTP ")}${trialHost(hostedBase)} — no free trial from it right now. Enter on an empty line tries again.`]
+            : []),
+          ...(trial.why === "bad-response"
+            ? [`The gu service${trialHost(hostedBase)} sent an answer without a token — Enter on an empty line tries again.`]
+            : []),
           getOne,
           required ? "Paste it here, then Enter." : "Paste it here — or Enter to skip for now.",
         ];

package/gonext_agent_chat.py

@@ -28,6 +28,7 @@ import json
 import os
 import random
 import re
+import socket
 from urllib.parse import urlparse
 import sys
 import threading
@@ -2615,6 +2616,69 @@ def _search_tally_bump(kind: str, found: int = 0, query: str = "") -> None:
         _SEARCH_TALLY["grep_hits"] += max(0, int(found or 0))
 
 
+# RAG FALLBACK (lexical). When the embedder is down — or indexing fails, or the query cannot be
+# embedded — a semantic search must not just return an error the model has to work around. The
+# user asked for exactly this: warn on the bar, but still ANSWER the search, by grep. A
+# plain-language question is turned into keywords, every text file is scanned for them, and
+# files are ranked by how many DIFFERENT keywords they contain (then by hit count), which is a
+# fair stand-in for "relevant" when no vectors are available.
+_FALLBACK_STOPWORDS = frozenset("""
+a an and are as at be been but by can could do does did for from has have how i if in into is it
+its me my no not of on or our so than that the their them then there these this those to too
+up us was we were what when where which while who why will with would you your about after all
+any also code file files find show tell get set use used using way work works handled happen
+happens project repo function method class does where's what's how's
+""".split())
+
+
+def _rag_fallback_terms(query: str, limit: int = 6) -> list:
+    """Keywords worth grepping for, from a plain-language query. Order kept, duplicates dropped."""
+    out, seen = [], set()
+    for tok in re.findall(r"[A-Za-z_][A-Za-z0-9_.$-]*", str(query or "")):
+        tok = tok.strip(".-$")
+        low = tok.lower()
+        if len(low) < 3 or low in _FALLBACK_STOPWORDS or low in seen:
+            continue
+        seen.add(low)
+        out.append(tok)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _lexical_fallback_search(query: str, files, max_files: int = 8, per_file: int = 3) -> dict:
+    """Rank files by how many distinct query keywords they contain. Never raises.
+
+    `files` yields (abs_path, rel_path). Returns {"terms", "results", "hits"} where results is a
+    list of {"file", "terms", "hits", "lines": [(lineno, text)]} best first.
+    """
+    terms = _rag_fallback_terms(query)
+    if not terms:
+        return {"terms": [], "results": [], "hits": 0}
+    rxs = [(t.lower(), re.compile(re.escape(t), re.IGNORECASE)) for t in terms]
+    scored = []
+    total = 0
+    for abs_path, _rel in files:
+        try:
+            with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
+                found, hits, lines = set(), 0, []
+                for i, line in enumerate(fh, 1):
+                    matched = [t for t, rx in rxs if rx.search(line)]
+                    if not matched:
+                        continue
+                    found.update(matched)
+                    hits += 1
+                    if len(lines) < per_file:
+                        lines.append((i, line.rstrip()[:200]))
+        except OSError:
+            continue
+        if found:
+            total += hits
+            scored.append({"file": abs_path, "terms": len(found), "hits": hits, "lines": lines})
+    scored.sort(key=lambda r: (-r["terms"], -r["hits"], r["file"]))
+    return {"terms": terms, "results": scored[:max_files], "hits": total}
+
+
 def _retrieval_summary(tally: dict, rag_state: str = "", note_unused: bool = True,
                        searches: list = None) -> str:
     """One line naming how this turn found code — "" when it never looked for any.
@@ -5297,7 +5361,17 @@ def _rag_base_dir(source_key: str) -> str:
     return _rag_workdir(source_key)
 
 
-def _rag_probe_embed(base: str, timeout: float = 1.5):
+def _probe_is_timeout(e: BaseException) -> bool:
+    """Did this probe failure come from the HOST not answering in time (vs. refused/DNS/etc.)?"""
+    if isinstance(e, (socket.timeout, TimeoutError)):
+        return True
+    reason = getattr(e, "reason", None)
+    if isinstance(reason, (socket.timeout, TimeoutError)):
+        return True
+    return "timed out" in str(e).lower()
+
+
+def _rag_probe_embed(base: str, timeout: float = 2.5, attempts: int = 2):
     """(reachable, detail) for an embeddings endpoint. Never raises.
 
     Config alone says nothing about whether embeddings will work: _embed() is only called
@@ -5309,6 +5383,24 @@ def _rag_probe_embed(base: str, timeout: float = 1.5):
     root = re.sub(r"/(v1|api/embeddings|api/embed|api)/?$", "", (base or "").rstrip("/")).rstrip("/")
     if not root:
         return False, "no URL configured"
+    # ONE BLIP IS NOT "DOWN". A single 1.5s attempt marked a healthy remote embedder as
+    # unreachable after one slow TLS handshake, and that verdict was then saved into the
+    # session and shown in red until the next turn. So: a slightly longer timeout, and a
+    # second pass before giving up. A timeout skips the remaining paths of that pass (the
+    # HOST is slow, not the path), which bounds a truly dead host to attempts*timeout.
+    last = "no response"
+    for attempt in range(max(1, attempts)):
+        if attempt:
+            time.sleep(0.3)
+        ok, detail, last = _rag_probe_embed_once(root, timeout)
+        if ok:
+            return True, detail
+    return False, last
+
+
+def _rag_probe_embed_once(root: str, timeout: float):
+    """One pass over the probe paths: (reachable, detail, last_error)."""
+    last = "no response"
     for path in ("/v1/models", "/api/tags"):  # OpenAI-compatible, then Ollama
         try:
             req = urllib.request.Request(f"{root}{path}", method="GET")
@@ -5318,13 +5410,16 @@ def _rag_probe_embed(base: str, timeout: float = 1.5):
             # CERTIFICATE_VERIFY_FAILED against an endpoint that was perfectly fine.
             with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
                 if 200 <= resp.status < 300:
-                    return True, ""
+                    return True, "", ""
+                last = f"HTTP {resp.status}"
         except urllib.error.HTTPError as e:
             # It ANSWERED — 401/403/404 still proves something is listening there.
-            return True, f"HTTP {e.code}"
+            return True, f"HTTP {e.code}", ""
         except Exception as e:  # noqa: BLE001
             last = f"{type(e).__name__}: {e}"
-    return False, last
+            if _probe_is_timeout(e):
+                break
+    return False, "", last
 
 
 def _same_embed_model(a: str, b: str) -> bool:
@@ -12284,6 +12379,45 @@ def run_agent_chat(cfg):
             _last_obs["text"] = out
             return out
 
+        def _rag_grep_fallback(query: str, why: str) -> str:
+            """Answer a semantic search lexically when RAG cannot, and SAY so (bar + terminal).
+
+            The state is left (or set) to a failure state so the bottom bar keeps its warning;
+            the model gets real file:line hits instead of an error to route around.
+            """
+            if _RAG_STATE.get("s") not in ("unreachable", "unavailable"):
+                _RAG_STATE["s"] = "fallback"
+            _emit({"type": "step",
+                   "text": f"⚠ RAG unavailable ({why}) → falling back to keyword search"})
+            _log(f"RAG fallback: {why} — answering {query[:80]!r} with a lexical search")
+            try:
+                root = _rag_read_allowed(_default_ws_root())
+                res = _lexical_fallback_search(
+                    query, ((a, r) for a, r, _e in _rag_iter_text_files(root)))
+            except Exception as e:  # noqa: BLE001
+                msg = (f"RAG is unavailable ({why}) and the keyword fallback failed: "
+                       f"{type(e).__name__}: {e}. Use grep_repo(pattern) / read_file_lines.")
+                _last_obs["text"] = msg
+                return msg
+            _search_tally_bump("grep", res["hits"], " | ".join(res["terms"]) or query)
+            head = (f"NOTE: semantic search is unavailable ({why}), so this is a KEYWORD search "
+                    f"for {', '.join(res['terms']) or '(no usable keywords)'} — results are "
+                    "lexical, not semantic.")
+            if not res["results"]:
+                msg = (head + f"\nNo file under {root} contains those keywords. Try grep_repo "
+                       "with a more specific identifier, or list_dir/read_file_lines.")
+                _last_obs["text"] = msg
+                return msg
+            parts = [head, f"Best matching files under {root} (by distinct keywords matched):"]
+            for r in res["results"]:
+                parts.append(f"\n{r['file']}  ({r['terms']}/{len(res['terms'])} keywords, "
+                             f"{r['hits']} line(s))")
+                parts.extend(f"  {r['file']}:{ln}: {txt}" for ln, txt in r["lines"])
+            parts.append("\nRead the relevant lines with read_file_lines before relying on them.")
+            out = _ws_cap_obs("\n".join(parts), cap=_OBS_CAPS["search"])
+            _last_obs["text"] = out
+            return out
+
         @tool
         def rag_search_workspace(query: str, k: int = 0) -> str:
             """Search the CURRENT PROJECT's knowledge base (built by rag_index_workspace) for code/text relevant to a question. Use for BROAD questions where you don't know the exact symbol to grep for — e.g. "how is auth handled?". For an exact name or string, grep_repo is faster and more precise.
@@ -12293,9 +12427,14 @@ def run_agent_chat(cfg):
                 k: number of chunks to return (0 = configured default).
             """
             if not _RAG_AVAILABLE:
-                msg = _rag_unavailable_msg()
-                _last_obs["text"] = msg
-                return msg
+                # Not configured / no embedder: still answer, lexically (bar keeps warning).
+                if rag_enabled:
+                    _RAG_STATE["s"] = "unavailable"
+                return _rag_grep_fallback(query, "RAG not available")
+            if _RAG_STATE.get("s") == "unreachable":
+                # The turn's probe already found the embedder down: indexing and query
+                # embedding would both fail (after timeouts). Go straight to the fallback.
+                return _rag_grep_fallback(query, "embedder unreachable")
             try:
                 root = _rag_read_allowed(_default_ws_root())
             except Exception as e:  # noqa: BLE001
@@ -12370,16 +12509,15 @@ def run_agent_chat(cfg):
                     _RAG_STATE["s"] = "no-index"
                     _emit({"type": "step",
                            "text": f"RAG indexing failed → {_rag_embed_failure_hint(_ie)}"})
-                    msg = (f"Could not build the knowledge base: {type(_ie).__name__}: {_ie}. "
-                           "Use grep_repo meanwhile.")
-                    _last_obs["text"] = msg
-                    return msg
+                    return _rag_grep_fallback(query, f"indexing failed: {type(_ie).__name__}")
                 if not _rag_load_all(root):
                     _RAG_STATE["s"] = "no-index"
-                    msg = "Indexing produced nothing to search — use grep_repo instead."
-                    _last_obs["text"] = msg
-                    return msg
-            return rag_search(root, query, k)
+                    return _rag_grep_fallback(query, "indexing produced nothing")
+            out = rag_search(root, query, k)
+            if isinstance(out, str) and out.startswith("Error: rag_search failed"):
+                # Typically the query could not be embedded (embedder went down mid-turn).
+                return _rag_grep_fallback(query, out[len("Error: rag_search failed: "):][:80])
+            return out
 
         @tool
         def grep_repo(pattern: str, path: str = "", glob: str = "") -> str:

package/gu-repl.mjs

@@ -2254,8 +2254,18 @@ const ragSegmentText = () => {
   //
   // UNREACHABLE IS THE EXCEPTION and stays visible: an embedder that is DOWN is a fault the user
   // can fix and would want to know about now, not a settled fact about the folder.
-  if (!REPL_VERBOSE && b.ragState !== "unreachable") return lexSeg;
-  return dim(" · ") + (b.ragState === "unreachable" ? red(label) : dim(label)) + lexSeg;
+  // …BUT ONLY WHEN IT IS THIS SESSION'S VERDICT. A breakdown reloaded from the saved session
+  // (stale: true) carries the LAST turn's probe result — possibly a one-off timeout hours ago.
+  // Showing that in red claimed "the embedder is down now" before anything had checked; it
+  // read as a live fault on a perfectly healthy server. Stale → dimmed and marked "·" like
+  // the carried numbers above, and hidden outside --verbose; the next turn re-probes.
+  const liveDown = b.ragState === "unreachable" && !b.stale;
+  // RAG FAILED MID-TURN and searches were answered by the grep fallback: a WARNING (yellow),
+  // always visible — the turn still got results, but not semantic ones.
+  const liveFallback = b.ragState === "fallback" && !b.stale;
+  if (!REPL_VERBOSE && !liveDown && !liveFallback) return lexSeg;
+  if (b.stale) return dim(" · ") + dim(`${label}·`) + lexSeg;
+  return dim(" · ") + (liveDown ? red(label) : liveFallback ? yellow(label) : dim(label)) + lexSeg;
 };
 /**
  * IS ANYTHING NEWER? (task #181) — cached, never probed from a repaint.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gu-cli",
-  "version": "1.0.798",
+  "version": "1.0.799",
   "description": "Gu CLI — the gu terminal plus the local worker that runs agent / OCR / PDF / embedding jobs on your Mac (Ollama / MLX / OpenAI-compatible).",
   "type": "module",
   "license": "MIT",

package/terminal-layout.mjs

@@ -87,13 +87,15 @@ export function clipVisible(s, max) {
 export function ragZeroReason(state) {
   switch (state) {
     case "unreachable":
-      return "the embedding server is DOWN — indexing and search would both fail";
+      return "the embedding server is DOWN — searches fall back to keyword (grep) search";
     case "no-index":
       return "nothing indexed for this project yet — run rag_index_workspace()";
     case "unavailable":
       return "RAG is not configured for this workspace";
     case "ready":
       return "indexed and reachable — the model chose not to search";
+    case "fallback":
+      return "RAG failed this turn — searches were answered by keyword (grep) fallback";
     default:
       return "the model never called rag_search this turn";
   }
@@ -106,7 +108,7 @@ export function ragSegmentLabel(b) {
   if (rag > 0) return null; // …the caller renders the number + percent instead
   switch (b.ragState) {
     case "unreachable":
-      return "rag ✗ embedder down";
+      return "rag ✗ embedder down → grep fallback";
     case "no-index":
       return "rag — not indexed";
     case "unavailable":
@@ -123,6 +125,10 @@ export function ragSegmentLabel(b) {
       return "rag — folder too large; register a project folder";
     case "indexing-later":
       return "rag — indexing after this turn";
+    // RAG FAILED MID-TURN (indexing or query embedding) and the search was answered by the
+    // keyword fallback instead. A warning, not an error: the turn still got results.
+    case "fallback":
+      return "rag ⚠ unavailable → grep fallback";
     // Nothing has been measured yet (the state before any turn has run). "0" is a claim we
     // cannot support; a dash says "no figure", which is the truth.
     case "unknown":