@tiens.nguyen/gonext-local-worker 1.0.153 → 1.0.157

package/gonext-local-worker.mjs

@@ -143,6 +143,8 @@ const OCR_CORRECT_TIMEOUT_MS = 120_000;
 // cold load of a big/multimodal model into VRAM can take ~20s+; the request isn't
 // awaited, so a long ceiling just avoids aborting an in-progress load.
 const AGENT_WARMUP_TIMEOUT_MS = 180_000;
+// How often the background keep-warm re-asserts the coding model's keep_alive:-1.
+const AGENT_WARMUP_INTERVAL_MS = 240_000; // 4 min — under Ollama's 5-min default unload
 // OCR translation/correction is hardcoded to a managed Ollama service + model.
 // (The user-configurable "OCR translation model" chooser was removed.)
 const OCR_TRANSLATE_OLLAMA_URL = "https://ollama1.gomarsic.cc";
@@ -1436,33 +1438,37 @@ async function runTranscribeJob(job) {
   }
 }
 
-/**
- * Fire-and-forget pre-load of the agent's Ollama coding model. Ollama evicts idle
- * models (default OLLAMA_KEEP_ALIVE=5m), so the first agent step otherwise blocks
- * ~20s+ while llama-server spins up and pages the weights into VRAM. The agent
- * talks to Ollama over the OpenAI-compat /v1 endpoint, which IGNORES keep_alive —
- * so we hit the NATIVE /api/generate here (no prompt = load-only, returns as soon
- * as the model is resident) with keep_alive:-1 to trigger the load AND pin it in
- * memory. Kicked off the moment the job is claimed so the load overlaps python
- * startup + prompt assembly. Best-effort: a non-Ollama coding server just 404s and
- * the failure is swallowed — the job proceeds regardless.
- */
-function warmUpOllamaCodingModel(codingBaseURL, codingModelId) {
-  if (!codingBaseURL || !codingModelId) return;
-  // codingBaseURL is normalized to …/v1 for OpenAI-compat; strip it to reach the
-  // Ollama root where /api/generate lives.
-  const root = String(codingBaseURL).replace(/\/+$/, "").replace(/\/v1$/i, "");
+// —— Ollama coding-model keep-warm ——————————————————————————————————————————————
+// Ollama evicts idle models (default OLLAMA_KEEP_ALIVE=5m), so a cold agent step
+// blocks ~20s+ while llama-server pages the weights into VRAM. Instead of warming on
+// EVERY question (noisy, redundant), we remember the coding model from agent jobs and
+// re-assert a keep_alive:-1 load on a BACKGROUND INTERVAL so it stays resident between
+// questions. The agent talks to Ollama over /v1 (which ignores keep_alive), so we hit
+// the NATIVE /api/generate (no prompt = load-only, returns once the model is resident).
+// Best-effort throughout: a non-Ollama coding server just 404s and it's swallowed.
+let _warmCoding = null; // { root, model }
+let _warmTimer = null;
+let _warmLoggedOk = false;
+
+function _warmupPing(quiet) {
+  if (!_warmCoding) return;
+  const { root, model } = _warmCoding;
   const url = `${root}/api/generate`;
   fetch(url, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ model: codingModelId, keep_alive: -1 }),
+    body: JSON.stringify({ model, keep_alive: -1 }),
     signal: AbortSignal.timeout(AGENT_WARMUP_TIMEOUT_MS),
   })
     .then((res) => {
-      console.log(
-        `[gonext-worker] agent warm-up ${res.ok ? "ok (model loading/resident)" : `HTTP ${res.status}`} → ${url} model=${codingModelId}`
-      );
+      if (!res.ok) {
+        console.log(`[gonext-worker] agent warm-up HTTP ${res.status} → ${url} model=${model}`);
+      } else if (!quiet || !_warmLoggedOk) {
+        // Quiet interval pings only log the first success per model; a fresh/changed
+        // model always logs so you can see it warm.
+        console.log(`[gonext-worker] agent warm-up ok (model loading/resident) → ${url} model=${model}`);
+        _warmLoggedOk = true;
+      }
     })
     .catch((err) => {
       const reason =
@@ -1471,12 +1477,37 @@ function warmUpOllamaCodingModel(codingBaseURL, codingModelId) {
     });
 }
 
+/**
+ * Remember the agent's Ollama coding model (from a claimed agent_chat job) and keep it
+ * resident via a background interval. Warms IMMEDIATELY only when the model is first
+ * seen or CHANGES — not on every question — so repeat questions don't re-warm/re-log.
+ */
+function trackCodingModelForWarmup(codingBaseURL, codingModelId) {
+  if (!codingBaseURL || !codingModelId) return;
+  // codingBaseURL is normalized to …/v1 for OpenAI-compat; strip it to reach the
+  // Ollama root where /api/generate lives.
+  const root = String(codingBaseURL).replace(/\/+$/, "").replace(/\/v1$/i, "");
+  const changed = !_warmCoding || _warmCoding.root !== root || _warmCoding.model !== codingModelId;
+  _warmCoding = { root, model: codingModelId };
+  if (changed) {
+    _warmLoggedOk = false;
+    _warmupPing(false); // warm the newly-seen/changed model now so the next question isn't cold
+  }
+  if (!_warmTimer) {
+    // 4 min — under Ollama's 5-min default unload, so the model stays resident even
+    // if a proxy strips keep_alive. unref() so this timer never blocks worker exit.
+    _warmTimer = setInterval(() => _warmupPing(true), AGENT_WARMUP_INTERVAL_MS);
+    if (typeof _warmTimer.unref === "function") _warmTimer.unref();
+  }
+}
+
 async function runAgentChatJob(job) {
   const { jobId, payload } = job;
   const start = Date.now();
-  // Kick the Ollama coding model's cold load off immediately (non-blocking) so it
-  // overlaps prompt assembly instead of stalling the first step. See helper above.
-  warmUpOllamaCodingModel(payload?.codingBaseURL ?? "", payload?.codingModelId ?? "");
+  // Remember this job's Ollama coding model and keep it resident via a background
+  // interval. Only warms now if the model is new/changed — repeat questions rely on
+  // the interval, so we don't re-warm (or re-log) on every question.
+  trackCodingModelForWarmup(payload?.codingBaseURL ?? "", payload?.codingModelId ?? "");
   const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
     method: "PATCH",
     body: JSON.stringify({ jobStatus: "running" }),
@@ -1570,6 +1601,16 @@ async function runAgentChatJob(job) {
       emailBodyTemplate: payload?.emailBodyTemplate ?? "",
       emailFrom: payload?.emailFrom ?? "",
       emailAllowList: payload?.emailAllowList ?? "",
+      // RAG tools (download/unzip/index/search over a user's ZIP-at-URL). Only enabled
+      // + registered when ragEnabled + AWS creds present. The worker talks to the user's
+      // OWN S3 bucket directly with these creds (boto3), NOT via presigned URLs.
+      ragEnabled: payload?.ragEnabled ?? false,
+      ragS3Location: payload?.ragS3Location ?? "",
+      ragAwsRegion: payload?.ragAwsRegion ?? "",
+      ragAwsAccessKeyId: payload?.ragAwsAccessKeyId ?? "",
+      ragAwsSecretAccessKey: payload?.ragAwsSecretAccessKey ?? "",
+      ragEmbedModel: payload?.ragEmbedModel ?? "",
+      ragTopK: payload?.ragTopK ?? 6,
     });
     // 30 min max for an agent run: multi-step ReAct on a 14B/31B with a cold prompt
     // cache — or a remote Ollama box on slow GPU cold-loading a big model — can run

package/gonext_agent_chat.py

@@ -518,7 +518,7 @@ _AGENT_KEYWORDS = re.compile(
     r"|api|endpoint|url|http|https"
     r"|external\s+source|external\s+api|external\s+service"
     r"|web\s+service|rest\s+api|rest\s+call"
-    r"|download|scrape|crawl"
+    r"|download|scrape|crawl|zip|unzip|\.zip|rag|index\s+the|knowledge\s+base|summari[sz]e"
     r"|search|find|look\s*up|lookup|weather|news|latest|current|today|tonight"
     r"|date|time|what\s+day|what\s+time"
     r"|read\s+(?:this\s+|that\s+|the\s+)?(?:page|article|url|link)|open\s+(?:this\s+|that\s+|the\s+)?(?:url|link|page)|contents?\s+of"
@@ -1080,6 +1080,270 @@ def _derive_pdf_title(doc_text: str) -> str:
     return "Document"
 
 
+# ============================ RAG (knowledge base) ============================
+# The agent can download a ZIP-at-URL, unzip it locally, index the text files into a
+# flat JSON vector store on the USER'S OWN S3 bucket (explicit AWS creds — NOT presigned),
+# then retrieve relevant chunks to answer questions. Embeddings run on the Ollama coding
+# server (/api/embed). Everything below is best-effort with hard safety caps.
+
+_RAG_TEXT_EXTS = {
+    ".cs", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".vue", ".svelte",
+    ".json", ".jsonc", ".md", ".mdx", ".txt", ".rst", ".py", ".rb", ".go",
+    ".rs", ".java", ".kt", ".swift", ".php", ".c", ".h", ".cpp", ".hpp", ".cc",
+    ".html", ".htm", ".css", ".scss", ".sass", ".less", ".xml", ".yaml", ".yml",
+    ".toml", ".ini", ".cfg", ".conf", ".env", ".sh", ".bash", ".zsh", ".sql",
+    ".graphql", ".proto", ".gradle", ".dockerfile", ".tsv", ".csv",
+}
+_RAG_SKIP_DIRS = {
+    "node_modules", ".git", ".svn", ".hg", "__pycache__", ".venv", "venv",
+    "dist", "build", ".next", ".nuxt", "out", ".cache", "bin", "obj", ".idea",
+    ".gradle", "vendor", "target",
+}
+_RAG_MAX_DOWNLOAD_BYTES = 500 * 1024 * 1024     # 500 MB zip cap
+_RAG_MAX_UNZIP_BYTES = 2 * 1024 * 1024 * 1024   # 2 GB uncompressed (zip-bomb guard)
+_RAG_MAX_FILES = 20000
+_RAG_MAX_FILE_BYTES = 2 * 1024 * 1024           # skip individual text files > 2 MB
+_RAG_CHUNK_CHARS = 3200                          # ~800 tokens
+_RAG_CHUNK_OVERLAP = 400
+
+
+def _rag_source_key(url: str) -> str:
+    import hashlib
+    return hashlib.sha256((url or "").strip().encode("utf-8")).hexdigest()[:32]
+
+
+def _rag_gdrive_direct(url: str) -> str:
+    """Rewrite a Google Drive share link to its direct-download form."""
+    m = re.search(r"drive\.google\.com/file/d/([A-Za-z0-9_-]+)", url or "")
+    if m:
+        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
+    m = re.search(r"drive\.google\.com/open\?id=([A-Za-z0-9_-]+)", url or "")
+    if m:
+        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
+    return url
+
+
+def _rag_assert_safe_url(url: str) -> None:
+    """SSRF guard: only http/https, and refuse private/link-local/loopback hosts."""
+    import ipaddress
+    import socket
+    from urllib.parse import urlparse
+    p = urlparse(url or "")
+    if p.scheme not in ("http", "https"):
+        raise ValueError("Only http/https URLs are allowed.")
+    host = p.hostname or ""
+    if not host:
+        raise ValueError("URL has no host.")
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except Exception as e:  # noqa: BLE001
+        raise ValueError(f"Cannot resolve host {host}: {e}")
+    for info in infos:
+        ip = info[4][0]
+        try:
+            addr = ipaddress.ip_address(ip)
+        except ValueError:
+            continue
+        if (addr.is_private or addr.is_loopback or addr.is_link_local
+                or addr.is_reserved or addr.is_multicast):
+            raise ValueError(f"Refusing to fetch a private/internal address ({ip}).")
+
+
+def _rag_workdir(source_key: str) -> str:
+    import os
+    base = os.path.join(os.path.expanduser("~"), ".gonext", "rag-work", source_key)
+    os.makedirs(base, exist_ok=True)
+    return base
+
+
+def _rag_download(url: str, dest_path: str = "") -> str:
+    """Download a public URL to a local path with SSRF + size guards. Returns the path."""
+    import os
+    src = _rag_gdrive_direct((url or "").strip())
+    _rag_assert_safe_url(src)
+    key = _rag_source_key(url)
+    if not dest_path:
+        name = os.path.basename(src.split("?")[0]) or "download.bin"
+        if not name.lower().endswith(".zip") and "zip" in src.lower():
+            name += ".zip"
+        dest_path = os.path.join(_rag_workdir(key), name)
+    os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
+    req = urllib.request.Request(src, headers={"User-Agent": "gonext-rag/1.0"})
+    total = 0
+    with urllib.request.urlopen(req, timeout=120, context=_ssl_context()) as resp, \
+            open(dest_path, "wb") as out:
+        while True:
+            block = resp.read(1024 * 256)
+            if not block:
+                break
+            total += len(block)
+            if total > _RAG_MAX_DOWNLOAD_BYTES:
+                out.close()
+                os.remove(dest_path)
+                raise ValueError(
+                    f"Download exceeds the {_RAG_MAX_DOWNLOAD_BYTES // (1024*1024)} MB cap."
+                )
+            out.write(block)
+    return dest_path
+
+
+def _rag_unzip(zip_path: str, dest_dir: str = "") -> tuple:
+    """Extract a local zip with Zip-Slip + zip-bomb guards. Returns (dir, file_list)."""
+    import os
+    import zipfile
+    if not dest_dir:
+        dest_dir = os.path.splitext(zip_path)[0] + "_unzipped"
+    os.makedirs(dest_dir, exist_ok=True)
+    dest_root = os.path.realpath(dest_dir)
+    written = 0
+    names = []
+    with zipfile.ZipFile(zip_path) as zf:
+        infos = zf.infolist()
+        if len(infos) > _RAG_MAX_FILES:
+            raise ValueError(f"Zip has too many entries (> {_RAG_MAX_FILES}).")
+        total_uncompressed = sum(i.file_size for i in infos)
+        if total_uncompressed > _RAG_MAX_UNZIP_BYTES:
+            raise ValueError("Zip uncompressed size exceeds the safety cap (possible zip bomb).")
+        for info in infos:
+            if info.is_dir():
+                continue
+            target = os.path.realpath(os.path.join(dest_dir, info.filename))
+            # Zip Slip: extracted path must stay within dest_dir.
+            if not (target == dest_root or target.startswith(dest_root + os.sep)):
+                continue
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            with zf.open(info) as src, open(target, "wb") as dst:
+                dst.write(src.read())
+            written += 1
+            names.append(info.filename)
+    return dest_dir, names
+
+
+def _rag_iter_text_files(root: str):
+    """Yield (abs_path, rel_path, ext) for indexable text files under root."""
+    import os
+    if os.path.isfile(root):
+        yield root, os.path.basename(root), os.path.splitext(root)[1].lower()
+        return
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in _RAG_SKIP_DIRS]
+        for fn in filenames:
+            ext = os.path.splitext(fn)[1].lower()
+            base = fn.lower()
+            if ext not in _RAG_TEXT_EXTS and base not in ("dockerfile", "makefile", ".gitignore"):
+                continue
+            abs_path = os.path.join(dirpath, fn)
+            try:
+                if os.path.getsize(abs_path) > _RAG_MAX_FILE_BYTES:
+                    continue
+            except OSError:
+                continue
+            yield abs_path, os.path.relpath(abs_path, root), ext
+
+
+def _rag_chunk_text(text: str, rel_path: str, ext: str):
+    """Line-aware windowed chunks with overlap. Returns list of {file,ext,start,end,text}."""
+    if not text.strip():
+        return []
+    chunks = []
+    n = len(text)
+    step = max(1, _RAG_CHUNK_CHARS - _RAG_CHUNK_OVERLAP)
+    i = 0
+    while i < n:
+        end = min(n, i + _RAG_CHUNK_CHARS)
+        # Prefer to cut on a newline near the window end.
+        if end < n:
+            nl = text.rfind("\n", i + step, end)
+            if nl > i:
+                end = nl + 1
+        piece = text[i:end].strip()
+        if piece:
+            chunks.append({"file": rel_path, "ext": ext, "start": i, "end": end, "text": piece})
+        if end >= n:
+            break
+        i = end - _RAG_CHUNK_OVERLAP if end - _RAG_CHUNK_OVERLAP > i else end
+    return chunks
+
+
+def _ollama_embed(ollama_root: str, model: str, texts: list) -> list:
+    """Embed a batch of texts via Ollama /api/embed (falls back to /api/embeddings)."""
+    root = (ollama_root or "").rstrip("/")
+    if not root:
+        raise RuntimeError("No Ollama endpoint available for embeddings.")
+    ctx = _ssl_context()
+    # Preferred: batch /api/embed (recent Ollama).
+    try:
+        req = urllib.request.Request(
+            f"{root}/api/embed",
+            data=json.dumps({"model": model, "input": texts}).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=300, context=ctx) as resp:
+            data = json.loads(resp.read().decode("utf-8", "replace"))
+        embs = data.get("embeddings")
+        if isinstance(embs, list) and embs and isinstance(embs[0], list):
+            return embs
+    except urllib.error.HTTPError as e:
+        if e.code != 404:
+            detail = e.read().decode("utf-8", "replace")[:200]
+            raise RuntimeError(f"Ollama embed failed (HTTP {e.code}): {detail}")
+    except urllib.error.URLError as e:
+        raise RuntimeError(f"Ollama embed unreachable: {getattr(e, 'reason', e)}")
+    # Fallback: per-text /api/embeddings (older Ollama).
+    out = []
+    for t in texts:
+        req = urllib.request.Request(
+            f"{root}/api/embeddings",
+            data=json.dumps({"model": model, "prompt": t}).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=120, context=ctx) as resp:
+            data = json.loads(resp.read().decode("utf-8", "replace"))
+        out.append(data.get("embedding") or [])
+    return out
+
+
+def _cosine(a: list, b: list) -> float:
+    import math
+    if not a or not b or len(a) != len(b):
+        return -1.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(y * y for y in b))
+    if na == 0 or nb == 0:
+        return -1.0
+    return dot / (na * nb)
+
+
+def _rag_parse_location(loc: str):
+    """Parse an ARN / s3:// URI / bucket[/prefix] into (bucket, prefix)."""
+    s = (loc or "").strip() or "arn:aws:s3:::gonext-rag"
+    rest = s
+    m = re.match(r"^arn:aws:s3:::(.+)$", s, re.I)
+    if m:
+        rest = m.group(1)
+    else:
+        m = re.match(r"^s3://(.+)$", s, re.I)
+        if m:
+            rest = m.group(1)
+    rest = rest.strip("/")
+    if "/" in rest:
+        bucket, prefix = rest.split("/", 1)
+    else:
+        bucket, prefix = rest, ""
+    return bucket, prefix.strip("/")
+
+
+def _rag_s3_client(region: str, akid: str, secret: str):
+    import boto3  # noqa: PLC0415
+    return boto3.client(
+        "s3", region_name=region or "ap-southeast-1",
+        aws_access_key_id=akid, aws_secret_access_key=secret,
+    )
+
+
 def run_agent_chat(cfg):
     try:
         from smolagents import CodeAgent, OpenAIServerModel, tool
@@ -1238,6 +1502,66 @@ def run_agent_chat(cfg):
     # action must never be silently available (same gating as the PDF reader).
     _EMAIL_AVAILABLE = bool(email_enabled and email_api_url and email_body_template)
 
+    # ---- RAG config (download/unzip/index/search over a user's ZIP-at-URL) ----
+    rag_enabled = bool(cfg.get("ragEnabled"))
+    rag_location = (cfg.get("ragS3Location") or "arn:aws:s3:::gonext-rag").strip()
+    rag_region = (cfg.get("ragAwsRegion") or "ap-southeast-1").strip()
+    rag_akid = (cfg.get("ragAwsAccessKeyId") or "").strip()
+    rag_secret = (cfg.get("ragAwsSecretAccessKey") or "").strip()
+    rag_embed_model = (cfg.get("ragEmbedModel") or "nomic-embed-text").strip()
+    try:
+        rag_top_k = int(cfg.get("ragTopK") or 6)
+    except (TypeError, ValueError):
+        rag_top_k = 6
+    rag_top_k = max(1, min(50, rag_top_k))
+    # Embeddings run on the Ollama coding server (same box as the coder). Strip the
+    # OpenAI-compat /v1 suffix to reach the native /api/embed root.
+    rag_ollama_root = re.sub(r"/v1/?$", "", (coding_base_url or agent_base_url or "").rstrip("/"))
+    try:
+        import boto3 as _boto3_probe  # noqa: F401,PLC0415
+        _boto3_ok = True
+    except Exception:  # noqa: BLE001
+        _boto3_ok = False
+    _RAG_AVAILABLE = bool(rag_enabled and rag_akid and rag_secret and rag_ollama_root and _boto3_ok)
+    if rag_enabled and not _boto3_ok:
+        _log("RAG requested but boto3 is not installed in the worker python — RAG tools disabled")
+
+    def _rag_s3_and_loc():
+        client = _rag_s3_client(rag_region, rag_akid, rag_secret)
+        bucket, prefix = _rag_parse_location(rag_location)
+        return client, bucket, prefix
+
+    def _rag_prefix_for(source_url: str, prefix: str) -> str:
+        key = _rag_source_key(source_url)
+        base = (prefix + "/") if prefix else ""
+        return f"{base}agent-rag/{key}"
+
+    def _rag_load_chunks(client, bucket: str, base_prefix: str) -> list:
+        """Load all chunk records (across shards) for a source. Returns [] if none."""
+        out = []
+        token = None
+        while True:
+            kw = {"Bucket": bucket, "Prefix": f"{base_prefix}/chunks"}
+            if token:
+                kw["ContinuationToken"] = token
+            resp = client.list_objects_v2(**kw)
+            for obj in resp.get("Contents", []) or []:
+                if not obj["Key"].endswith(".jsonl"):
+                    continue
+                body = client.get_object(Bucket=bucket, Key=obj["Key"])["Body"].read()
+                for line in body.decode("utf-8", "replace").splitlines():
+                    line = line.strip()
+                    if line:
+                        try:
+                            out.append(json.loads(line))
+                        except json.JSONDecodeError:
+                            pass
+            if resp.get("IsTruncated"):
+                token = resp.get("NextContinuationToken")
+            else:
+                break
+        return out
+
     def _prior_email_preview() -> bool:
         """True if an earlier assistant turn showed a send_email preview awaiting confirm."""
         for m in reversed(messages[:last_user_idx]):
@@ -1384,7 +1708,25 @@ def run_agent_chat(cfg):
     if _EMAIL_AVAILABLE:
         _tool_num += 1
         _email_num = _tool_num
+    if _RAG_AVAILABLE:
+        _tool_num += 5  # download_file, unzip_file, rag_index, rag_add, rag_search
     _tool_count_word = {6: "SIX", 7: "SEVEN", 8: "EIGHT"}.get(_tool_num, str(_tool_num))
+    _rag_tool_block = (
+        "  KNOWLEDGE BASE (RAG) — to SUMMARIZE or ANSWER questions about a ZIP of files at a URL:\n"
+        "    - download_file(url) — download a file (e.g. a .zip) to this machine.\n"
+        "    - unzip_file(zip_path) — extract a downloaded .zip locally.\n"
+        "    - rag_index(path, source_url) — index the unzipped text/code files into the knowledge "
+        "base (embeds to S3). source_url MUST be the ORIGINAL url the user gave.\n"
+        "    - rag_add(source_url, text) — add extra info the user provides to that knowledge base.\n"
+        "    - rag_search(source_url, query) — retrieve the most relevant chunks to answer a question.\n"
+    ) if _RAG_AVAILABLE else ""
+    _rag_choose_line = (
+        "- user gives a URL to a ZIP of files and asks to summarize/analyze/RAG them -> "
+        "download_file(url) THEN unzip_file(zip) THEN rag_index(dir, source_url=url) THEN "
+        "rag_search(source_url=url, query=<the user's question>), THEN answer from the results. "
+        "For FOLLOW-UP questions about the SAME url, just rag_search (do NOT re-download). "
+        "If the user adds information, rag_add(source_url=url, text=...).\n"
+    ) if _RAG_AVAILABLE else ""
     _pdf_read_tool_line = (
         f"  {_pdf_num}. extract_text_from_pdf(url) — READ/summarize an EXISTING PDF file "
         "at a URL. This does NOT create a PDF (use create_pdf for that).\n"
@@ -1421,7 +1763,7 @@ def run_agent_chat(cfg):
         "Use this ONLY when the task explicitly asks for the date or time.\n"
         "  6. create_pdf(text, title='') — make a PDF document from text/data and return a "
         "download link. Use ONLY when the task asks to create/make/generate/export a PDF.\n"
-        + _pdf_read_tool_line + _email_tool_line +
+        + _pdf_read_tool_line + _email_tool_line + _rag_tool_block +
         "\n"
         "http_request RETURN FORMAT: 'HTTP 200\\n{body}' — first line is 'HTTP <code>', body follows.\n"
         "\n"
@@ -1445,7 +1787,7 @@ def run_agent_chat(cfg):
         "- a PDF of info you must LOOK UP first (e.g. 'get the world cup schedule then make a "
         "pdf') -> FIRST web_search/fetch_url to gather the real facts, THEN in a later step "
         "create_pdf(text=<the gathered facts>, title=...). Never PDF the request wording itself.\n"
-        + _pdf_read_choose_line + _email_choose_line +
+        + _pdf_read_choose_line + _email_choose_line + _rag_choose_line +
         "- A specific known API/URL was given -> http_request().\n"
         "\n"
         "RULES:\n"
@@ -2275,14 +2617,160 @@ def run_agent_chat(cfg):
             client_kwargs={"timeout": 600.0, "max_retries": 0},
             **extra_model_kwargs,
         )
+        @tool
+        def download_file(url: str, dest_path: str = "") -> str:
+            """Download a file (e.g. a .zip) from a public http(s) URL to a local path on this machine. Use this FIRST when the user gives a URL to a zip of files; then unzip_file, then rag_index.
+
+            Args:
+                url: public http(s) URL of the file to download.
+                dest_path: optional local save path; leave empty for an automatic work-dir path.
+            """
+            _emit({"type": "step", "text": f"Downloading → {url[:80]}"})
+            try:
+                import os
+                path = _rag_download(url, (dest_path or "").strip())
+                return f"Downloaded to {path} ({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}') if it is a zip."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: download failed: {type(e).__name__}: {e}"
+
+        @tool
+        def unzip_file(zip_path: str, dest_dir: str = "") -> str:
+            """Extract a local .zip file into a local directory. Use after download_file. Returns the directory and a short file listing.
+
+            Args:
+                zip_path: local path to the .zip (from download_file).
+                dest_dir: optional target directory; leave empty for an automatic one.
+            """
+            _emit({"type": "step", "text": "Unzipping…"})
+            try:
+                d, names = _rag_unzip((zip_path or "").strip(), (dest_dir or "").strip())
+                preview = ", ".join(names[:15])
+                more = f" …(+{len(names) - 15} more)" if len(names) > 15 else ""
+                return f"Unzipped {len(names)} files to {d}. Files: {preview}{more}. Next: rag_index(path='{d}', source_url=<the original url>)."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: unzip failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_index(path: str, source_url: str) -> str:
+            """Index a local file or directory of TEXT/code files into the knowledge base for source_url (stores embeddings on S3). Use after unzip_file so the files become searchable. source_url MUST be the original URL the user provided.
+
+            Args:
+                path: local file or directory to index (from unzip_file).
+                source_url: the original URL the user gave — used as the knowledge-base key.
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured (enable it + add AWS credentials in Settings)."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": "Indexing files for RAG…"})
+            try:
+                import time as _t
+                records = []
+                files = 0
+                for abs_path, rel, ext in _rag_iter_text_files((path or "").strip()):
+                    try:
+                        with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
+                            text = fh.read()
+                    except OSError:
+                        continue
+                    files += 1
+                    records.extend(_rag_chunk_text(text, rel, ext))
+                if not records:
+                    return "No indexable text files found at that path."
+                for i in range(0, len(records), 64):
+                    batch = records[i:i + 64]
+                    vecs = _ollama_embed(rag_ollama_root, rag_embed_model, [r["text"] for r in batch])
+                    for r, v in zip(batch, vecs):
+                        r["embedding"] = v
+                    _emit({"type": "step", "text": f"Embedded {min(i + 64, len(records))}/{len(records)} chunks…"})
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                shard = f"{base}/chunks-{int(_t.time())}.jsonl"
+                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
+                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                manifest = {
+                    "sourceUrl": source_url, "embedModel": rag_embed_model,
+                    "dims": len(records[0].get("embedding") or []),
+                    "files": files, "chunks": len(records),
+                    "updatedAt": _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()),
+                }
+                client.put_object(Bucket=bucket, Key=f"{base}/manifest.json",
+                                  Body=json.dumps(manifest).encode("utf-8"),
+                                  ContentType="application/json")
+                return (f"Indexed {files} files into {len(records)} chunks for {source_url}. "
+                        f"Use rag_search(source_url='{source_url}', query=...) to retrieve, then answer the user.")
+            except Exception as e:  # noqa: BLE001
+                return f"Error: indexing failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_add(source_url: str, text: str) -> str:
+            """Add extra text/notes to an EXISTING knowledge base (keyed by source_url). Use when the user gives more information to remember for later questions.
+
+            Args:
+                source_url: the knowledge-base key (the original URL).
+                text: the new information to embed and store.
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured."
+            if not (text or "").strip():
+                return "Error: no text to add."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": "Adding info to RAG…"})
+            try:
+                import time as _t
+                records = _rag_chunk_text(text, "user-note", ".txt")
+                vecs = _ollama_embed(rag_ollama_root, rag_embed_model, [r["text"] for r in records])
+                for r, v in zip(records, vecs):
+                    r["embedding"] = v
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                shard = f"{base}/chunks-add-{int(_t.time())}.jsonl"
+                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
+                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                return f"Added {len(records)} chunk(s) to the knowledge base for {source_url}."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: rag_add failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_search(source_url: str, query: str, k: int = 0) -> str:
+            """Retrieve the most relevant chunks from the knowledge base for source_url. Use to ANSWER questions about the indexed files; write your answer from the returned chunks.
+
+            Args:
+                source_url: the knowledge-base key (the original URL).
+                query: what to look for.
+                k: number of chunks to return (0 = configured default).
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": f"Searching RAG → {query[:60]}"})
+            try:
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                chunks = _rag_load_chunks(client, bucket, base)
+                if not chunks:
+                    return f"No knowledge base found for {source_url}. Run rag_index first."
+                qvec = _ollama_embed(rag_ollama_root, rag_embed_model, [query])[0]
+                scored = sorted(
+                    ((_cosine(qvec, c.get("embedding") or []), c) for c in chunks),
+                    key=lambda x: x[0], reverse=True,
+                )
+                topk = scored[: (k if k and k > 0 else rag_top_k)]
+                parts = [f"[{c.get('file', '?')}] (score {score:.2f})\n{c.get('text', '')}"
+                         for score, c in topk]
+                return "\n\n---\n\n".join(parts) if parts else "No matches."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: rag_search failed: {type(e).__name__}: {e}"
+
         agent_tools = [http_request, web_search, fetch_url, calculate,
                        get_current_datetime, create_pdf]
-        # Only register the PDF reader / email tools when available (kept in sync with
-        # the gated hint entries above so the model never sees a tool it can't run).
+        # Only register the PDF reader / email / RAG tools when available (kept in sync
+        # with the gated hint entries above so the model never sees a tool it can't run).
         if _PDF_READ_AVAILABLE:
             agent_tools.append(extract_text_from_pdf)
         if _EMAIL_AVAILABLE:
             agent_tools.append(send_email)
+        if _RAG_AVAILABLE:
+            agent_tools += [download_file, unzip_file, rag_index, rag_add, rag_search]
         agent_kwargs = dict(
             tools=agent_tools,
             model=model,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.153",
+  "version": "1.0.157",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",