@tiens.nguyen/gonext-local-worker 1.0.155 → 1.0.158

package/gonext-local-worker.mjs

@@ -1601,6 +1601,17 @@ async function runAgentChatJob(job) {
       emailBodyTemplate: payload?.emailBodyTemplate ?? "",
       emailFrom: payload?.emailFrom ?? "",
       emailAllowList: payload?.emailAllowList ?? "",
+      // RAG tools (download/unzip/index/search over a user's ZIP-at-URL). Only enabled
+      // + registered when ragEnabled + AWS creds present. The worker talks to the user's
+      // OWN S3 bucket directly with these creds (boto3), NOT via presigned URLs.
+      ragEnabled: payload?.ragEnabled ?? false,
+      ragS3Location: payload?.ragS3Location ?? "",
+      ragAwsRegion: payload?.ragAwsRegion ?? "",
+      ragAwsAccessKeyId: payload?.ragAwsAccessKeyId ?? "",
+      ragAwsSecretAccessKey: payload?.ragAwsSecretAccessKey ?? "",
+      ragEmbedModel: payload?.ragEmbedModel ?? "",
+      ragEmbedUrl: payload?.ragEmbedUrl ?? "",
+      ragTopK: payload?.ragTopK ?? 6,
     });
     // 30 min max for an agent run: multi-step ReAct on a 14B/31B with a cold prompt
     // cache — or a remote Ollama box on slow GPU cold-loading a big model — can run

package/gonext_agent_chat.py

@@ -518,7 +518,7 @@ _AGENT_KEYWORDS = re.compile(
     r"|api|endpoint|url|http|https"
     r"|external\s+source|external\s+api|external\s+service"
     r"|web\s+service|rest\s+api|rest\s+call"
-    r"|download|scrape|crawl"
+    r"|download|scrape|crawl|zip|unzip|\.zip|rag|index\s+the|knowledge\s+base|summari[sz]e"
     r"|search|find|look\s*up|lookup|weather|news|latest|current|today|tonight"
     r"|date|time|what\s+day|what\s+time"
     r"|read\s+(?:this\s+|that\s+|the\s+)?(?:page|article|url|link)|open\s+(?:this\s+|that\s+|the\s+)?(?:url|link|page)|contents?\s+of"
@@ -1080,6 +1080,283 @@ def _derive_pdf_title(doc_text: str) -> str:
     return "Document"
 
 
+# ============================ RAG (knowledge base) ============================
+# The agent can download a ZIP-at-URL, unzip it locally, index the text files into a
+# flat JSON vector store on the USER'S OWN S3 bucket (explicit AWS creds — NOT presigned),
+# then retrieve relevant chunks to answer questions. Embeddings run on the Ollama coding
+# server (/api/embed). Everything below is best-effort with hard safety caps.
+
+_RAG_TEXT_EXTS = {
+    ".cs", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".vue", ".svelte",
+    ".json", ".jsonc", ".md", ".mdx", ".txt", ".rst", ".py", ".rb", ".go",
+    ".rs", ".java", ".kt", ".swift", ".php", ".c", ".h", ".cpp", ".hpp", ".cc",
+    ".html", ".htm", ".css", ".scss", ".sass", ".less", ".xml", ".yaml", ".yml",
+    ".toml", ".ini", ".cfg", ".conf", ".env", ".sh", ".bash", ".zsh", ".sql",
+    ".graphql", ".proto", ".gradle", ".dockerfile", ".tsv", ".csv",
+}
+_RAG_SKIP_DIRS = {
+    "node_modules", ".git", ".svn", ".hg", "__pycache__", ".venv", "venv",
+    "dist", "build", ".next", ".nuxt", "out", ".cache", "bin", "obj", ".idea",
+    ".gradle", "vendor", "target",
+}
+_RAG_MAX_DOWNLOAD_BYTES = 500 * 1024 * 1024     # 500 MB zip cap
+_RAG_MAX_UNZIP_BYTES = 2 * 1024 * 1024 * 1024   # 2 GB uncompressed (zip-bomb guard)
+_RAG_MAX_FILES = 20000
+_RAG_MAX_FILE_BYTES = 2 * 1024 * 1024           # skip individual text files > 2 MB
+_RAG_CHUNK_CHARS = 3200                          # ~800 tokens
+_RAG_CHUNK_OVERLAP = 400
+
+
+def _rag_source_key(url: str) -> str:
+    import hashlib
+    return hashlib.sha256((url or "").strip().encode("utf-8")).hexdigest()[:32]
+
+
+def _rag_gdrive_direct(url: str) -> str:
+    """Rewrite a Google Drive share link to its direct-download form."""
+    m = re.search(r"drive\.google\.com/file/d/([A-Za-z0-9_-]+)", url or "")
+    if m:
+        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
+    m = re.search(r"drive\.google\.com/open\?id=([A-Za-z0-9_-]+)", url or "")
+    if m:
+        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
+    return url
+
+
+def _rag_assert_safe_url(url: str) -> None:
+    """SSRF guard: only http/https, and refuse private/link-local/loopback hosts."""
+    import ipaddress
+    import socket
+    from urllib.parse import urlparse
+    p = urlparse(url or "")
+    if p.scheme not in ("http", "https"):
+        raise ValueError("Only http/https URLs are allowed.")
+    host = p.hostname or ""
+    if not host:
+        raise ValueError("URL has no host.")
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except Exception as e:  # noqa: BLE001
+        raise ValueError(f"Cannot resolve host {host}: {e}")
+    for info in infos:
+        ip = info[4][0]
+        try:
+            addr = ipaddress.ip_address(ip)
+        except ValueError:
+            continue
+        if (addr.is_private or addr.is_loopback or addr.is_link_local
+                or addr.is_reserved or addr.is_multicast):
+            raise ValueError(f"Refusing to fetch a private/internal address ({ip}).")
+
+
+def _rag_workdir(source_key: str) -> str:
+    import os
+    base = os.path.join(os.path.expanduser("~"), ".gonext", "rag-work", source_key)
+    os.makedirs(base, exist_ok=True)
+    return base
+
+
+def _rag_download(url: str, dest_path: str = "") -> str:
+    """Download a public URL to a local path with SSRF + size guards. Returns the path."""
+    import os
+    src = _rag_gdrive_direct((url or "").strip())
+    _rag_assert_safe_url(src)
+    key = _rag_source_key(url)
+    if not dest_path:
+        name = os.path.basename(src.split("?")[0]) or "download.bin"
+        if not name.lower().endswith(".zip") and "zip" in src.lower():
+            name += ".zip"
+        dest_path = os.path.join(_rag_workdir(key), name)
+    os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
+    req = urllib.request.Request(src, headers={"User-Agent": "gonext-rag/1.0"})
+    total = 0
+    with urllib.request.urlopen(req, timeout=120, context=_ssl_context()) as resp, \
+            open(dest_path, "wb") as out:
+        while True:
+            block = resp.read(1024 * 256)
+            if not block:
+                break
+            total += len(block)
+            if total > _RAG_MAX_DOWNLOAD_BYTES:
+                out.close()
+                os.remove(dest_path)
+                raise ValueError(
+                    f"Download exceeds the {_RAG_MAX_DOWNLOAD_BYTES // (1024*1024)} MB cap."
+                )
+            out.write(block)
+    return dest_path
+
+
+def _rag_unzip(zip_path: str, dest_dir: str = "") -> tuple:
+    """Extract a local zip with Zip-Slip + zip-bomb guards. Returns (dir, file_list)."""
+    import os
+    import zipfile
+    if not dest_dir:
+        dest_dir = os.path.splitext(zip_path)[0] + "_unzipped"
+    os.makedirs(dest_dir, exist_ok=True)
+    dest_root = os.path.realpath(dest_dir)
+    written = 0
+    names = []
+    with zipfile.ZipFile(zip_path) as zf:
+        infos = zf.infolist()
+        if len(infos) > _RAG_MAX_FILES:
+            raise ValueError(f"Zip has too many entries (> {_RAG_MAX_FILES}).")
+        total_uncompressed = sum(i.file_size for i in infos)
+        if total_uncompressed > _RAG_MAX_UNZIP_BYTES:
+            raise ValueError("Zip uncompressed size exceeds the safety cap (possible zip bomb).")
+        for info in infos:
+            if info.is_dir():
+                continue
+            target = os.path.realpath(os.path.join(dest_dir, info.filename))
+            # Zip Slip: extracted path must stay within dest_dir.
+            if not (target == dest_root or target.startswith(dest_root + os.sep)):
+                continue
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            with zf.open(info) as src, open(target, "wb") as dst:
+                dst.write(src.read())
+            written += 1
+            names.append(info.filename)
+    return dest_dir, names
+
+
+def _rag_iter_text_files(root: str):
+    """Yield (abs_path, rel_path, ext) for indexable text files under root."""
+    import os
+    if os.path.isfile(root):
+        yield root, os.path.basename(root), os.path.splitext(root)[1].lower()
+        return
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in _RAG_SKIP_DIRS]
+        for fn in filenames:
+            ext = os.path.splitext(fn)[1].lower()
+            base = fn.lower()
+            if ext not in _RAG_TEXT_EXTS and base not in ("dockerfile", "makefile", ".gitignore"):
+                continue
+            abs_path = os.path.join(dirpath, fn)
+            try:
+                if os.path.getsize(abs_path) > _RAG_MAX_FILE_BYTES:
+                    continue
+            except OSError:
+                continue
+            yield abs_path, os.path.relpath(abs_path, root), ext
+
+
+def _rag_chunk_text(text: str, rel_path: str, ext: str):
+    """Line-aware windowed chunks with overlap. Returns list of {file,ext,start,end,text}."""
+    if not text.strip():
+        return []
+    chunks = []
+    n = len(text)
+    step = max(1, _RAG_CHUNK_CHARS - _RAG_CHUNK_OVERLAP)
+    i = 0
+    while i < n:
+        end = min(n, i + _RAG_CHUNK_CHARS)
+        # Prefer to cut on a newline near the window end.
+        if end < n:
+            nl = text.rfind("\n", i + step, end)
+            if nl > i:
+                end = nl + 1
+        piece = text[i:end].strip()
+        if piece:
+            chunks.append({"file": rel_path, "ext": ext, "start": i, "end": end, "text": piece})
+        if end >= n:
+            break
+        i = end - _RAG_CHUNK_OVERLAP if end - _RAG_CHUNK_OVERLAP > i else end
+    return chunks
+
+
+def _embed(base: str, model: str, texts: list) -> list:
+    """Embed a batch of texts. Auto-detects the endpoint from `base` (a bare host:port,
+    an .../v1, or an Ollama host): tries OpenAI-compatible /v1/embeddings first (MLX
+    servers, modern Ollama), then Ollama /api/embed, then legacy /api/embeddings."""
+    # Normalize to a bare root (strip any /v1, /api, /api/embed(dings) suffix).
+    root = re.sub(r"/(v1|api/embeddings|api/embed|api)/?$", "", (base or "").rstrip("/")).rstrip("/")
+    if not root:
+        raise RuntimeError("No embedding endpoint configured (set the RAG embedding server URL).")
+    ctx = _ssl_context()
+    last_err = None
+
+    def _post(url, payload, timeout):
+        req = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"),
+            headers={"Content-Type": "application/json"}, method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+            return json.loads(resp.read().decode("utf-8", "replace"))
+
+    # 1) OpenAI-compatible /v1/embeddings — MLX servers on a port, and modern Ollama.
+    try:
+        data = _post(f"{root}/v1/embeddings", {"model": model, "input": texts}, 300)
+        items = data.get("data")
+        if isinstance(items, list) and items and isinstance(items[0], dict) and "embedding" in items[0]:
+            return [it["embedding"] for it in items]
+    except urllib.error.HTTPError as e:
+        if e.code not in (400, 404, 405, 501):
+            raise RuntimeError(f"Embeddings failed (HTTP {e.code}): {e.read().decode('utf-8', 'replace')[:200]}")
+        last_err = f"/v1/embeddings HTTP {e.code}"
+    except urllib.error.URLError as e:
+        raise RuntimeError(f"Embedding server unreachable at {root}: {getattr(e, 'reason', e)}")
+
+    # 2) Ollama batch /api/embed.
+    try:
+        data = _post(f"{root}/api/embed", {"model": model, "input": texts}, 300)
+        embs = data.get("embeddings")
+        if isinstance(embs, list) and embs and isinstance(embs[0], list):
+            return embs
+    except urllib.error.HTTPError as e:
+        if e.code not in (400, 404, 405, 501):
+            raise RuntimeError(f"Ollama embed failed (HTTP {e.code}): {e.read().decode('utf-8', 'replace')[:200]}")
+        last_err = f"/api/embed HTTP {e.code}"
+
+    # 3) Legacy Ollama /api/embeddings (per text).
+    out = []
+    for t in texts:
+        data = _post(f"{root}/api/embeddings", {"model": model, "prompt": t}, 120)
+        out.append(data.get("embedding") or [])
+    if any(out):
+        return out
+    raise RuntimeError(f"No embedding endpoint responded at {root} ({last_err or 'unknown'}).")
+
+
+def _cosine(a: list, b: list) -> float:
+    import math
+    if not a or not b or len(a) != len(b):
+        return -1.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(y * y for y in b))
+    if na == 0 or nb == 0:
+        return -1.0
+    return dot / (na * nb)
+
+
+def _rag_parse_location(loc: str):
+    """Parse an ARN / s3:// URI / bucket[/prefix] into (bucket, prefix)."""
+    s = (loc or "").strip() or "arn:aws:s3:::gonext-rag"
+    rest = s
+    m = re.match(r"^arn:aws:s3:::(.+)$", s, re.I)
+    if m:
+        rest = m.group(1)
+    else:
+        m = re.match(r"^s3://(.+)$", s, re.I)
+        if m:
+            rest = m.group(1)
+    rest = rest.strip("/")
+    if "/" in rest:
+        bucket, prefix = rest.split("/", 1)
+    else:
+        bucket, prefix = rest, ""
+    return bucket, prefix.strip("/")
+
+
+def _rag_s3_client(region: str, akid: str, secret: str):
+    import boto3  # noqa: PLC0415
+    return boto3.client(
+        "s3", region_name=region or "ap-southeast-1",
+        aws_access_key_id=akid, aws_secret_access_key=secret,
+    )
+
+
 def run_agent_chat(cfg):
     try:
         from smolagents import CodeAgent, OpenAIServerModel, tool
@@ -1238,6 +1515,68 @@ def run_agent_chat(cfg):
     # action must never be silently available (same gating as the PDF reader).
     _EMAIL_AVAILABLE = bool(email_enabled and email_api_url and email_body_template)
 
+    # ---- RAG config (download/unzip/index/search over a user's ZIP-at-URL) ----
+    rag_enabled = bool(cfg.get("ragEnabled"))
+    rag_location = (cfg.get("ragS3Location") or "arn:aws:s3:::gonext-rag").strip()
+    rag_region = (cfg.get("ragAwsRegion") or "ap-southeast-1").strip()
+    rag_akid = (cfg.get("ragAwsAccessKeyId") or "").strip()
+    rag_secret = (cfg.get("ragAwsSecretAccessKey") or "").strip()
+    rag_embed_model = (cfg.get("ragEmbedModel") or "nomic-embed-text").strip()
+    rag_embed_url = (cfg.get("ragEmbedUrl") or "").strip()
+    try:
+        rag_top_k = int(cfg.get("ragTopK") or 6)
+    except (TypeError, ValueError):
+        rag_top_k = 6
+    rag_top_k = max(1, min(50, rag_top_k))
+    # Embedding server: an explicit MLX/Ollama endpoint from Settings if given, else the
+    # agent's Ollama coding server. _embed() auto-detects OpenAI-compat (/v1/embeddings,
+    # e.g. an MLX server on a port) vs Ollama (/api/embed) — so a bare host:port works.
+    rag_embed_base = rag_embed_url or (coding_base_url or agent_base_url or "")
+    try:
+        import boto3 as _boto3_probe  # noqa: F401,PLC0415
+        _boto3_ok = True
+    except Exception:  # noqa: BLE001
+        _boto3_ok = False
+    _RAG_AVAILABLE = bool(rag_enabled and rag_akid and rag_secret and rag_embed_base and _boto3_ok)
+    if rag_enabled and not _boto3_ok:
+        _log("RAG requested but boto3 is not installed in the worker python — RAG tools disabled")
+
+    def _rag_s3_and_loc():
+        client = _rag_s3_client(rag_region, rag_akid, rag_secret)
+        bucket, prefix = _rag_parse_location(rag_location)
+        return client, bucket, prefix
+
+    def _rag_prefix_for(source_url: str, prefix: str) -> str:
+        key = _rag_source_key(source_url)
+        base = (prefix + "/") if prefix else ""
+        return f"{base}agent-rag/{key}"
+
+    def _rag_load_chunks(client, bucket: str, base_prefix: str) -> list:
+        """Load all chunk records (across shards) for a source. Returns [] if none."""
+        out = []
+        token = None
+        while True:
+            kw = {"Bucket": bucket, "Prefix": f"{base_prefix}/chunks"}
+            if token:
+                kw["ContinuationToken"] = token
+            resp = client.list_objects_v2(**kw)
+            for obj in resp.get("Contents", []) or []:
+                if not obj["Key"].endswith(".jsonl"):
+                    continue
+                body = client.get_object(Bucket=bucket, Key=obj["Key"])["Body"].read()
+                for line in body.decode("utf-8", "replace").splitlines():
+                    line = line.strip()
+                    if line:
+                        try:
+                            out.append(json.loads(line))
+                        except json.JSONDecodeError:
+                            pass
+            if resp.get("IsTruncated"):
+                token = resp.get("NextContinuationToken")
+            else:
+                break
+        return out
+
     def _prior_email_preview() -> bool:
         """True if an earlier assistant turn showed a send_email preview awaiting confirm."""
         for m in reversed(messages[:last_user_idx]):
@@ -1384,7 +1723,25 @@ def run_agent_chat(cfg):
     if _EMAIL_AVAILABLE:
         _tool_num += 1
         _email_num = _tool_num
+    if _RAG_AVAILABLE:
+        _tool_num += 5  # download_file, unzip_file, rag_index, rag_add, rag_search
     _tool_count_word = {6: "SIX", 7: "SEVEN", 8: "EIGHT"}.get(_tool_num, str(_tool_num))
+    _rag_tool_block = (
+        "  KNOWLEDGE BASE (RAG) — to SUMMARIZE or ANSWER questions about a ZIP of files at a URL:\n"
+        "    - download_file(url) — download a file (e.g. a .zip) to this machine.\n"
+        "    - unzip_file(zip_path) — extract a downloaded .zip locally.\n"
+        "    - rag_index(path, source_url) — index the unzipped text/code files into the knowledge "
+        "base (embeds to S3). source_url MUST be the ORIGINAL url the user gave.\n"
+        "    - rag_add(source_url, text) — add extra info the user provides to that knowledge base.\n"
+        "    - rag_search(source_url, query) — retrieve the most relevant chunks to answer a question.\n"
+    ) if _RAG_AVAILABLE else ""
+    _rag_choose_line = (
+        "- user gives a URL to a ZIP of files and asks to summarize/analyze/RAG them -> "
+        "download_file(url) THEN unzip_file(zip) THEN rag_index(dir, source_url=url) THEN "
+        "rag_search(source_url=url, query=<the user's question>), THEN answer from the results. "
+        "For FOLLOW-UP questions about the SAME url, just rag_search (do NOT re-download). "
+        "If the user adds information, rag_add(source_url=url, text=...).\n"
+    ) if _RAG_AVAILABLE else ""
     _pdf_read_tool_line = (
         f"  {_pdf_num}. extract_text_from_pdf(url) — READ/summarize an EXISTING PDF file "
         "at a URL. This does NOT create a PDF (use create_pdf for that).\n"
@@ -1421,7 +1778,7 @@ def run_agent_chat(cfg):
         "Use this ONLY when the task explicitly asks for the date or time.\n"
         "  6. create_pdf(text, title='') — make a PDF document from text/data and return a "
         "download link. Use ONLY when the task asks to create/make/generate/export a PDF.\n"
-        + _pdf_read_tool_line + _email_tool_line +
+        + _pdf_read_tool_line + _email_tool_line + _rag_tool_block +
         "\n"
         "http_request RETURN FORMAT: 'HTTP 200\\n{body}' — first line is 'HTTP <code>', body follows.\n"
         "\n"
@@ -1445,7 +1802,7 @@ def run_agent_chat(cfg):
         "- a PDF of info you must LOOK UP first (e.g. 'get the world cup schedule then make a "
         "pdf') -> FIRST web_search/fetch_url to gather the real facts, THEN in a later step "
         "create_pdf(text=<the gathered facts>, title=...). Never PDF the request wording itself.\n"
-        + _pdf_read_choose_line + _email_choose_line +
+        + _pdf_read_choose_line + _email_choose_line + _rag_choose_line +
         "- A specific known API/URL was given -> http_request().\n"
         "\n"
         "RULES:\n"
@@ -2275,14 +2632,160 @@ def run_agent_chat(cfg):
             client_kwargs={"timeout": 600.0, "max_retries": 0},
             **extra_model_kwargs,
         )
+        @tool
+        def download_file(url: str, dest_path: str = "") -> str:
+            """Download a file (e.g. a .zip) from a public http(s) URL to a local path on this machine. Use this FIRST when the user gives a URL to a zip of files; then unzip_file, then rag_index.
+
+            Args:
+                url: public http(s) URL of the file to download.
+                dest_path: optional local save path; leave empty for an automatic work-dir path.
+            """
+            _emit({"type": "step", "text": f"Downloading → {url[:80]}"})
+            try:
+                import os
+                path = _rag_download(url, (dest_path or "").strip())
+                return f"Downloaded to {path} ({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}') if it is a zip."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: download failed: {type(e).__name__}: {e}"
+
+        @tool
+        def unzip_file(zip_path: str, dest_dir: str = "") -> str:
+            """Extract a local .zip file into a local directory. Use after download_file. Returns the directory and a short file listing.
+
+            Args:
+                zip_path: local path to the .zip (from download_file).
+                dest_dir: optional target directory; leave empty for an automatic one.
+            """
+            _emit({"type": "step", "text": "Unzipping…"})
+            try:
+                d, names = _rag_unzip((zip_path or "").strip(), (dest_dir or "").strip())
+                preview = ", ".join(names[:15])
+                more = f" …(+{len(names) - 15} more)" if len(names) > 15 else ""
+                return f"Unzipped {len(names)} files to {d}. Files: {preview}{more}. Next: rag_index(path='{d}', source_url=<the original url>)."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: unzip failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_index(path: str, source_url: str) -> str:
+            """Index a local file or directory of TEXT/code files into the knowledge base for source_url (stores embeddings on S3). Use after unzip_file so the files become searchable. source_url MUST be the original URL the user provided.
+
+            Args:
+                path: local file or directory to index (from unzip_file).
+                source_url: the original URL the user gave — used as the knowledge-base key.
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured (enable it + add AWS credentials in Settings)."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": "Indexing files for RAG…"})
+            try:
+                import time as _t
+                records = []
+                files = 0
+                for abs_path, rel, ext in _rag_iter_text_files((path or "").strip()):
+                    try:
+                        with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
+                            text = fh.read()
+                    except OSError:
+                        continue
+                    files += 1
+                    records.extend(_rag_chunk_text(text, rel, ext))
+                if not records:
+                    return "No indexable text files found at that path."
+                for i in range(0, len(records), 64):
+                    batch = records[i:i + 64]
+                    vecs = _embed(rag_embed_base, rag_embed_model, [r["text"] for r in batch])
+                    for r, v in zip(batch, vecs):
+                        r["embedding"] = v
+                    _emit({"type": "step", "text": f"Embedded {min(i + 64, len(records))}/{len(records)} chunks…"})
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                shard = f"{base}/chunks-{int(_t.time())}.jsonl"
+                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
+                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                manifest = {
+                    "sourceUrl": source_url, "embedModel": rag_embed_model,
+                    "dims": len(records[0].get("embedding") or []),
+                    "files": files, "chunks": len(records),
+                    "updatedAt": _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()),
+                }
+                client.put_object(Bucket=bucket, Key=f"{base}/manifest.json",
+                                  Body=json.dumps(manifest).encode("utf-8"),
+                                  ContentType="application/json")
+                return (f"Indexed {files} files into {len(records)} chunks for {source_url}. "
+                        f"Use rag_search(source_url='{source_url}', query=...) to retrieve, then answer the user.")
+            except Exception as e:  # noqa: BLE001
+                return f"Error: indexing failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_add(source_url: str, text: str) -> str:
+            """Add extra text/notes to an EXISTING knowledge base (keyed by source_url). Use when the user gives more information to remember for later questions.
+
+            Args:
+                source_url: the knowledge-base key (the original URL).
+                text: the new information to embed and store.
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured."
+            if not (text or "").strip():
+                return "Error: no text to add."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": "Adding info to RAG…"})
+            try:
+                import time as _t
+                records = _rag_chunk_text(text, "user-note", ".txt")
+                vecs = _embed(rag_embed_base, rag_embed_model, [r["text"] for r in records])
+                for r, v in zip(records, vecs):
+                    r["embedding"] = v
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                shard = f"{base}/chunks-add-{int(_t.time())}.jsonl"
+                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
+                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                return f"Added {len(records)} chunk(s) to the knowledge base for {source_url}."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: rag_add failed: {type(e).__name__}: {e}"
+
+        @tool
+        def rag_search(source_url: str, query: str, k: int = 0) -> str:
+            """Retrieve the most relevant chunks from the knowledge base for source_url. Use to ANSWER questions about the indexed files; write your answer from the returned chunks.
+
+            Args:
+                source_url: the knowledge-base key (the original URL).
+                query: what to look for.
+                k: number of chunks to return (0 = configured default).
+            """
+            if not _RAG_AVAILABLE:
+                return "Error: RAG is not configured."
+            _emit({"type": "step", "text": f"RAG source → {source_url}"})
+            _emit({"type": "step", "text": f"Searching RAG → {query[:60]}"})
+            try:
+                client, bucket, prefix = _rag_s3_and_loc()
+                base = _rag_prefix_for(source_url, prefix)
+                chunks = _rag_load_chunks(client, bucket, base)
+                if not chunks:
+                    return f"No knowledge base found for {source_url}. Run rag_index first."
+                qvec = _embed(rag_embed_base, rag_embed_model, [query])[0]
+                scored = sorted(
+                    ((_cosine(qvec, c.get("embedding") or []), c) for c in chunks),
+                    key=lambda x: x[0], reverse=True,
+                )
+                topk = scored[: (k if k and k > 0 else rag_top_k)]
+                parts = [f"[{c.get('file', '?')}] (score {score:.2f})\n{c.get('text', '')}"
+                         for score, c in topk]
+                return "\n\n---\n\n".join(parts) if parts else "No matches."
+            except Exception as e:  # noqa: BLE001
+                return f"Error: rag_search failed: {type(e).__name__}: {e}"
+
         agent_tools = [http_request, web_search, fetch_url, calculate,
                        get_current_datetime, create_pdf]
-        # Only register the PDF reader / email tools when available (kept in sync with
-        # the gated hint entries above so the model never sees a tool it can't run).
+        # Only register the PDF reader / email / RAG tools when available (kept in sync
+        # with the gated hint entries above so the model never sees a tool it can't run).
         if _PDF_READ_AVAILABLE:
             agent_tools.append(extract_text_from_pdf)
         if _EMAIL_AVAILABLE:
             agent_tools.append(send_email)
+        if _RAG_AVAILABLE:
+            agent_tools += [download_file, unzip_file, rag_index, rag_add, rag_search]
         agent_kwargs = dict(
             tools=agent_tools,
             model=model,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.155",
+  "version": "1.0.158",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",