@tiens.nguyen/gonext-local-worker 1.0.266 → 1.0.268

package/gonext-local-worker.mjs

@@ -1948,6 +1948,10 @@ async function runAgentChatJob(job) {
       ragEmbedModel: payload?.ragEmbedModel ?? "",
       ragEmbedUrl: payload?.ragEmbedUrl ?? "",
       ragTopK: payload?.ragTopK ?? 6,
+      // RAG storage backend for this folder (task #97): "local" = on disk under ~/.gonext
+      // (per-folder, no S3 creds), else "cloud" (the user's S3). Terminal sends the folder's
+      // choice; web never sets it → "cloud". python re-validates and defaults to cloud.
+      ragMode: payload?.ragMode === "local" ? "local" : "cloud",
       // Workspaces the agent may read/edit/test code in (see the terminal-vs-web split
       // computed above): the real registry for the terminal, a lone temp folder for web.
       workspaces: jobWorkspaces,

package/gonext-repl.mjs

@@ -189,6 +189,7 @@ if (!workerKey || !apiBase) {
 const COMMANDS = [
   { name: "/model", desc: "switch the coding model for this session" },
   { name: "/test-auto", desc: "toggle auto-test (verify & fix until it passes) for this folder" },
+  { name: "/rag-local", desc: "toggle storing this folder's RAG knowledge base locally (vs cloud/S3)" },
   { name: "/server", desc: "pick a deployment server (host/user) for this session" },
   { name: "/revert", usage: "/revert [runId]", desc: "undo the agent's file edits (latest run)" },
   { name: "/reset", aliases: ["/new"], desc: "forget this folder's saved conversation and start fresh" },
@@ -611,6 +612,37 @@ async function loadSessionMaxCodeTokens(cwd) {
   }
 }
 
+// RAG storage backend remembered per folder (task #97): "local" | "cloud" | null (never
+// chosen → we ask once at startup). Local keeps the index on disk under ~/.gonext; cloud
+// uses the user's S3 bucket (the original behavior).
+async function loadSessionRagMode(cwd) {
+  try {
+    const raw = JSON.parse(await readFile(sessionFilePath(cwd), "utf8"));
+    return raw?.ragMode === "local" || raw?.ragMode === "cloud" ? raw.ragMode : null;
+  } catch {
+    return null;
+  }
+}
+
+// Persist just the RAG-mode choice without touching history (used at startup, before the
+// full session is loaded). Read-merge-write so nothing else in the file is lost.
+async function saveSessionRagMode(cwd, mode) {
+  try {
+    let raw = {};
+    try {
+      raw = JSON.parse(await readFile(sessionFilePath(cwd), "utf8")) || {};
+    } catch {
+      raw = {};
+    }
+    raw.ragMode = mode;
+    raw.updatedAt = new Date().toISOString();
+    await mkdir(SESSIONS_DIR, { recursive: true });
+    await writeFile(sessionFilePath(cwd), JSON.stringify(raw, null, 2) + "\n");
+  } catch (err) {
+    console.error(dim(`(rag-mode save failed: ${err.message})`));
+  }
+}
+
 async function saveSession(cwd, history) {
   try {
     await mkdir(SESSIONS_DIR, { recursive: true });
@@ -624,6 +656,7 @@ async function saveSession(cwd, history) {
           testAuto: sessionTestAuto,
           deployServer: selectedServer, // remember the /server pick per folder (task #69)
           maxCodeTokens: sessionMaxCodeTokens, // peak per-request tokens (task #84)
+          ragMode: sessionRagMode, // local vs cloud RAG for this folder (task #97)
           history: trimmed,
         },
         null,
@@ -975,6 +1008,10 @@ let sessionCodingModel = "";
 // and fixes → re-tests until it passes before finishing. Default off; remembered per
 // folder in the session file; sent to /agent-ask as autoTest so python drives the loop.
 let sessionTestAuto = false;
+// RAG storage backend for THIS folder (task #97): "local" (index on disk under ~/.gonext,
+// per-folder, no S3 needed) | "cloud" (the user's S3 bucket) | null (never chosen → ask once
+// at startup when RAG is enabled). Remembered per folder; sent to /agent-ask as ragMode.
+let sessionRagMode = null;
 // Step-budget opt-out (task #80): set once the user picks "yes, don't ask again" at a
 // max-step prompt. Session-scoped (not persisted per folder); sent to /agent-ask each turn
 // so the agent auto-extends past the step budget without prompting again.
@@ -1146,6 +1183,9 @@ async function runAgentTurn(history) {
       ...(alwaysExtendOnMaxStep ? { alwaysExtendOnMaxStep: true } : {}),
       // Auto-test mode (/test-auto): the agent verifies its work and fixes until it passes.
       ...(sessionTestAuto ? { autoTest: true } : {}),
+      // RAG storage backend for this folder (task #97): local disk vs cloud/S3. Only sent
+      // once chosen; absent → the API/python default (cloud), preserving old behavior.
+      ...(sessionRagMode ? { ragMode: sessionRagMode } : {}),
       // Deploy target chosen via /server — host/user only (no secret), for key-auth deploys.
       ...(selectedServer ? { deployServer: selectedServer } : {}),
       ...(sessionCodingModel ? { codingModelOverride: sessionCodingModel } : {}),
@@ -1288,15 +1328,25 @@ async function runAgentTurn(history) {
     // "almost done" > "Thinking". When clickable we drive line 1 from the SAME fuller string, so
     // the grey expansion CONTINUES where line 1 was cut off instead of repeating the head.
     const clickable = !!fullLiveThought && !runningCmd && fullLiveThought.length > max;
-    const primary = runningCmd
-      ? `Running ${runningCmd}`
-      : clickable
-        ? fullLiveThought
+    let label, rest;
+    if (clickable) {
+      // Split at a WORD boundary near the line-1 width so nothing is cut mid-word: the last
+      // space at/before max-1 (unless that snaps back too far into the line — then hard-cut).
+      // Line 1 shows the head; the grey expansion CONTINUES from the same point (no dup, no
+      // split word).
+      const sp = fullLiveThought.lastIndexOf(" ", max - 1);
+      const cut = sp >= Math.floor(max * 0.6) ? sp : max - 1;
+      label = `${fullLiveThought.slice(0, cut).replace(/\s+$/, "")}… (${secs}s)`;
+      rest = fullLiveThought.slice(cut).trim();
+    } else {
+      const primary = runningCmd
+        ? `Running ${runningCmd}`
         : liveThought || (almostDone ? "Almost done" : "Thinking");
-    const label = `${fit(primary)}… (${secs}s)`;
-    // When clicked, unfold the REMAINDER (what fit() dropped, i.e. from char max-1 on) as up to
-    // 4 wrapped GREY lines BETWEEN the thought and the playful-word line.
-    const rest = clickable ? fullLiveThought.slice(max - 1).trim() : "";
+      label = `${fit(primary)}… (${secs}s)`;
+      rest = "";
+    }
+    // When clicked, unfold the remainder as up to 4 wrapped GREY lines BETWEEN the thought and
+    // the playful-word line.
     const expLines = thoughtExpanded && rest ? wrapThought(rest, max, 4) : [];
     const drawn = 2 + expLines.length;
     const rows = process.stdout.rows || 24;
@@ -1759,15 +1809,27 @@ async function main() {
   sessionTestAuto = await loadSessionTestAuto(resolve(process.cwd()));
   selectedServer = await loadSessionServer(resolve(process.cwd()));
   sessionMaxCodeTokens = await loadSessionMaxCodeTokens(resolve(process.cwd()));
+  sessionRagMode = await loadSessionRagMode(resolve(process.cwd())); // task #97: local/cloud/null
   // Show which models will answer, straight from user settings (also validates auth).
   try {
     const probe = await fetchAgentPayload([{ role: "user", content: "ping" }]);
     const p = probe?.payload ?? {};
+    // RAG storage choice (task #97): ask ONCE per folder, and only when RAG is actually
+    // enabled in Settings. Default Yes = LOCAL (kept on this machine, no S3 needed).
+    if (p.ragEnabled && sessionRagMode === null) {
+      const local = await askYesNo(
+        "Store this folder's RAG knowledge base LOCALLY on this machine?\n" +
+          "(Yes = local, kept in ~/.gonext per folder — no S3 needed; No = cloud, your S3 bucket)",
+        true /* default Yes = local */
+      );
+      sessionRagMode = local ? "local" : "cloud";
+      await saveSessionRagMode(resolve(process.cwd()), sessionRagMode);
+    }
     console.log(
       dim(
         `agent model: ${p.agentModelId || probe?.modelKey || "?"}` +
           (p.codingModelId ? ` · coder: ${p.codingModelId}` : "") +
-          (p.ragEnabled ? " · RAG on" : "") +
+          (p.ragEnabled ? ` · RAG ${sessionRagMode === "cloud" ? "cloud" : "local"}` : "") +
           ` · auto-test: ${sessionTestAuto ? "on" : "off"}` +
           (selectedServer ? ` · deploy: ${selectedServer.name} (${selectedServer.user}@${selectedServer.host})` : "")
       ) + (sessionTestAuto ? "" : dim(" (/test-auto to enable)"))
@@ -1875,6 +1937,20 @@ async function main() {
       );
       continue;
     }
+    if (line === "/rag-local") {
+      // Toggle this folder's RAG store between local disk and the cloud (S3). task #97.
+      sessionRagMode = sessionRagMode === "local" ? "cloud" : "local";
+      await saveSession(cwd, history); // persist the choice for this folder
+      console.log(
+        (sessionRagMode === "local" ? green("RAG: LOCAL") : dim("RAG: cloud")) +
+          dim(
+            sessionRagMode === "local"
+              ? " — this folder's knowledge base is stored on this machine (~/.gonext), no S3 needed.\n"
+              : " — this folder's knowledge base is stored in your S3 bucket.\n"
+          )
+      );
+      continue;
+    }
     if (line === "/reset" || line === "/new") {
       history.length = 0;
       await clearSession(cwd);

package/gonext_agent_chat.py

@@ -1757,6 +1757,21 @@ def _rag_base_dir(source_key: str) -> str:
     return _rag_workdir(source_key)
 
 
+def _rag_local_index_dir(source_url: str) -> str:
+    """Local RAG index dir for a source, scoped PER-FOLDER (task #97): the vector shards
+    live at ~/.gonext/rag/<sha256(activeFolder)>/<sha256(sourceUrl)>/. Keying on the active
+    terminal folder (the SAME sha256 scheme the REPL uses for its session files) isolates
+    each project's knowledge base, so the same URL indexed from two folders stays separate."""
+    import hashlib
+    import os
+    folder = _WS_ACTIVE or os.getcwd()
+    fhash = hashlib.sha256(os.path.realpath(folder).encode("utf-8")).hexdigest()[:32]
+    base = os.path.join(os.path.expanduser("~"), ".gonext", "rag", fhash,
+                        _rag_source_key(source_url))
+    os.makedirs(base, exist_ok=True)
+    return base
+
+
 def _rag_download(url: str, dest_path: str = "") -> tuple:
     """Download a public URL with SSRF + size guards — IDEMPOTENT per URL. Every URL
     maps to a deterministic work dir (~/.gonext/rag-work/<urlHash>/); relative
@@ -2827,9 +2842,24 @@ def run_agent_chat(cfg):
         _boto3_ok = True
     except Exception:  # noqa: BLE001
         _boto3_ok = False
-    _RAG_AVAILABLE = bool(rag_enabled and rag_akid and rag_secret and rag_embed_base and _boto3_ok)
-    if rag_enabled and not _boto3_ok:
-        _log("RAG requested but boto3 is not installed in the worker python — RAG tools disabled")
+    # RAG storage backend (task #97): "local" keeps the index on disk under ~/.gonext,
+    # PER-FOLDER, and needs NO S3 creds / boto3 — only the embedding server. "cloud" (default,
+    # and what the web app always uses) stores it on the user's own S3 bucket via boto3. The
+    # terminal REPL picks this per-folder and sends it as cfg.ragMode.
+    rag_mode = (cfg.get("ragMode") or "cloud").strip().lower()
+    if rag_mode not in ("local", "cloud"):
+        rag_mode = "cloud"
+    _RAG_LOCAL = rag_mode == "local"
+    _RAG_AVAILABLE = bool(
+        rag_enabled and rag_embed_base and (
+            _RAG_LOCAL or (rag_akid and rag_secret and _boto3_ok)
+        )
+    )
+    if rag_enabled and not _RAG_LOCAL and not _boto3_ok:
+        _log("RAG (cloud) requested but boto3 is not installed in the worker python — RAG "
+             "tools disabled (switch to local RAG with /rag-local to avoid needing S3/boto3)")
+    if _RAG_AVAILABLE:
+        _log(f"RAG mode: {'LOCAL (index under ~/.gonext/rag, per-folder)' if _RAG_LOCAL else 'cloud (S3)'}")
 
     # ---- Workspaces (local folders the agent may read/edit/test code in) ----
     # Registered on the Mac via `gonext-local-worker workspace add <path>`; the worker
@@ -2937,6 +2967,59 @@ def run_agent_chat(cfg):
                 break
         return out
 
+    # --- Storage backend abstraction (task #97): local disk vs the user's S3, chosen by
+    # ragMode. The three rag_* tools go through these so their logic stays identical. ---
+    def _rag_write_shard(source_url: str, records: list, tag: str) -> None:
+        """Write one JSONL shard of chunk records. tag is 'chunks' (index) or 'chunks-add'."""
+        import os
+        import time as _t
+        body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records)
+        name = f"{tag}-{int(_t.time())}.jsonl"
+        if _RAG_LOCAL:
+            path = os.path.join(_rag_local_index_dir(source_url), name)
+            with open(path, "w", encoding="utf-8") as fh:
+                fh.write(body)
+            return
+        client, bucket, prefix = _rag_s3_and_loc()
+        base = _rag_prefix_for(source_url, prefix)
+        client.put_object(Bucket=bucket, Key=f"{base}/{name}",
+                          Body=body.encode("utf-8"), ContentType="application/x-ndjson")
+
+    def _rag_write_manifest(source_url: str, manifest: dict) -> None:
+        import os
+        data = json.dumps(manifest).encode("utf-8")
+        if _RAG_LOCAL:
+            with open(os.path.join(_rag_local_index_dir(source_url), "manifest.json"), "wb") as fh:
+                fh.write(data)
+            return
+        client, bucket, prefix = _rag_s3_and_loc()
+        base = _rag_prefix_for(source_url, prefix)
+        client.put_object(Bucket=bucket, Key=f"{base}/manifest.json",
+                          Body=data, ContentType="application/json")
+
+    def _rag_load_all(source_url: str) -> list:
+        """Load every chunk record for a source across all shards (local or S3)."""
+        import glob as _glob
+        import os
+        if _RAG_LOCAL:
+            out = []
+            for fp in sorted(_glob.glob(os.path.join(_rag_local_index_dir(source_url), "chunks*.jsonl"))):
+                try:
+                    with open(fp, "r", encoding="utf-8", errors="replace") as fh:
+                        for line in fh:
+                            line = line.strip()
+                            if line:
+                                try:
+                                    out.append(json.loads(line))
+                                except json.JSONDecodeError:
+                                    pass
+                except OSError:
+                    continue
+            return out
+        client, bucket, prefix = _rag_s3_and_loc()
+        base = _rag_prefix_for(source_url, prefix)
+        return _rag_load_chunks(client, bucket, base)
+
     def _prior_email_preview() -> bool:
         """True if an earlier assistant turn showed a send_email preview awaiting confirm."""
         for m in reversed(messages[:last_user_idx]):
@@ -3172,7 +3255,7 @@ def run_agent_chat(cfg):
         "    - read_text_file(path) — read ONE file's text (e.g. a README) directly. Best for a QUICK "
         "project summary — you do NOT need to index for that.\n"
         "    - rag_index(path, source_url) — index MANY text/code files into a searchable knowledge "
-        "base (embeds to S3). source_url MUST be the ORIGINAL url the user gave. Use for deep Q&A "
+        "base (embeds the chunks). source_url MUST be the ORIGINAL url the user gave. Use for deep Q&A "
         "across the whole project.\n"
         "    - rag_add(source_url, text) — add extra info the user provides to that knowledge base.\n"
         "    - rag_search(source_url, query) — retrieve the most relevant chunks to answer a question.\n"
@@ -3312,7 +3395,7 @@ def run_agent_chat(cfg):
         "edit_lines(file, start, end, new_content) to change them, THEN "
         "run_command('…test…') to VERIFY if run permission exists — read failures and fix, "
         "iterate. Finish with final_answer summarizing WHAT you changed (files + lines). "
-        "AFTER editing, re-read with grep_repo/read_file_lines (an S3 knowledge base may be "
+        "AFTER editing, re-read with grep_repo/read_file_lines (an indexed knowledge base may be "
         "stale — do not trust rag_search for freshly edited code).\n"
     ) if _WS_AVAILABLE else ""
     _pdf_read_choose_line = (
@@ -4991,14 +5074,16 @@ def run_agent_chat(cfg):
 
         @tool
         def rag_index(path: str, source_url: str) -> str:
-            """Index a local file or directory of TEXT/code files into the knowledge base for source_url (stores embeddings on S3). Use after unzip_file so the files become searchable. source_url MUST be the original URL the user provided.
+            """Index a local file or directory of TEXT/code files into the knowledge base for source_url (stores embeddings in the knowledge base). Use after unzip_file so the files become searchable. source_url MUST be the original URL the user provided.
 
             Args:
                 path: local file or directory to index (from unzip_file).
                 source_url: the original URL the user gave — used as the knowledge-base key.
             """
             if not _RAG_AVAILABLE:
-                msg = "Error: RAG is not configured (enable it + add AWS credentials in Settings)."
+                msg = ("Error: RAG is not configured (enable RAG in Settings"
+                       + ("" if _RAG_LOCAL else " and add AWS credentials, or switch to local RAG")
+                       + ").")
                 _last_obs["text"] = msg
                 return msg
             _emit({"type": "step", "text": f"RAG source → {source_url}"})
@@ -5025,21 +5110,16 @@ def run_agent_chat(cfg):
                     for r, v in zip(batch, vecs):
                         r["embedding"] = v
                     _emit({"type": "step", "text": f"Embedded {min(i + 64, len(records))}/{len(records)} chunks…"})
-                client, bucket, prefix = _rag_s3_and_loc()
-                base = _rag_prefix_for(source_url, prefix)
-                shard = f"{base}/chunks-{int(_t.time())}.jsonl"
-                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
-                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                _rag_write_shard(source_url, records, "chunks")
                 manifest = {
                     "sourceUrl": source_url, "embedModel": rag_embed_model,
                     "dims": len(records[0].get("embedding") or []),
                     "files": files, "chunks": len(records),
                     "updatedAt": _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()),
                 }
-                client.put_object(Bucket=bucket, Key=f"{base}/manifest.json",
-                                  Body=json.dumps(manifest).encode("utf-8"),
-                                  ContentType="application/json")
-                out = (f"Indexed {files} files into {len(records)} chunks for {source_url}. "
+                _rag_write_manifest(source_url, manifest)
+                where = "locally" if _RAG_LOCAL else "in the cloud knowledge base"
+                out = (f"Indexed {files} files into {len(records)} chunks for {source_url} ({where}). "
                        f"Use rag_search(source_url='{source_url}', query=...) to retrieve, then answer the user.")
                 _last_obs["text"] = out
                 return out
@@ -5067,16 +5147,11 @@ def run_agent_chat(cfg):
             _emit({"type": "step", "text": f"RAG source → {source_url}"})
             _emit({"type": "step", "text": "Adding info to RAG…"})
             try:
-                import time as _t
                 records = _rag_chunk_text(text, "user-note", ".txt")
                 vecs = _embed(rag_embed_base, rag_embed_model, [r["text"] for r in records])
                 for r, v in zip(records, vecs):
                     r["embedding"] = v
-                client, bucket, prefix = _rag_s3_and_loc()
-                base = _rag_prefix_for(source_url, prefix)
-                shard = f"{base}/chunks-add-{int(_t.time())}.jsonl"
-                body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records).encode("utf-8")
-                client.put_object(Bucket=bucket, Key=shard, Body=body, ContentType="application/x-ndjson")
+                _rag_write_shard(source_url, records, "chunks-add")
                 out = f"Added {len(records)} chunk(s) to the knowledge base for {source_url}."
                 _last_obs["text"] = out
                 return out
@@ -5101,9 +5176,7 @@ def run_agent_chat(cfg):
             _emit({"type": "step", "text": f"RAG source → {source_url}"})
             _emit({"type": "step", "text": f"Searching RAG → {query[:60]}"})
             try:
-                client, bucket, prefix = _rag_s3_and_loc()
-                base = _rag_prefix_for(source_url, prefix)
-                chunks = _rag_load_chunks(client, bucket, base)
+                chunks = _rag_load_all(source_url)
                 if not chunks:
                     msg = f"No knowledge base found for {source_url}. Run rag_index first."
                     _last_obs["text"] = msg

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.266",
+  "version": "1.0.268",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",