@melaya/runner 1.0.40 → 1.0.42

package/dist/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/dist/pythonEnv.js

@@ -240,6 +240,20 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
     if (scrCode !== 0) {
         onProgress(`(scrapling install exited ${scrCode} — fast-path fetcher still works; stealth-rescue disabled)`);
     }
+    // NLTK tokenizer data — agentscope.rag readers default to split_by=
+    // "sentence" which calls nltk.sent_tokenize → loads tokenizers/punkt_tab.
+    // The readers do `nltk.download("punkt_tab", quiet=True)` lazily, but
+    // when the process is sandboxed or the user's network blocks the NLTK
+    // CDN that silently no-ops and EVERY file fails with "Resource
+    // 'punkt_tab' not found". We download here, once, so the data is on
+    // disk before the first ingest runs. Non-fatal: if the download fails
+    // we log it; the user can manually run
+    //   `python -m nltk.downloader punkt punkt_tab` once with network.
+    onProgress("downloading NLTK tokenizer data (punkt, punkt_tab) — one-time, ~3MB");
+    const nltkCode = await runProc(venvPython(), ["-m", "nltk.downloader", "punkt", "punkt_tab"], onProgress);
+    if (nltkCode !== 0) {
+        onProgress(`(nltk.downloader exited ${nltkCode} — Mode B vector RAG ingest will fail with "Resource 'punkt_tab' not found" until you run \`python -m nltk.downloader punkt punkt_tab\` manually with network access)`);
+    }
     writeFileSync(VENV_MARK, venvMarkerValue(expectedVersion), "utf-8");
     // sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
     // Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at

package/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.40",
+  "version": "1.0.42",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,