@melaya/runner 1.0.39 → 1.0.41

package/dist/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/dist/pythonEnv.js

@@ -108,6 +108,17 @@ const PIP_DEPS = [
     // ships Windows wheels, MilvusLite does not — using Qdrant for OS
     // parity across runners.)
     "ollama",
+    // nltk — used by agentscope.rag readers' default split_by="sentence"
+    // chunker. Without it, every reader raises ImportError at chunk time
+    // (the lazy import happens INSIDE the chunking call, not at module load,
+    // so the error surfaces per-file as "nltk is not installed"). The
+    // readers also lazily `nltk.download("punkt", quiet=True)` on first use.
+    "nltk",
+    // pandas — agentscope.rag.ExcelReader uses pandas.read_excel for sheet
+    // iteration. Without it, .xlsx files in a Mode B folder fail with
+    // "pandas is not installed". Numpy is already in PIP_DEPS (line 57) so
+    // the wheel install is fast.
+    "pandas",
 ];
 export function venvPython() {
     return platform() === "win32"

package/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.39",
+  "version": "1.0.41",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,