@melaya/runner 1.0.40 → 1.0.41

package/dist/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/localRagIngest.py

@@ -37,6 +37,35 @@ except ImportError as exc:
     print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
     sys.exit(2)
 
+# Ensure NLTK's sentence tokenizer data is on disk. agentscope's readers
+# do `nltk.download(..., quiet=True)` inside the chunking call but with
+# quiet=True a network failure surfaces only as "Resource 'punkt_tab' not
+# found" at sent_tokenize time. We force the download once up-front,
+# loudly, so the user sees a clear network-error if it fails — beats
+# 240 per-file failures with the same opaque message.
+def _ensure_nltk_data() -> None:
+    try:
+        import nltk  # type: ignore
+    except ImportError:
+        print("nltk missing — runner venv needs `pip install nltk`", file=sys.stderr)
+        return
+    needed = ["punkt", "punkt_tab"]
+    for pkg in needed:
+        try:
+            nltk.data.find(f"tokenizers/{pkg}")
+        except LookupError:
+            try:
+                ok = nltk.download(pkg, quiet=True)
+                if not ok:
+                    print(f"nltk.download({pkg!r}) returned False — likely offline. "
+                          f"Run `python -m nltk.downloader {pkg}` once with network.",
+                          file=sys.stderr)
+            except Exception as exc:  # noqa: BLE001
+                print(f"nltk.download({pkg!r}) failed: {type(exc).__name__}: {exc}",
+                      file=sys.stderr)
+
+_ensure_nltk_data()
+
 
 def _emit(kind: str, **fields):
     """Stream a progress event to the runner subprocess parent which

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.40",
+  "version": "1.0.41",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,