@melaya/runner 1.0.42 → 1.0.43

package/dist/pythonEnv.js

@@ -170,8 +170,62 @@ function runProc(cmd, args, onLine, envExtra = {}) {
         child.on("error", () => resolve(1));
     });
 }
+/**
+ * Idempotent: ensures NLTK's tokenizer data is present in ~/nltk_data so
+ * agentscope.rag readers (default split_by="sentence") can chunk without
+ * silently failing. Safe to call on every runner launch — the downloader
+ * skips packages that are already present and finishes in <100ms.
+ *
+ * Why this is outside the venv-rebuild guard: previously this was only
+ * triggered when PIP_DEPS changed, so any runner box that already had a
+ * matching marker would skip the download. Result: Mode B ingest failed
+ * with `Resource 'punkt_tab' not found` for every file. Moving it here
+ * makes it self-healing on every invocation.
+ */
+async function ensureNltkData(onProgress) {
+    if (!existsSync(venvPython()))
+        return; // venv not built yet, will get caught later
+    const pyCode = [
+        "import sys, os",
+        "try:",
+        "    import nltk",
+        "except ImportError:",
+        "    print('nltk_missing'); sys.exit(0)",
+        "needed = ['punkt', 'punkt_tab']",
+        "missing = []",
+        "for p in needed:",
+        "    try: nltk.data.find(f'tokenizers/{p}')",
+        "    except LookupError: missing.append(p)",
+        "if not missing:",
+        "    print('nltk_data_ok'); sys.exit(0)",
+        "print('downloading: ' + ', '.join(missing))",
+        "ok_all = True",
+        "for p in missing:",
+        "    try:",
+        "        if not nltk.download(p, quiet=True):",
+        "            ok_all = False",
+        "            print(f'nltk.download({p!r}) returned False')",
+        "    except Exception as exc:",
+        "        ok_all = False",
+        "        print(f'nltk.download({p!r}) raised {type(exc).__name__}: {exc}')",
+        "print('nltk_data_ok' if ok_all else 'nltk_data_partial')",
+    ].join("\n");
+    const code = await runProc(venvPython(), ["-c", pyCode], (line) => {
+        // Surface meaningful lines, skip generic nltk download chatter.
+        if (/^downloading|^nltk_data_|raised|returned False|nltk_missing/.test(line)) {
+            onProgress(`nltk: ${line}`);
+        }
+    });
+    if (code !== 0) {
+        onProgress(`(nltk check exited ${code} — Mode B vector RAG ingest may fail with "Resource 'punkt_tab' not found")`);
+    }
+}
 export async function ensurePythonEnv(systemPython, expectedVersion, onProgress = (m) => console.log(chalk.gray(`  [venv] ${m}`))) {
     if (venvIsValid(expectedVersion)) {
+        // Self-heal: ensure NLTK data is present even when the venv marker
+        // says we're up to date. This is what previously broke for boxes
+        // that upgraded the runner without invalidating the marker.
+        await ensureNltkData(onProgress);
         return { ok: true, pythonPath: venvPython() };
     }
     if (!existsSync(AGENTSCOPE)) {
@@ -240,20 +294,10 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
     if (scrCode !== 0) {
         onProgress(`(scrapling install exited ${scrCode} — fast-path fetcher still works; stealth-rescue disabled)`);
     }
-    // NLTK tokenizer data — agentscope.rag readers default to split_by=
-    // "sentence" which calls nltk.sent_tokenize → loads tokenizers/punkt_tab.
-    // The readers do `nltk.download("punkt_tab", quiet=True)` lazily, but
-    // when the process is sandboxed or the user's network blocks the NLTK
-    // CDN that silently no-ops and EVERY file fails with "Resource
-    // 'punkt_tab' not found". We download here, once, so the data is on
-    // disk before the first ingest runs. Non-fatal: if the download fails
-    // we log it; the user can manually run
-    //   `python -m nltk.downloader punkt punkt_tab` once with network.
-    onProgress("downloading NLTK tokenizer data (punkt, punkt_tab) — one-time, ~3MB");
-    const nltkCode = await runProc(venvPython(), ["-m", "nltk.downloader", "punkt", "punkt_tab"], onProgress);
-    if (nltkCode !== 0) {
-        onProgress(`(nltk.downloader exited ${nltkCode} — Mode B vector RAG ingest will fail with "Resource 'punkt_tab' not found" until you run \`python -m nltk.downloader punkt punkt_tab\` manually with network access)`);
-    }
+    // NLTK tokenizer data — same idempotent check the always-runs path uses.
+    // Centralised in `ensureNltkData()` so the success/failure surface is
+    // identical whether the venv is fresh or already valid.
+    await ensureNltkData(onProgress);
     writeFileSync(VENV_MARK, venvMarkerValue(expectedVersion), "utf-8");
     // sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
     // Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.42",
+  "version": "1.0.43",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,