@melaya/runner 1.0.42 → 1.0.44

package/dist/connection.js

@@ -529,6 +529,19 @@ export async function connect(opts) {
             // lives.
             const storePath = join(homedir(), ".melaya-runner", "rag", payload.pipelineName, "store");
             console.log(chalk.gray(`  [rag] store → ${storePath}`));
+            // NLTK_DATA points at the English-only tokenizer bundle shipped
+            // inside this npm package (`<runner>/nltk_data/`). Hermetic — no
+            // network, no SSL/proxy quirks, no `Resource 'punkt_tab' not found`.
+            // The runner published 1.0.43 and earlier relied on `nltk.download`
+            // at runtime which silently no-op'd on some macOS Pythons (cert
+            // verification path issue). Bundling the ~657KB English bundle
+            // eliminates the failure mode entirely.
+            const candidateNltkPaths = [
+                join(__dirname, "..", "nltk_data"),
+                join(__dirname, "nltk_data"),
+            ];
+            const { existsSync: _exists2 } = await import("fs");
+            const bundledNltkData = candidateNltkPaths.find((p) => _exists2(p)) || "";
             const proc = spawn(env.pythonPath, [
                 "-u", stagedScript,
                 "--pipeline-name", payload.pipelineName,
@@ -539,6 +552,7 @@ export async function connect(opts) {
                 env: {
                     ...process.env,
                     PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves
+                    ...(bundledNltkData ? { NLTK_DATA: bundledNltkData } : {}),
                 },
                 cwd: workDir,
             });

package/dist/pythonEnv.js

@@ -170,8 +170,62 @@ function runProc(cmd, args, onLine, envExtra = {}) {
         child.on("error", () => resolve(1));
     });
 }
+/**
+ * Idempotent: ensures NLTK's tokenizer data is present in ~/nltk_data so
+ * agentscope.rag readers (default split_by="sentence") can chunk without
+ * silently failing. Safe to call on every runner launch — the downloader
+ * skips packages that are already present and finishes in <100ms.
+ *
+ * Why this is outside the venv-rebuild guard: previously this was only
+ * triggered when PIP_DEPS changed, so any runner box that already had a
+ * matching marker would skip the download. Result: Mode B ingest failed
+ * with `Resource 'punkt_tab' not found` for every file. Moving it here
+ * makes it self-healing on every invocation.
+ */
+async function ensureNltkData(onProgress) {
+    if (!existsSync(venvPython()))
+        return; // venv not built yet, will get caught later
+    const pyCode = [
+        "import sys, os",
+        "try:",
+        "    import nltk",
+        "except ImportError:",
+        "    print('nltk_missing'); sys.exit(0)",
+        "needed = ['punkt', 'punkt_tab']",
+        "missing = []",
+        "for p in needed:",
+        "    try: nltk.data.find(f'tokenizers/{p}')",
+        "    except LookupError: missing.append(p)",
+        "if not missing:",
+        "    print('nltk_data_ok'); sys.exit(0)",
+        "print('downloading: ' + ', '.join(missing))",
+        "ok_all = True",
+        "for p in missing:",
+        "    try:",
+        "        if not nltk.download(p, quiet=True):",
+        "            ok_all = False",
+        "            print(f'nltk.download({p!r}) returned False')",
+        "    except Exception as exc:",
+        "        ok_all = False",
+        "        print(f'nltk.download({p!r}) raised {type(exc).__name__}: {exc}')",
+        "print('nltk_data_ok' if ok_all else 'nltk_data_partial')",
+    ].join("\n");
+    const code = await runProc(venvPython(), ["-c", pyCode], (line) => {
+        // Surface meaningful lines, skip generic nltk download chatter.
+        if (/^downloading|^nltk_data_|raised|returned False|nltk_missing/.test(line)) {
+            onProgress(`nltk: ${line}`);
+        }
+    });
+    if (code !== 0) {
+        onProgress(`(nltk check exited ${code} — Mode B vector RAG ingest may fail with "Resource 'punkt_tab' not found")`);
+    }
+}
 export async function ensurePythonEnv(systemPython, expectedVersion, onProgress = (m) => console.log(chalk.gray(`  [venv] ${m}`))) {
     if (venvIsValid(expectedVersion)) {
+        // Self-heal: ensure NLTK data is present even when the venv marker
+        // says we're up to date. This is what previously broke for boxes
+        // that upgraded the runner without invalidating the marker.
+        await ensureNltkData(onProgress);
         return { ok: true, pythonPath: venvPython() };
     }
     if (!existsSync(AGENTSCOPE)) {
@@ -240,20 +294,10 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
     if (scrCode !== 0) {
         onProgress(`(scrapling install exited ${scrCode} — fast-path fetcher still works; stealth-rescue disabled)`);
     }
-    // NLTK tokenizer data — agentscope.rag readers default to split_by=
-    // "sentence" which calls nltk.sent_tokenize → loads tokenizers/punkt_tab.
-    // The readers do `nltk.download("punkt_tab", quiet=True)` lazily, but
-    // when the process is sandboxed or the user's network blocks the NLTK
-    // CDN that silently no-ops and EVERY file fails with "Resource
-    // 'punkt_tab' not found". We download here, once, so the data is on
-    // disk before the first ingest runs. Non-fatal: if the download fails
-    // we log it; the user can manually run
-    //   `python -m nltk.downloader punkt punkt_tab` once with network.
-    onProgress("downloading NLTK tokenizer data (punkt, punkt_tab) — one-time, ~3MB");
-    const nltkCode = await runProc(venvPython(), ["-m", "nltk.downloader", "punkt", "punkt_tab"], onProgress);
-    if (nltkCode !== 0) {
-        onProgress(`(nltk.downloader exited ${nltkCode} — Mode B vector RAG ingest will fail with "Resource 'punkt_tab' not found" until you run \`python -m nltk.downloader punkt punkt_tab\` manually with network access)`);
-    }
+    // NLTK tokenizer data — same idempotent check the always-runs path uses.
+    // Centralised in `ensureNltkData()` so the success/failure surface is
+    // identical whether the venv is fresh or already valid.
+    await ensureNltkData(onProgress);
     writeFileSync(VENV_MARK, venvMarkerValue(expectedVersion), "utf-8");
     // sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
     // Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at

package/nltk_data/tokenizers/punkt/PY3/README

@@ -0,0 +1,98 @@
+Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
+
+Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
+been contributed by various people using NLTK for sentence boundary detection.
+
+For information about how to use these models, please confer the tokenization HOWTO:
+http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
+and chapter 3.8 of the NLTK book:
+http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
+
+There are pretrained tokenizers for the following languages:
+
+File                Language            Source                             Contents                Size of training corpus(in tokens)           Model contributed by
+=======================================================================================================================================================================
+czech.pickle        Czech               Multilingual Corpus 1 (ECI)        Lidove Noviny                   ~345,000                             Jan Strunk / Tibor Kiss
+                                                                           Literarni Noviny
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+danish.pickle       Danish              Avisdata CD-Rom Ver. 1.1. 1995     Berlingske Tidende              ~550,000                             Jan Strunk / Tibor Kiss
+                                        (Berlingske Avisdata, Copenhagen)  Weekend Avisen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+dutch.pickle        Dutch               Multilingual Corpus 1 (ECI)        De Limburger                    ~340,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+english.pickle      English             Penn Treebank (LDC)                Wall Street Journal             ~469,000                             Jan Strunk / Tibor Kiss
+                    (American)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+estonian.pickle     Estonian            University of Tartu, Estonia       Eesti Ekspress                  ~359,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+finnish.pickle      Finnish             Finnish Parole Corpus, Finnish     Books and major national        ~364,000                             Jan Strunk / Tibor Kiss
+                                        Text Bank (Suomen Kielen           newspapers
+                                        Tekstipankki)
+                                        Finnish Center for IT Science
+                                        (CSC)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+french.pickle       French              Multilingual Corpus 1 (ECI)        Le Monde                        ~370,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+german.pickle       German              Neue Zürcher Zeitung AG            Neue Zürcher Zeitung            ~847,000                             Jan Strunk / Tibor Kiss
+                    (Switzerland)       CD-ROM
+                    (Uses "ss"
+                     instead of "ß")
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+greek.pickle        Greek               Efstathios Stamatatos              To Vima (TO BHMA)               ~227,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+italian.pickle      Italian             Multilingual Corpus 1 (ECI)        La Stampa, Il Mattino           ~312,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+norwegian.pickle    Norwegian           Centre for Humanities              Bergens Tidende                 ~479,000                             Jan Strunk / Tibor Kiss
+                    (Bokmål and         Information Technologies,
+                     Nynorsk)           Bergen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+polish.pickle       Polish              Polish National Corpus             Literature, newspapers, etc.  ~1,000,000                             Krzysztof Langner
+                                        (http://www.nkjp.pl/)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+portuguese.pickle   Portuguese          CETENFolha Corpus                  Folha de São Paulo              ~321,000                             Jan Strunk / Tibor Kiss
+                    (Brazilian)         (Linguateca)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+slovene.pickle      Slovene             TRACTOR                            Delo                            ~354,000                             Jan Strunk / Tibor Kiss
+                                        Slovene Academy for Arts
+                                        and Sciences
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+spanish.pickle      Spanish             Multilingual Corpus 1 (ECI)        Sur                             ~353,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+swedish.pickle      Swedish             Multilingual Corpus 1 (ECI)        Dagens Nyheter                  ~339,000                             Jan Strunk / Tibor Kiss
+                                                                           (and some other texts)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+turkish.pickle      Turkish             METU Turkish Corpus                Milliyet                        ~333,000                             Jan Strunk / Tibor Kiss
+                                        (Türkçe Derlem Projesi)
+                                        University of Ankara
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
+Unicode using the codecs module.
+
+Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
+Computational Linguistics 32: 485-525.
+
+---- Training Code ----
+
+# import punkt
+import nltk.tokenize.punkt
+
+# Make a new Tokenizer
+tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
+
+# Read in training corpus (one example: Slovene)
+import codecs
+text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
+
+# Train tokenizer
+tokenizer.train(text)
+
+# Dump pickled tokenizer
+import pickle
+out = open("slovene.pickle","wb")
+pickle.dump(tokenizer, out)
+out.close()
+
+---------

package/nltk_data/tokenizers/punkt_tab/english/abbrev_types.txt

@@ -0,0 +1,156 @@
+ct
+m.j
+t
+a.c
+n.h
+ms
+p.a.m
+dr
+pa
+p.m
+u.k
+st
+dec
+u.s.a
+lt
+g.k
+adm
+p
+h.m
+ga
+tenn
+yr
+sen
+n.c
+j.j
+d.h
+s.g
+inc
+vs
+s.p.a
+a.t
+n
+feb
+sr
+jan
+s.a.y
+n.y
+col
+g.f
+c.o.m.b
+d
+ft
+va
+r.k
+e.f
+chg
+r.i
+a.g
+minn
+a.h
+k
+n.j
+m
+l.f
+f.j
+gen
+i.m.s
+s.a
+aug
+j.p
+okla
+m.d.c
+ltd
+oct
+s
+vt
+r.a
+j.c
+ariz
+w.w
+b.v
+ore
+h
+w.r
+e.h
+mrs
+cie
+corp
+w
+n.v
+a.d
+r.j
+ok
+. . 
+e.m
+w.c
+ill
+nov
+u.s
+prof
+conn
+u.s.s.r
+mg
+f.g
+ph.d
+g
+calif
+messrs
+h.f
+wash
+tues
+sw
+bros
+u.n
+l
+wis
+mr
+sep
+d.c
+ave
+e.l
+co
+s.s
+reps
+c
+r.t
+h.c
+r
+wed
+a.s
+v
+fla
+jr
+r.h
+c.v
+m.b.a
+rep
+a.a
+e
+c.i.t
+l.a
+b.f
+j.b
+d.w
+j.k
+ala
+f
+w.va
+sept
+mich
+n.m
+j.r
+l.p
+s.c
+colo
+fri
+a.m
+g.d
+kan
+maj
+ky
+a.m.e
+n.d
+t.j
+cos
+nev

package/nltk_data/tokenizers/punkt_tab/english/collocations.tab

@@ -0,0 +1,37 @@
+##number##	international
+##number##	rj
+##number##	commodities
+##number##	cooper
+b	stewart
+##number##	genentech
+##number##	wedgestone
+i	toussie
+##number##	pepper
+j	fialka
+o	ludcke
+##number##	insider
+##number##	aes
+i	magnin
+##number##	credit
+##number##	corrections
+##number##	financing
+##number##	henley
+##number##	business
+##number##	pay-fone
+b	wigton
+b	edelman
+b	levine
+##number##	leisure
+b	smith
+j	walter
+##number##	pegasus
+##number##	dividend
+j	aron
+##number##	review
+##number##	abreast
+##number##	who
+##number##	letters
+##number##	colgate
+##number##	cbot
+##number##	notable
+##number##	zimmer