@melaya/runner 1.0.41 → 1.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pythonEnv.js +14 -0
- package/package.json +1 -1
package/dist/pythonEnv.js
CHANGED
|
@@ -240,6 +240,20 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
|
|
|
240
240
|
if (scrCode !== 0) {
|
|
241
241
|
onProgress(`(scrapling install exited ${scrCode} — fast-path fetcher still works; stealth-rescue disabled)`);
|
|
242
242
|
}
|
|
243
|
+
// NLTK tokenizer data — agentscope.rag readers default to split_by=
|
|
244
|
+
// "sentence" which calls nltk.sent_tokenize → loads tokenizers/punkt_tab.
|
|
245
|
+
// The readers do `nltk.download("punkt_tab", quiet=True)` lazily, but
|
|
246
|
+
// when the process is sandboxed or the user's network blocks the NLTK
|
|
247
|
+
// CDN that silently no-ops and EVERY file fails with "Resource
|
|
248
|
+
// 'punkt_tab' not found". We download here, once, so the data is on
|
|
249
|
+
// disk before the first ingest runs. Non-fatal: if the download fails
|
|
250
|
+
// we log it; the user can manually run
|
|
251
|
+
// `python -m nltk.downloader punkt punkt_tab` once with network.
|
|
252
|
+
onProgress("downloading NLTK tokenizer data (punkt, punkt_tab) — one-time, ~3MB");
|
|
253
|
+
const nltkCode = await runProc(venvPython(), ["-m", "nltk.downloader", "punkt", "punkt_tab"], onProgress);
|
|
254
|
+
if (nltkCode !== 0) {
|
|
255
|
+
onProgress(`(nltk.downloader exited ${nltkCode} — Mode B vector RAG ingest will fail with "Resource 'punkt_tab' not found" until you run \`python -m nltk.downloader punkt punkt_tab\` manually with network access)`);
|
|
256
|
+
}
|
|
243
257
|
writeFileSync(VENV_MARK, venvMarkerValue(expectedVersion), "utf-8");
|
|
244
258
|
// sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
|
|
245
259
|
// Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at
|