@vespermcp/mcp-server 1.2.5 → 1.2.6

package/build/index.js

@@ -131,8 +131,83 @@ function extractRequestedRows(query, requirements) {
         return Math.max(...allNums);
     return undefined;
 }
+const verifiedPythonModules = new Set();
+function getPythonCommand() {
+    return process.platform === "win32" ? "py" : "python";
+}
+function runPythonProcess(args, timeoutMs = 300000) {
+    const pyCmd = getPythonCommand();
+    return new Promise((resolve, reject) => {
+        const proc = spawn(pyCmd, args, {
+            env: {
+                ...process.env,
+                PIP_DISABLE_PIP_VERSION_CHECK: "1",
+                PYTHONUTF8: "1",
+            },
+        });
+        let stdout = "";
+        let stderr = "";
+        proc.stdout.on("data", (d) => (stdout += d.toString()));
+        proc.stderr.on("data", (d) => (stderr += d.toString()));
+        const timer = setTimeout(() => {
+            try {
+                proc.kill();
+            }
+            catch {
+                // no-op
+            }
+            reject(new Error(`Python command timed out after ${timeoutMs}ms: ${args.join(" ")}`));
+        }, timeoutMs);
+        proc.on("close", (code) => {
+            clearTimeout(timer);
+            resolve({ code: code ?? 1, stdout, stderr });
+        });
+        proc.on("error", (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+async function ensurePythonModules(modulePackagePairs) {
+    const missing = [];
+    for (const pair of modulePackagePairs) {
+        if (verifiedPythonModules.has(pair.module)) {
+            continue;
+        }
+        const check = await runPythonProcess([
+            "-c",
+            `import importlib.util,sys; sys.exit(0 if importlib.util.find_spec(${JSON.stringify(pair.module)}) else 1)`
+        ], 20000);
+        if (check.code === 0) {
+            verifiedPythonModules.add(pair.module);
+        }
+        else {
+            missing.push(pair);
+        }
+    }
+    if (missing.length === 0) {
+        return;
+    }
+    const packages = [...new Set(missing.map(m => m.packageName))];
+    console.error(`[Vesper] Installing missing Python packages: ${packages.join(", ")}`);
+    const installArgs = ["-m", "pip", "install", "--disable-pip-version-check", ...packages];
+    let install = await runPythonProcess(installArgs, 600000);
+    if (install.code !== 0) {
+        console.error(`[Vesper] pip install failed (trying --user fallback): ${(install.stderr || "").slice(0, 300)}`);
+        const userInstallArgs = ["-m", "pip", "install", "--disable-pip-version-check", "--user", ...packages];
+        install = await runPythonProcess(userInstallArgs, 600000);
+    }
+    if (install.code !== 0) {
+        const details = (install.stderr || install.stdout || "Unknown pip install error").trim();
+        throw new Error(`Failed to install required Python packages (${packages.join(", ")}). ${details}`);
+    }
+    console.error(`[Vesper] Successfully installed: ${packages.join(", ")}`);
+    for (const pair of missing) {
+        verifiedPythonModules.add(pair.module);
+    }
+}
 function runPythonJson(scriptPath, args) {
-    const pyCmd = process.platform === "win32" ? "py" : "python";
+    const pyCmd = getPythonCommand();
     return new Promise((resolve, reject) => {
         const proc = spawn(pyCmd, [scriptPath, ...args]);
         let stdout = "";
@@ -932,6 +1007,30 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     isError: true,
                 };
             }
+            const requiredModules = [
+                { module: "aiohttp", packageName: "aiohttp" },
+            ];
+            if (source === "url") {
+                requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
+            }
+            if (source === "huggingface") {
+                requiredModules.push({ module: "datasets", packageName: "datasets" });
+            }
+            if (source === "kaggle") {
+                requiredModules.push({ module: "kaggle", packageName: "kaggle" });
+            }
+            if (outputFormat === "webdataset") {
+                requiredModules.push({ module: "webdataset", packageName: "webdataset" });
+            }
+            try {
+                await ensurePythonModules(requiredModules);
+            }
+            catch (error) {
+                return {
+                    content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
+                    isError: true,
+                };
+            }
             const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
             const payload = {
                 dataset_id: datasetId,

package/build/python/vesper/core/asset_downloader.py

@@ -101,27 +101,60 @@ class AssetDownloader:
         if source not in {"huggingface", "kaggle", "url"}:
             raise ValueError("source must be one of: huggingface, kaggle, url")
 
-        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
-        images_dir = dataset_dir / "images"
-        dataset_dir.mkdir(parents=True, exist_ok=True)
-        images_dir.mkdir(parents=True, exist_ok=True)
-
-        errors_file = dataset_dir / "errors.jsonl"
-        metadata_file = dataset_dir / "metadata.jsonl"
-
+        # --- Validate imports and args BEFORE creating any directories ---
         if source == "huggingface":
             if not repo_id:
                 raise ValueError("repo_id is required for source=huggingface")
-            summary = await self._download_huggingface(repo_id, dataset_id, images_dir, metadata_file, errors_file, max_items, image_column)
+            try:
+                from datasets import load_dataset as _ld  # noqa: F401
+            except Exception as e:
+                raise RuntimeError(
+                    f"datasets package is required for HuggingFace downloads. "
+                    f"Install with: pip install datasets. Details: {e}"
+                )
         elif source == "kaggle":
             ref = kaggle_ref or repo_id
             if not ref:
                 raise ValueError("kaggle_ref is required for source=kaggle")
-            summary = await self._download_kaggle(ref, dataset_id, images_dir, metadata_file, errors_file, max_items)
+            try:
+                from kaggle.api.kaggle_api_extended import KaggleApi as _Ka  # noqa: F401
+            except Exception as e:
+                raise RuntimeError(
+                    f"kaggle package is required for Kaggle downloads. "
+                    f"Install with: pip install kaggle. Details: {e}"
+                )
         else:
             if not urls:
                 raise ValueError("urls are required for source=url")
-            summary = await self._download_urls(urls, dataset_id, images_dir, metadata_file, errors_file, max_items)
+
+        if output_format == "webdataset" and wds is None:
+            raise RuntimeError(
+                "webdataset package is required for webdataset output. "
+                "Install with: pip install webdataset"
+            )
+
+        # --- Now safe to create directories ---
+        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
+        images_dir = dataset_dir / "images"
+        dataset_dir.mkdir(parents=True, exist_ok=True)
+        images_dir.mkdir(parents=True, exist_ok=True)
+
+        errors_file = dataset_dir / "errors.jsonl"
+        metadata_file = dataset_dir / "metadata.jsonl"
+
+        try:
+            if source == "huggingface":
+                summary = await self._download_huggingface(repo_id, dataset_id, images_dir, metadata_file, errors_file, max_items, image_column)
+            elif source == "kaggle":
+                ref = kaggle_ref or repo_id
+                summary = await self._download_kaggle(ref, dataset_id, images_dir, metadata_file, errors_file, max_items)
+            else:
+                summary = await self._download_urls(urls, dataset_id, images_dir, metadata_file, errors_file, max_items)
+        except Exception:
+            # Clean up empty directories on failure so we don't leave ghost artifacts
+            if images_dir.exists() and not any(images_dir.iterdir()):
+                shutil.rmtree(dataset_dir, ignore_errors=True)
+            raise
 
         if output_format == "webdataset":
             await self._write_webdataset(dataset_dir, images_dir, metadata_file)
@@ -150,10 +183,7 @@ class AssetDownloader:
         max_items: Optional[int],
         image_column: Optional[str],
     ) -> Dict[str, int]:
-        try:
-            from datasets import load_dataset
-        except Exception as e:
-            raise RuntimeError(f"datasets package is required. Install with: pip install datasets. Details: {e}")
+        from datasets import load_dataset  # validated in download_assets()
 
         await self._emit("start", {"source": "huggingface", "repo_id": repo_id})
 
@@ -212,10 +242,7 @@ class AssetDownloader:
         errors_file: Path,
         max_items: Optional[int],
     ) -> Dict[str, int]:
-        try:
-            from kaggle.api.kaggle_api_extended import KaggleApi
-        except Exception as e:
-            raise RuntimeError(f"kaggle package is required: {e}")
+        from kaggle.api.kaggle_api_extended import KaggleApi  # validated in download_assets()
 
         await self._emit("start", {"source": "kaggle", "dataset": kaggle_ref})
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vespermcp/mcp-server",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",

package/scripts/postinstall.cjs

@@ -23,7 +23,12 @@ const pythonPackages = [
     'pillow',
     'numpy',
     'librosa',
-    'soundfile'
+    'soundfile',
+    'aiohttp',
+    'aiofiles',
+    'datasets',
+    'webdataset',
+    'kaggle'
 ];
 
 try {

package/src/python/vesper/core/asset_downloader.py

@@ -101,27 +101,60 @@ class AssetDownloader:
         if source not in {"huggingface", "kaggle", "url"}:
             raise ValueError("source must be one of: huggingface, kaggle, url")
 
-        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
-        images_dir = dataset_dir / "images"
-        dataset_dir.mkdir(parents=True, exist_ok=True)
-        images_dir.mkdir(parents=True, exist_ok=True)
-
-        errors_file = dataset_dir / "errors.jsonl"
-        metadata_file = dataset_dir / "metadata.jsonl"
-
+        # --- Validate imports and args BEFORE creating any directories ---
         if source == "huggingface":
             if not repo_id:
                 raise ValueError("repo_id is required for source=huggingface")
-            summary = await self._download_huggingface(repo_id, dataset_id, images_dir, metadata_file, errors_file, max_items, image_column)
+            try:
+                from datasets import load_dataset as _ld  # noqa: F401
+            except Exception as e:
+                raise RuntimeError(
+                    f"datasets package is required for HuggingFace downloads. "
+                    f"Install with: pip install datasets. Details: {e}"
+                )
         elif source == "kaggle":
             ref = kaggle_ref or repo_id
             if not ref:
                 raise ValueError("kaggle_ref is required for source=kaggle")
-            summary = await self._download_kaggle(ref, dataset_id, images_dir, metadata_file, errors_file, max_items)
+            try:
+                from kaggle.api.kaggle_api_extended import KaggleApi as _Ka  # noqa: F401
+            except Exception as e:
+                raise RuntimeError(
+                    f"kaggle package is required for Kaggle downloads. "
+                    f"Install with: pip install kaggle. Details: {e}"
+                )
         else:
             if not urls:
                 raise ValueError("urls are required for source=url")
-            summary = await self._download_urls(urls, dataset_id, images_dir, metadata_file, errors_file, max_items)
+
+        if output_format == "webdataset" and wds is None:
+            raise RuntimeError(
+                "webdataset package is required for webdataset output. "
+                "Install with: pip install webdataset"
+            )
+
+        # --- Now safe to create directories ---
+        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
+        images_dir = dataset_dir / "images"
+        dataset_dir.mkdir(parents=True, exist_ok=True)
+        images_dir.mkdir(parents=True, exist_ok=True)
+
+        errors_file = dataset_dir / "errors.jsonl"
+        metadata_file = dataset_dir / "metadata.jsonl"
+
+        try:
+            if source == "huggingface":
+                summary = await self._download_huggingface(repo_id, dataset_id, images_dir, metadata_file, errors_file, max_items, image_column)
+            elif source == "kaggle":
+                ref = kaggle_ref or repo_id
+                summary = await self._download_kaggle(ref, dataset_id, images_dir, metadata_file, errors_file, max_items)
+            else:
+                summary = await self._download_urls(urls, dataset_id, images_dir, metadata_file, errors_file, max_items)
+        except Exception:
+            # Clean up empty directories on failure so we don't leave ghost artifacts
+            if images_dir.exists() and not any(images_dir.iterdir()):
+                shutil.rmtree(dataset_dir, ignore_errors=True)
+            raise
 
         if output_format == "webdataset":
             await self._write_webdataset(dataset_dir, images_dir, metadata_file)
@@ -150,10 +183,7 @@ class AssetDownloader:
         max_items: Optional[int],
         image_column: Optional[str],
     ) -> Dict[str, int]:
-        try:
-            from datasets import load_dataset
-        except Exception as e:
-            raise RuntimeError(f"datasets package is required. Install with: pip install datasets. Details: {e}")
+        from datasets import load_dataset  # validated in download_assets()
 
         await self._emit("start", {"source": "huggingface", "repo_id": repo_id})
 
@@ -212,10 +242,7 @@ class AssetDownloader:
         errors_file: Path,
         max_items: Optional[int],
     ) -> Dict[str, int]:
-        try:
-            from kaggle.api.kaggle_api_extended import KaggleApi
-        except Exception as e:
-            raise RuntimeError(f"kaggle package is required: {e}")
+        from kaggle.api.kaggle_api_extended import KaggleApi  # validated in download_assets()
 
         await self._emit("start", {"source": "kaggle", "dataset": kaggle_ref})