@vespermcp/mcp-server 1.2.19 → 1.2.20

package/build/index.js

@@ -453,6 +453,17 @@ jobManager.on("processJob", async (job, execute) => {
 async function handlePrepareJob(jobId, query, requirements) {
     hydrateExternalKeys();
     const update = (updates) => jobManager.updateJob(jobId, updates);
+    // Ensure core Python packages are available for dataset operations
+    try {
+        await ensurePythonModules([
+            { module: "polars", packageName: "polars" },
+            { module: "datasets", packageName: "datasets" },
+        ]);
+    }
+    catch (e) {
+        console.error(`[Prepare] Python dependency setup warning: ${e.message}`);
+        // Continue anyway - direct file downloads may still work without datasets lib
+    }
     const requestedRows = extractRequestedRows(query, requirements);
     let selectedDataset;
     let datasetIdForDownload = "";
@@ -1225,6 +1236,17 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     isError: true,
                 };
             }
+            // Pre-install Python datasets library for HuggingFace fallback
+            if (source === "huggingface") {
+                try {
+                    await ensurePythonModules([
+                        { module: "datasets", packageName: "datasets" },
+                    ]);
+                }
+                catch {
+                    // Continue - direct download may still work
+                }
+            }
             try {
                 const localPath = await dataIngestor.ensureData(datasetId, source, () => undefined);
                 try {

package/build/ingestion/ingestor.js

@@ -1,5 +1,6 @@
 import path from "path";
 import fs from "fs";
+import { spawn } from "child_process";
 import { HFDownloader } from "./hf-downloader.js";
 import { KaggleSource } from "../metadata/kaggle-source.js";
 import { OpenMLSource } from "../metadata/openml-source.js";
@@ -63,25 +64,42 @@ export class DataIngestor {
         if (source === "huggingface") {
             onProgress?.("Discovering data files on HuggingFace Hub...");
             const remotePath = await this.hfDownloader.findBestFile(datasetId);
-            if (!remotePath)
-                throw new Error(`No suitable data files found in HuggingFace repo: ${datasetId}`);
-            const ext = path.extname(remotePath).substring(1).toLowerCase() || "csv";
-            const targetPath = this.getTargetPath(datasetId, ext);
-            this.store.registerDownload(datasetId, targetPath, "downloading");
-            try {
-                await this.hfDownloader.download(datasetId, remotePath, targetPath, (progress) => {
-                    onProgress?.(`Downloading ${path.basename(remotePath)}...`, progress);
-                });
-                const resolvedPath = await this.hfDownloader.resolveExternalDataFromMetadata(targetPath, (progress) => {
-                    onProgress?.("Resolving external dataset file...", progress);
-                });
-                const stats = fs.statSync(resolvedPath);
-                this.completeDownload(datasetId, resolvedPath, stats.size);
-                return resolvedPath;
+            if (remotePath) {
+                // Direct file download path (repo has raw data files)
+                const ext = path.extname(remotePath).substring(1).toLowerCase() || "csv";
+                const targetPath = this.getTargetPath(datasetId, ext);
+                this.store.registerDownload(datasetId, targetPath, "downloading");
+                try {
+                    await this.hfDownloader.download(datasetId, remotePath, targetPath, (progress) => {
+                        onProgress?.(`Downloading ${path.basename(remotePath)}...`, progress);
+                    });
+                    const resolvedPath = await this.hfDownloader.resolveExternalDataFromMetadata(targetPath, (progress) => {
+                        onProgress?.("Resolving external dataset file...", progress);
+                    });
+                    const stats = fs.statSync(resolvedPath);
+                    this.completeDownload(datasetId, resolvedPath, stats.size);
+                    return resolvedPath;
+                }
+                catch (e) {
+                    this.failDownload(datasetId, e.message);
+                    throw e;
+                }
             }
-            catch (e) {
-                this.failDownload(datasetId, e.message);
-                throw e;
+            else {
+                // Fallback: Use Python datasets library to download and convert
+                onProgress?.("No raw files found. Using HuggingFace datasets library to download...");
+                const targetPath = this.getTargetPath(datasetId, "parquet");
+                this.store.registerDownload(datasetId, targetPath, "downloading");
+                try {
+                    const result = await this.hfDatasetsFallback(datasetId, targetPath, onProgress);
+                    const stats = fs.statSync(result);
+                    this.completeDownload(datasetId, result, stats.size);
+                    return result;
+                }
+                catch (e) {
+                    this.failDownload(datasetId, e.message);
+                    throw e;
+                }
             }
         }
         else if (source === "kaggle") {
@@ -159,4 +177,85 @@ export class DataIngestor {
         const safeId = datasetId.replace(/\//g, "_").replace(/:/g, "_");
         return path.join(this.rawDataDir, `${safeId}.${extension}`);
     }
+    /**
+     * Fallback: Use Python `datasets` library to download a HuggingFace dataset
+     * when no raw data files are found in the repo file listing.
+     */
+    async hfDatasetsFallback(datasetId, targetPath, onProgress) {
+        const pyCmd = process.platform === "win32" ? "py" : "python";
+        // Resolve the fallback script path
+        const homeDir = process.env.HOME || process.env.USERPROFILE || this.projectRoot;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptCandidates = [
+            path.resolve(dataRoot, "python", "hf_fallback.py"),
+            path.resolve(this.projectRoot, "python", "hf_fallback.py"),
+            path.resolve(this.projectRoot, "..", "src", "python", "hf_fallback.py"),
+            path.resolve(this.projectRoot, "..", "python", "hf_fallback.py"),
+        ];
+        let scriptPath = scriptCandidates.find(p => fs.existsSync(p));
+        if (!scriptPath) {
+            scriptPath = scriptCandidates[0]; // Will fail with a clear error
+        }
+        const token = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN || undefined;
+        const payload = {
+            repo_id: datasetId,
+            output_path: targetPath,
+            token: token || null,
+            max_rows: 500000,
+        };
+        onProgress?.("Downloading via datasets library (this may take a moment)...", 30);
+        return new Promise((resolve, reject) => {
+            const proc = spawn(pyCmd, [scriptPath, JSON.stringify(payload)], {
+                env: {
+                    ...process.env,
+                    PYTHONUTF8: "1",
+                    PIP_DISABLE_PIP_VERSION_CHECK: "1",
+                },
+            });
+            let stdout = "";
+            let stderr = "";
+            proc.stdout.on("data", (d) => (stdout += d.toString()));
+            proc.stderr.on("data", (d) => {
+                const msg = d.toString();
+                stderr += msg;
+                // Forward progress info
+                if (msg.includes("Downloading") || msg.includes("Loading")) {
+                    onProgress?.(msg.trim().split("\n").pop() || "Downloading...", 50);
+                }
+            });
+            const timer = setTimeout(() => {
+                try {
+                    proc.kill();
+                }
+                catch { /* no-op */ }
+                reject(new Error(`HuggingFace datasets download timed out after 10 minutes for ${datasetId}`));
+            }, 600000); // 10 min timeout
+            proc.on("close", (code) => {
+                clearTimeout(timer);
+                if (code !== 0) {
+                    let errorMsg = stderr || stdout || `Python exited with code ${code}`;
+                    try {
+                        const parsed = JSON.parse(stdout);
+                        if (parsed.error)
+                            errorMsg = parsed.error;
+                    }
+                    catch { /* use stderr */ }
+                    reject(new Error(`HuggingFace datasets fallback failed: ${errorMsg}`));
+                    return;
+                }
+                try {
+                    const result = JSON.parse(stdout);
+                    if (!result.ok) {
+                        reject(new Error(result.error || "Unknown error from HF fallback"));
+                        return;
+                    }
+                    onProgress?.(`Downloaded ${result.rows?.toLocaleString() || "?"} rows (${result.columns?.length || "?"} columns)`, 90);
+                    resolve(result.path);
+                }
+                catch {
+                    reject(new Error(`Failed to parse HF fallback output: ${stdout}`));
+                }
+            });
+        });
+    }
 }

package/build/python/hf_fallback.py

@@ -0,0 +1,147 @@
+"""
+HuggingFace Datasets Library Fallback Downloader.
+
+Used when the HF Hub file listing finds no suitable data files
+(e.g. script-based datasets, gated datasets, datasets that use
+the `datasets` library format).
+
+Usage:
+    python hf_fallback.py '{"repo_id": "user/dataset", "output_path": "/path/to/output.parquet", "token": "optional", "max_rows": 100000, "split": "train"}'
+
+Output: JSON to stdout
+    {"ok": true, "path": "/path/to/output.parquet", "rows": 12345, "columns": ["col1", "col2"]}
+    {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"ok": False, "error": "Missing payload argument"}))
+        sys.exit(1)
+
+    try:
+        payload = json.loads(sys.argv[1])
+    except json.JSONDecodeError as e:
+        print(json.dumps({"ok": False, "error": f"Invalid JSON payload: {e}"}))
+        sys.exit(1)
+
+    repo_id = payload.get("repo_id", "").strip()
+    output_path = payload.get("output_path", "").strip()
+    token = payload.get("token") or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or None
+    max_rows = payload.get("max_rows", 500000)
+    split = payload.get("split")  # None = auto-detect
+
+    if not repo_id:
+        print(json.dumps({"ok": False, "error": "repo_id is required"}))
+        sys.exit(1)
+
+    if not output_path:
+        print(json.dumps({"ok": False, "error": "output_path is required"}))
+        sys.exit(1)
+
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        print(json.dumps({"ok": False, "error": "Python 'datasets' library not installed. Install with: pip install datasets"}))
+        sys.exit(1)
+
+    try:
+        import polars as pl
+    except ImportError:
+        pl = None
+
+    try:
+        # Try loading with streaming first (memory-efficient)
+        # If split is not specified, try common ones
+        splits_to_try = [split] if split else ["train", "test", "validation", None]
+        
+        ds = None
+        used_split = None
+        
+        for s in splits_to_try:
+            try:
+                kwargs = {
+                    "path": repo_id,
+                    "trust_remote_code": True,
+                }
+                if token:
+                    kwargs["token"] = token
+                if s:
+                    kwargs["split"] = s
+
+                ds = load_dataset(**kwargs)
+                used_split = s
+                break
+            except (ValueError, KeyError):
+                # Split doesn't exist, try next
+                continue
+            except Exception as e:
+                if "split" in str(e).lower() or "key" in str(e).lower():
+                    continue
+                raise
+
+        if ds is None:
+            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}'. No valid splits found."}))
+            sys.exit(1)
+
+        # Handle DatasetDict (when no split specified)
+        from datasets import DatasetDict, Dataset
+        if isinstance(ds, DatasetDict):
+            # Pick the best split
+            for preferred in ["train", "test", "validation"]:
+                if preferred in ds:
+                    ds = ds[preferred]
+                    used_split = preferred
+                    break
+            else:
+                # Just pick the first available split
+                first_key = list(ds.keys())[0]
+                ds = ds[first_key]
+                used_split = first_key
+
+        # Limit rows if needed
+        total_rows = len(ds)
+        if max_rows and total_rows > max_rows:
+            ds = ds.select(range(max_rows))
+            total_rows = max_rows
+
+        # Ensure output directory exists
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+        # Export to parquet
+        columns = ds.column_names
+
+        if output_path.endswith(".parquet"):
+            ds.to_parquet(output_path)
+        elif output_path.endswith(".csv"):
+            ds.to_csv(output_path)
+        else:
+            # Default to parquet
+            if not output_path.endswith(".parquet"):
+                output_path = output_path + ".parquet"
+            ds.to_parquet(output_path)
+
+        print(json.dumps({
+            "ok": True,
+            "path": output_path,
+            "rows": total_rows,
+            "columns": columns,
+            "split": used_split
+        }))
+
+    except Exception as e:
+        error_msg = str(e)
+        # Provide helpful hints
+        if "401" in error_msg or "403" in error_msg or "gated" in error_msg.lower():
+            error_msg += " (This dataset may be gated/private. Set HF_TOKEN via configure_keys tool.)"
+        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower():
+            error_msg = f"Dataset '{repo_id}' not found on HuggingFace. Check the dataset ID."
+
+        print(json.dumps({"ok": False, "error": error_msg}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vespermcp/mcp-server",
-  "version": "1.2.19",
+  "version": "1.2.20",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",

package/src/python/hf_fallback.py

@@ -0,0 +1,147 @@
+"""
+HuggingFace Datasets Library Fallback Downloader.
+
+Used when the HF Hub file listing finds no suitable data files
+(e.g. script-based datasets, gated datasets, datasets that use
+the `datasets` library format).
+
+Usage:
+    python hf_fallback.py '{"repo_id": "user/dataset", "output_path": "/path/to/output.parquet", "token": "optional", "max_rows": 100000, "split": "train"}'
+
+Output: JSON to stdout
+    {"ok": true, "path": "/path/to/output.parquet", "rows": 12345, "columns": ["col1", "col2"]}
+    {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"ok": False, "error": "Missing payload argument"}))
+        sys.exit(1)
+
+    try:
+        payload = json.loads(sys.argv[1])
+    except json.JSONDecodeError as e:
+        print(json.dumps({"ok": False, "error": f"Invalid JSON payload: {e}"}))
+        sys.exit(1)
+
+    repo_id = payload.get("repo_id", "").strip()
+    output_path = payload.get("output_path", "").strip()
+    token = payload.get("token") or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or None
+    max_rows = payload.get("max_rows", 500000)
+    split = payload.get("split")  # None = auto-detect
+
+    if not repo_id:
+        print(json.dumps({"ok": False, "error": "repo_id is required"}))
+        sys.exit(1)
+
+    if not output_path:
+        print(json.dumps({"ok": False, "error": "output_path is required"}))
+        sys.exit(1)
+
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        print(json.dumps({"ok": False, "error": "Python 'datasets' library not installed. Install with: pip install datasets"}))
+        sys.exit(1)
+
+    try:
+        import polars as pl
+    except ImportError:
+        pl = None
+
+    try:
+        # Try loading with streaming first (memory-efficient)
+        # If split is not specified, try common ones
+        splits_to_try = [split] if split else ["train", "test", "validation", None]
+        
+        ds = None
+        used_split = None
+        
+        for s in splits_to_try:
+            try:
+                kwargs = {
+                    "path": repo_id,
+                    "trust_remote_code": True,
+                }
+                if token:
+                    kwargs["token"] = token
+                if s:
+                    kwargs["split"] = s
+
+                ds = load_dataset(**kwargs)
+                used_split = s
+                break
+            except (ValueError, KeyError):
+                # Split doesn't exist, try next
+                continue
+            except Exception as e:
+                if "split" in str(e).lower() or "key" in str(e).lower():
+                    continue
+                raise
+
+        if ds is None:
+            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}'. No valid splits found."}))
+            sys.exit(1)
+
+        # Handle DatasetDict (when no split specified)
+        from datasets import DatasetDict, Dataset
+        if isinstance(ds, DatasetDict):
+            # Pick the best split
+            for preferred in ["train", "test", "validation"]:
+                if preferred in ds:
+                    ds = ds[preferred]
+                    used_split = preferred
+                    break
+            else:
+                # Just pick the first available split
+                first_key = list(ds.keys())[0]
+                ds = ds[first_key]
+                used_split = first_key
+
+        # Limit rows if needed
+        total_rows = len(ds)
+        if max_rows and total_rows > max_rows:
+            ds = ds.select(range(max_rows))
+            total_rows = max_rows
+
+        # Ensure output directory exists
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+        # Export to parquet
+        columns = ds.column_names
+
+        if output_path.endswith(".parquet"):
+            ds.to_parquet(output_path)
+        elif output_path.endswith(".csv"):
+            ds.to_csv(output_path)
+        else:
+            # Default to parquet
+            if not output_path.endswith(".parquet"):
+                output_path = output_path + ".parquet"
+            ds.to_parquet(output_path)
+
+        print(json.dumps({
+            "ok": True,
+            "path": output_path,
+            "rows": total_rows,
+            "columns": columns,
+            "split": used_split
+        }))
+
+    except Exception as e:
+        error_msg = str(e)
+        # Provide helpful hints
+        if "401" in error_msg or "403" in error_msg or "gated" in error_msg.lower():
+            error_msg += " (This dataset may be gated/private. Set HF_TOKEN via configure_keys tool.)"
+        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower():
+            error_msg = f"Dataset '{repo_id}' not found on HuggingFace. Check the dataset ID."
+
+        print(json.dumps({"ok": False, "error": error_msg}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()