@vespermcp/mcp-server 1.2.2 → 1.2.4

package/build/index.js

@@ -12,6 +12,8 @@ import { Embedder } from "./search/embedder.js";
 import { SearchEngine } from "./search/engine.js";
 import { HuggingFaceScraper } from "./metadata/scraper.js";
 import { KaggleSource } from "./metadata/kaggle-source.js";
+import { OpenMLSource } from "./metadata/openml-source.js";
+import { DataWorldSource } from "./metadata/dataworld-source.js";
 import { formatSearchResults, formatDatasetInfo, formatJobStatus } from "./tools/formatter.js";
 import { JobManager } from "./jobs/manager.js";
 import { QualityAnalyzer } from "./quality/analyzer.js";
@@ -228,6 +230,12 @@ function hydrateExternalKeys() {
     if (!process.env.KAGGLE_KEY && keys.kaggle_key) {
         process.env.KAGGLE_KEY = String(keys.kaggle_key);
     }
+    if (!process.env.DW_AUTH_TOKEN && keys.dataworld_token) {
+        process.env.DW_AUTH_TOKEN = String(keys.dataworld_token);
+    }
+}
+function hasDataWorldToken() {
+    return !!(process.env.DW_AUTH_TOKEN || secureKeys.getAll().dataworld_token);
 }
 // CRITICAL FIX: Pass __dirname (build directory) to analyzers
 // Python scripts are in build/python/, so analyzers should look relative to build/
@@ -429,7 +437,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                         },
                         source: {
                             type: "string",
-                            enum: ["huggingface", "kaggle"],
+                            enum: ["huggingface", "kaggle", "openml", "dataworld"],
                             description: "Data source to discover from.",
                         },
                         limit: {
@@ -448,7 +456,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                     properties: {
                         source: {
                             type: "string",
-                            enum: ["huggingface", "kaggle"],
+                            enum: ["huggingface", "kaggle", "openml", "dataworld"],
                             description: "Dataset source.",
                         },
                         dataset_id: {
@@ -477,13 +485,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
             },
             {
                 name: "configure_keys",
-                description: "One-time optional key setup for external sources (Kaggle + gated HF). Core tools do not require keys.",
+                description: "One-time optional key setup for external sources (Kaggle, data.world, gated HF). Core tools do not require keys.",
                 inputSchema: {
                     type: "object",
                     properties: {
                         hf_token: { type: "string", description: "Optional Hugging Face token for gated/private datasets" },
                         kaggle_username: { type: "string", description: "Optional Kaggle username" },
-                        kaggle_key: { type: "string", description: "Optional Kaggle API key" }
+                        kaggle_key: { type: "string", description: "Optional Kaggle API key" },
+                        dataworld_token: { type: "string", description: "Optional data.world API token" }
                     },
                 },
             },
@@ -495,7 +504,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                     properties: {
                         dataset_id: {
                             type: "string",
-                            description: "The unique dataset ID (e.g., 'user/dataset_name' for HuggingFace or 'kaggle:username/dataset' for Kaggle)",
+                            description: "The unique dataset ID (e.g., 'user/dataset_name' for HuggingFace, 'kaggle:username/dataset' for Kaggle, 'openml:1234' for OpenML, or 'dataworld:owner/id' for data.world)",
                         },
                     },
                     required: ["dataset_id"],
@@ -786,6 +795,20 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     }
                     results = await kaggleSource.discover(query, limit);
                 }
+                else if (source === "openml") {
+                    const openmlSource = new OpenMLSource();
+                    results = await openmlSource.discover(query, limit);
+                }
+                else if (source === "dataworld") {
+                    if (!hasDataWorldToken()) {
+                        return {
+                            content: [{ type: "text", text: "data.world requires API token. Run 'vespermcp config keys' and set dataworld_token." }],
+                            isError: true,
+                        };
+                    }
+                    const dataworldSource = new DataWorldSource();
+                    results = await dataworldSource.discover(query, limit);
+                }
                 else {
                     const hf = new HuggingFaceScraper();
                     results = await hf.scrape(Math.max(1, limit), true, query);
@@ -815,6 +838,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     isError: true,
                 };
             }
+            if (source === "dataworld" && !hasDataWorldToken()) {
+                return {
+                    content: [{ type: "text", text: "data.world requires API token. Run 'vespermcp config keys' and set dataworld_token." }],
+                    isError: true,
+                };
+            }
             try {
                 const localPath = await dataIngestor.ensureData(datasetId, source, () => undefined);
                 return {
@@ -846,6 +875,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             const hfToken = String(request.params.arguments?.hf_token || "").trim();
             const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
             const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
+            const dataworldToken = String(request.params.arguments?.dataworld_token || "").trim();
             const saved = [];
             const methods = [];
             if (hfToken) {
@@ -875,6 +905,15 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         methods.push(r.method);
                 }
             }
+            if (dataworldToken) {
+                const r = secureKeys.set("dataworld_token", dataworldToken);
+                if (r.ok) {
+                    process.env.DW_AUTH_TOKEN = dataworldToken;
+                    saved.push("data.world token");
+                    if (r.method)
+                        methods.push(r.method);
+                }
+            }
             if (saved.length === 0) {
                 return {
                     content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
@@ -1402,6 +1441,7 @@ async function runConfigCli(args) {
         const hfToken = (await ask(`Hugging Face token [${current.hf_token ? "saved" : "empty"}]: `)).trim();
         const kaggleUsername = (await ask(`Kaggle username [${current.kaggle_username ? "saved" : "empty"}]: `)).trim();
         const kaggleKey = (await ask(`Kaggle key [${current.kaggle_key ? "saved" : "empty"}]: `)).trim();
+        const dataworldToken = (await ask(`data.world token [${current.dataworld_token ? "saved" : "empty"}]: `)).trim();
         rl.close();
         const saved = [];
         if (hfToken) {
@@ -1425,12 +1465,19 @@ async function runConfigCli(args) {
                 saved.push("Kaggle key");
             }
         }
+        if (dataworldToken) {
+            const res = secureKeys.set("dataworld_token", dataworldToken);
+            if (res.ok) {
+                process.env.DW_AUTH_TOKEN = dataworldToken;
+                saved.push("data.world token");
+            }
+        }
         if (saved.length === 0) {
             console.log("No new keys saved (all skipped). Core tools continue to work without keys.");
             return;
         }
         console.log(`Key(s) saved securely: ${saved.join(", ")}`);
-        console.log("You can now use Kaggle and gated Hugging Face datasets.");
+        console.log("You can now use Kaggle, data.world, and gated Hugging Face datasets.");
         return;
     }
     // Backward-compatible Kaggle-specific path
@@ -1479,7 +1526,7 @@ async function runDiscoverCli(args) {
     }
     const query = queryParts.join(" ").trim();
     if (!query) {
-        console.error("Usage: vespermcp discover --source kaggle \"credit risk\" --limit 10");
+        console.error("Usage: vespermcp discover --source <huggingface|kaggle|openml|dataworld> \"credit risk\" --limit 10");
         process.exit(1);
     }
     if (source === "kaggle") {
@@ -1512,6 +1559,34 @@ async function runDiscoverCli(args) {
         }
         return;
     }
+    else if (source === "openml") {
+        try {
+            const openmlSource = new OpenMLSource();
+            const results = await openmlSource.discover(query, limit);
+            console.log(formatSearchResults(results));
+        }
+        catch (error) {
+            console.error(`OpenML discover failed: ${error.message || error}`);
+            process.exit(1);
+        }
+        return;
+    }
+    else if (source === "dataworld") {
+        if (!hasDataWorldToken()) {
+            console.error("data.world requires API token. Run 'vespermcp config keys' and set dataworld_token.");
+            process.exit(1);
+        }
+        try {
+            const dataworldSource = new DataWorldSource();
+            const results = await dataworldSource.discover(query, limit);
+            console.log(formatSearchResults(results));
+        }
+        catch (error) {
+            console.error(`data.world discover failed: ${error.message || error}`);
+            process.exit(1);
+        }
+        return;
+    }
     const hf = new HuggingFaceScraper();
     const results = await hf.scrape(limit, true, query);
     console.log(formatSearchResults(results));
@@ -1530,7 +1605,7 @@ async function runDownloadCli(args) {
     const source = (nonFlags[1] || "").toLowerCase();
     const datasetId = nonFlags[2] || "";
     if (!source || !datasetId) {
-        console.error("Usage: vespermcp download kaggle <username/dataset-name> [--target-dir C:/path]");
+        console.error("Usage: vespermcp download <huggingface|kaggle|openml|dataworld> <dataset-id> [--target-dir C:/path]");
         process.exit(1);
     }
     if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
@@ -1546,6 +1621,10 @@ async function runDownloadCli(args) {
         if (!dataIngestor.hasKaggleCredentials())
             process.exit(1);
     }
+    if (source === "dataworld" && !hasDataWorldToken()) {
+        console.error("data.world requires API token. Run 'vespermcp config keys' and set dataworld_token.");
+        process.exit(1);
+    }
     let localPath = "";
     try {
         if (source === "kaggle" && targetDir) {

package/build/ingestion/ingestor.js

@@ -2,6 +2,8 @@ import path from "path";
 import fs from "fs";
 import { HFDownloader } from "./hf-downloader.js";
 import { KaggleSource } from "../metadata/kaggle-source.js";
+import { OpenMLSource } from "../metadata/openml-source.js";
+import { DataWorldSource } from "../metadata/dataworld-source.js";
 import { SecureKeysManager } from "../config/secure-keys.js";
 export class DataIngestor {
     projectRoot;
@@ -9,6 +11,8 @@ export class DataIngestor {
     rawDataDir;
     hfDownloader;
     kaggleSource;
+    openmlSource;
+    dataworldSource;
     secureKeys;
     constructor(projectRoot, store) {
         this.projectRoot = projectRoot;
@@ -19,6 +23,8 @@ export class DataIngestor {
         }
         this.hfDownloader = new HFDownloader();
         this.kaggleSource = new KaggleSource();
+        this.openmlSource = new OpenMLSource();
+        this.dataworldSource = new DataWorldSource();
         this.secureKeys = new SecureKeysManager();
     }
     /**
@@ -96,6 +102,38 @@ export class DataIngestor {
                 throw e;
             }
         }
+        else if (source === "openml") {
+            const targetDir = path.join(this.rawDataDir, datasetId.replace(/:/g, "_"));
+            this.store.registerDownload(datasetId, targetDir, "downloading");
+            try {
+                onProgress?.("Downloading from OpenML...");
+                const result = await this.openmlSource.download(datasetId, targetDir);
+                const stats = fs.statSync(result.local_path);
+                this.completeDownload(datasetId, result.local_path, stats.size);
+                onProgress?.("OpenML download complete", 100);
+                return result.local_path;
+            }
+            catch (e) {
+                this.failDownload(datasetId, e.message);
+                throw e;
+            }
+        }
+        else if (source === "dataworld") {
+            const targetDir = path.join(this.rawDataDir, datasetId.replace(/[:\/]/g, "_"));
+            this.store.registerDownload(datasetId, targetDir, "downloading");
+            try {
+                onProgress?.("Downloading from data.world...");
+                const result = await this.dataworldSource.download(datasetId, targetDir);
+                const stats = fs.statSync(result.local_path);
+                this.completeDownload(datasetId, result.local_path, stats.size);
+                onProgress?.("data.world download complete", 100);
+                return result.local_path;
+            }
+            catch (e) {
+                this.failDownload(datasetId, e.message);
+                throw e;
+            }
+        }
         throw new Error(`Download logic for ${source} not yet implemented`);
     }
     /**

package/build/metadata/dataworld-source.js

@@ -0,0 +1,89 @@
+import { spawn } from "child_process";
+import path from "path";
+import fs from "fs";
+import os from "os";
+export class DataWorldSource {
+    pythonPath = "python";
+    scriptPath;
+    constructor(buildDir = process.cwd()) {
+        const homeDir = os.homedir() || process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptPath0 = path.resolve(dataRoot, "python", "dataworld_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "dataworld_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "dataworld_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else {
+            this.scriptPath = scriptPath0;
+        }
+        if (process.platform === "win32") {
+            const venvPy = path.resolve(buildDir, ".venv", "Scripts", "python.exe");
+            if (fs.existsSync(venvPy)) {
+                this.pythonPath = venvPy;
+            }
+            else {
+                this.pythonPath = "py";
+            }
+        }
+        else {
+            const venvPy = path.resolve(buildDir, ".venv", "bin", "python");
+            if (fs.existsSync(venvPy)) {
+                this.pythonPath = venvPy;
+            }
+        }
+    }
+    async discover(query, limit = 20) {
+        const result = await this.run(["discover", query, String(limit)]);
+        if (!result.ok) {
+            throw new Error(result.error || "data.world discover failed");
+        }
+        return (result.results || []);
+    }
+    async download(datasetRef, targetDir) {
+        const args = ["download", datasetRef];
+        if (targetDir)
+            args.push(targetDir);
+        const result = await this.run(args);
+        if (!result.ok) {
+            throw new Error(result.error || "data.world download failed");
+        }
+        return {
+            local_path: result.local_path,
+            target_dir: result.target_dir,
+        };
+    }
+    run(args) {
+        return new Promise((resolve, reject) => {
+            const proc = spawn(this.pythonPath, [this.scriptPath, ...args], {
+                env: process.env
+            });
+            let stdout = "";
+            let stderr = "";
+            proc.stdout.on("data", (data) => {
+                stdout += data.toString();
+            });
+            proc.stderr.on("data", (data) => {
+                stderr += data.toString();
+            });
+            proc.on("close", (code) => {
+                if (code !== 0) {
+                    return reject(new Error(`data.world engine exited with code ${code}: ${stderr}`));
+                }
+                try {
+                    const parsed = JSON.parse(stdout.trim());
+                    resolve(parsed);
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse data.world engine output: ${stdout}`));
+                }
+            });
+        });
+    }
+}

package/build/metadata/openml-source.js

@@ -0,0 +1,87 @@
+import { spawn } from "child_process";
+import path from "path";
+import fs from "fs";
+import os from "os";
+export class OpenMLSource {
+    pythonPath = "python";
+    scriptPath;
+    constructor(buildDir = process.cwd()) {
+        const homeDir = os.homedir() || process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptPath0 = path.resolve(dataRoot, "python", "openml_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "openml_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "openml_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else {
+            this.scriptPath = scriptPath0;
+        }
+        if (process.platform === "win32") {
+            const venvPy = path.resolve(buildDir, ".venv", "Scripts", "python.exe");
+            if (fs.existsSync(venvPy)) {
+                this.pythonPath = venvPy;
+            }
+            else {
+                this.pythonPath = "py";
+            }
+        }
+        else {
+            const venvPy = path.resolve(buildDir, ".venv", "bin", "python");
+            if (fs.existsSync(venvPy)) {
+                this.pythonPath = venvPy;
+            }
+        }
+    }
+    async discover(query, limit = 20) {
+        const result = await this.run(["discover", query, String(limit)]);
+        if (!result.ok) {
+            throw new Error(result.error || "OpenML discover failed");
+        }
+        return (result.results || []);
+    }
+    async download(datasetRef, targetDir) {
+        const args = ["download", datasetRef];
+        if (targetDir)
+            args.push(targetDir);
+        const result = await this.run(args);
+        if (!result.ok) {
+            throw new Error(result.error || "OpenML download failed");
+        }
+        return {
+            local_path: result.local_path,
+            target_dir: result.target_dir,
+        };
+    }
+    run(args) {
+        return new Promise((resolve, reject) => {
+            const proc = spawn(this.pythonPath, [this.scriptPath, ...args]);
+            let stdout = "";
+            let stderr = "";
+            proc.stdout.on("data", (data) => {
+                stdout += data.toString();
+            });
+            proc.stderr.on("data", (data) => {
+                stderr += data.toString();
+            });
+            proc.on("close", (code) => {
+                if (code !== 0) {
+                    return reject(new Error(`OpenML engine exited with code ${code}: ${stderr}`));
+                }
+                try {
+                    const parsed = JSON.parse(stdout.trim());
+                    resolve(parsed);
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse OpenML engine output: ${stdout}`));
+                }
+            });
+        });
+    }
+}

package/build/python/config.py

@@ -13,6 +13,7 @@ KEY_ALIASES = {
     "hf_token": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
     "kaggle_username": ["KAGGLE_USERNAME"],
     "kaggle_key": ["KAGGLE_KEY"],
+    "dataworld_token": ["DW_AUTH_TOKEN"],
 }
 
 try:
@@ -207,6 +208,7 @@ def get_all() -> Dict[str, Optional[str]]:
         "hf_token": get_key("hf_token"),
         "kaggle_username": get_key("kaggle_username"),
         "kaggle_key": get_key("kaggle_key"),
+        "dataworld_token": get_key("dataworld_token"),
     }
 
 

package/build/python/dataworld_engine.py

@@ -0,0 +1,208 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+import urllib.request
+import urllib.error
+import urllib.parse
+from typing import Dict, Any, List
+
+def _get_token() -> str:
+    token = os.environ.get("DW_AUTH_TOKEN")
+    if not token:
+        raise ValueError("DW_AUTH_TOKEN environment variable is required for data.world")
+    return token
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    owner_field = ds.get("owner", "")
+    if isinstance(owner_field, dict):
+        owner = owner_field.get("id") or owner_field.get("name") or ""
+    else:
+        owner = owner_field or ""
+
+    id_str = ds.get("id", "")
+    title = ds.get("title", "")
+
+    if (not owner or not id_str) and isinstance(ds.get("resourceLink"), str):
+        # Expected format includes /<owner>/<dataset-id>
+        parts = ds["resourceLink"].strip("/").split("/")
+        if len(parts) >= 2:
+            owner = owner or parts[-2]
+            id_str = id_str or parts[-1]
+
+    if isinstance(id_str, str) and "/" in id_str and not owner:
+        split_ref = id_str.split("/", 1)
+        owner = split_ref[0]
+        id_str = split_ref[1]
+
+    if not owner and not id_str:
+        owner = "unknown"
+        id_str = "unknown"
+
+    if not title:
+        title = f"{owner}/{id_str}"
+    
+    return {
+        "id": f"dataworld:{owner}/{id_str}",
+        "name": title,
+        "source": "dataworld",
+        "description": ds.get("description", f"data.world dataset {title}"),
+        "author": owner,
+        "license": {
+            "id": "Unknown",
+            "category": "unknown",
+            "commercial_use": None,
+            "warnings": []
+        },
+        "tags": ds.get("tags", []) + ["dataworld"],
+        "downloads": 0,
+        "likes": 0,
+        "created_at": ds.get("created", ""),
+        "updated_at": ds.get("updated", ""),
+        "size_bytes": 0,
+        "quality_score": 0.8,
+        "domain": "general",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": len(ds.get("description", "")),
+        "has_readme": False,
+        "download_url": f"https://data.world/{owner}/{id_str}",
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # data.world simple search API
+        url = f"https://api.data.world/v0/search/resources?size={limit}"
+        
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/json"
+        }
+        
+        # Search datasets and include community results to improve recall
+        body = {
+            "query": query,
+            "category": ["dataset"],
+            "includeCommunityResults": True,
+        }
+        
+        req = urllib.request.Request(url, data=json.dumps(body).encode('utf-8'), headers=headers, method="POST")
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode('utf-8'))
+            
+        records = data.get("records", [])
+
+        # Fallback to advanced endpoint if simple search returns nothing
+        if not records:
+            adv_url = f"https://api.data.world/v0/search?size={limit}"
+            adv_body = {
+                "query": query,
+                "category": ["dataset"],
+            }
+            adv_req = urllib.request.Request(
+                adv_url,
+                data=json.dumps(adv_body).encode("utf-8"),
+                headers=headers,
+                method="POST",
+            )
+            with urllib.request.urlopen(adv_req) as response:
+                adv_data = json.loads(response.read().decode("utf-8"))
+            records = adv_data.get("records", [])
+
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"data.world discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # dataset_ref is expected to be "dataworld:owner/id"
+        if dataset_ref.startswith("dataworld:"):
+            ref = dataset_ref.split(":", 1)[1]
+        else:
+            ref = dataset_ref
+            
+        parts = ref.split("/")
+        if len(parts) != 2:
+            return {"ok": False, "error": f"Invalid data.world dataset ID format. Expected owner/id, got {ref}"}
+            
+        owner, dataset_id = parts
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_dataworld_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # First, get the dataset metadata to find the files
+        url = f"https://api.data.world/v0/datasets/{owner}/{dataset_id}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json"
+        }
+        
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req) as response:
+            dataset_meta = json.loads(response.read().decode('utf-8'))
+            
+        files = dataset_meta.get("files", [])
+        if not files:
+            return {"ok": False, "error": "No files found in this dataset"}
+            
+        # Find the best file to download (prefer csv, parquet, jsonl)
+        best_file = None
+        for ext in [".parquet", ".csv", ".jsonl", ".json"]:
+            for f in files:
+                if f.get("name", "").lower().endswith(ext):
+                    best_file = f
+                    break
+            if best_file:
+                break
+                
+        if not best_file:
+            best_file = files[0] # Just take the first one if no preferred format
+            
+        filename = best_file.get("name")
+        
+        # Download the file
+        download_url = f"https://api.data.world/v0/file_download/{owner}/{dataset_id}/{urllib.parse.quote(filename)}"
+        
+        file_path = os.path.join(target_dir, filename)
+        
+        download_req = urllib.request.Request(download_url, headers=headers)
+        with urllib.request.urlopen(download_req) as response, open(file_path, 'wb') as out_file:
+            out_file.write(response.read())
+            
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"data.world download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper data.world Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/build/python/kaggle_engine.py

@@ -109,8 +109,28 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
 
     api: KaggleApi = auth["api"]
     try:
-        datasets = api.dataset_list(search=query, page_size=max(1, min(limit, 100)))
-        items = [_dataset_to_dict(ds) for ds in datasets[:limit]]
+        desired = max(1, min(limit, 100))
+
+        try:
+            datasets = api.dataset_list(search=query, page_size=desired)
+            items = [_dataset_to_dict(ds) for ds in datasets[:limit]]
+            return {"ok": True, "results": items, "count": len(items)}
+        except TypeError:
+            pass
+
+        collected = []
+        page = 1
+        while len(collected) < limit:
+            page_items = api.dataset_list(search=query, page=page)
+            if not page_items:
+                break
+
+            collected.extend(page_items)
+            if len(page_items) < 20:
+                break
+            page += 1
+
+        items = [_dataset_to_dict(ds) for ds in collected[:limit]]
         return {"ok": True, "results": items, "count": len(items)}
     except Exception as e:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}

package/build/python/openml_engine.py

@@ -0,0 +1,146 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+from typing import Dict, Any, List
+
+try:
+    import openml
+except ImportError:
+    openml = None
+
+def _ensure_openml() -> Dict[str, Any]:
+    if openml is None:
+        return {"ok": False, "error": "openml package is not installed. Run 'pip install openml'"}
+    return {"ok": True}
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    # OpenML dataset dict from list_datasets
+    did = ds.get("did", "")
+    name = ds.get("name", f"dataset_{did}")
+    version = ds.get("version", "1")
+    status = ds.get("status", "active")
+    format = ds.get("format", "unknown")
+    
+    # Map to Vesper DatasetMetadata format
+    return {
+        "id": f"openml:{did}",
+        "name": name,
+        "source": "openml",
+        "description": f"OpenML Dataset {name} (ID: {did}, Version: {version}, Format: {format}, Status: {status})",
+        "author": "OpenML Community",
+        "license": "Public",
+        "tags": ["openml", format.lower()],
+        "downloads": ds.get("NumberOfDownloads", 0),
+        "likes": ds.get("NumberOfLikes", 0),
+        "created_at": ds.get("upload_date", ""),
+        "updated_at": ds.get("upload_date", ""),
+        "size_bytes": 0, # Not always available in list
+        "quality_score": 0.8, # Default good score for OpenML
+        "domain": "machine_learning",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": 100,
+        "has_readme": False,
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    check = _ensure_openml()
+    if not check.get("ok"):
+        return check
+
+    try:
+        # OpenML list_datasets doesn't have a direct text search in the python API easily exposed without downloading all.
+        # But we can filter by tag or just get a list and filter locally if needed, or use the REST API directly.
+        # Actually, openml.datasets.list_datasets() returns a dict of datasets.
+        # We can fetch a larger batch and filter by name/keyword.
+        
+        # Fetching a batch of datasets
+        datasets = openml.datasets.list_datasets(output_format='dataframe')
+        
+        if query:
+            # Simple case-insensitive search in name
+            mask = datasets['name'].str.contains(query, case=False, na=False)
+            filtered = datasets[mask]
+        else:
+            filtered = datasets
+            
+        # Sort by NumberOfDownloads if available, else just take top
+        if 'NumberOfDownloads' in filtered.columns:
+            filtered = filtered.sort_values('NumberOfDownloads', ascending=False)
+            
+        top_k = filtered.head(limit)
+        
+        # Convert to list of dicts
+        records = top_k.to_dict(orient='records')
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"OpenML discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    check = _ensure_openml()
+    if not check.get("ok"):
+        return check
+
+    try:
+        # dataset_ref is expected to be "openml:ID"
+        if dataset_ref.startswith("openml:"):
+            did_str = dataset_ref.split(":")[1]
+        else:
+            did_str = dataset_ref
+            
+        did = int(did_str)
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_openml_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # Get the dataset
+        dataset = openml.datasets.get_dataset(did, download_data=True, download_qualities=False, download_features_meta_data=False)
+        
+        # Get the pandas dataframe
+        X, y, categorical_indicator, attribute_names = dataset.get_data(
+            dataset_format="dataframe"
+        )
+        
+        # If there's a target column (y), we might want to join it back if it was separated
+        # get_data() can return X and y separately if target is specified, but usually X contains everything if target=None
+        # Let's just get everything
+        df, _, _, _ = dataset.get_data(target=None, dataset_format="dataframe")
+        
+        # Save to parquet in the target directory
+        safe_name = "".join([c if c.isalnum() else "_" for c in dataset.name])
+        file_path = os.path.join(target_dir, f"{safe_name}_{did}.parquet")
+        
+        df.to_parquet(file_path, index=False)
+        
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"OpenML download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper OpenML Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vespermcp/mcp-server",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",

package/src/python/config.py

@@ -13,6 +13,7 @@ KEY_ALIASES = {
     "hf_token": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
     "kaggle_username": ["KAGGLE_USERNAME"],
     "kaggle_key": ["KAGGLE_KEY"],
+    "dataworld_token": ["DW_AUTH_TOKEN"],
 }
 
 try:
@@ -207,6 +208,7 @@ def get_all() -> Dict[str, Optional[str]]:
         "hf_token": get_key("hf_token"),
         "kaggle_username": get_key("kaggle_username"),
         "kaggle_key": get_key("kaggle_key"),
+        "dataworld_token": get_key("dataworld_token"),
     }
 
 

package/src/python/dataworld_engine.py

@@ -0,0 +1,208 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+import urllib.request
+import urllib.error
+import urllib.parse
+from typing import Dict, Any, List
+
+def _get_token() -> str:
+    token = os.environ.get("DW_AUTH_TOKEN")
+    if not token:
+        raise ValueError("DW_AUTH_TOKEN environment variable is required for data.world")
+    return token
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    owner_field = ds.get("owner", "")
+    if isinstance(owner_field, dict):
+        owner = owner_field.get("id") or owner_field.get("name") or ""
+    else:
+        owner = owner_field or ""
+
+    id_str = ds.get("id", "")
+    title = ds.get("title", "")
+
+    if (not owner or not id_str) and isinstance(ds.get("resourceLink"), str):
+        # Expected format includes /<owner>/<dataset-id>
+        parts = ds["resourceLink"].strip("/").split("/")
+        if len(parts) >= 2:
+            owner = owner or parts[-2]
+            id_str = id_str or parts[-1]
+
+    if isinstance(id_str, str) and "/" in id_str and not owner:
+        split_ref = id_str.split("/", 1)
+        owner = split_ref[0]
+        id_str = split_ref[1]
+
+    if not owner and not id_str:
+        owner = "unknown"
+        id_str = "unknown"
+
+    if not title:
+        title = f"{owner}/{id_str}"
+    
+    return {
+        "id": f"dataworld:{owner}/{id_str}",
+        "name": title,
+        "source": "dataworld",
+        "description": ds.get("description", f"data.world dataset {title}"),
+        "author": owner,
+        "license": {
+            "id": "Unknown",
+            "category": "unknown",
+            "commercial_use": None,
+            "warnings": []
+        },
+        "tags": ds.get("tags", []) + ["dataworld"],
+        "downloads": 0,
+        "likes": 0,
+        "created_at": ds.get("created", ""),
+        "updated_at": ds.get("updated", ""),
+        "size_bytes": 0,
+        "quality_score": 0.8,
+        "domain": "general",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": len(ds.get("description", "")),
+        "has_readme": False,
+        "download_url": f"https://data.world/{owner}/{id_str}",
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # data.world simple search API
+        url = f"https://api.data.world/v0/search/resources?size={limit}"
+        
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/json"
+        }
+        
+        # Search datasets and include community results to improve recall
+        body = {
+            "query": query,
+            "category": ["dataset"],
+            "includeCommunityResults": True,
+        }
+        
+        req = urllib.request.Request(url, data=json.dumps(body).encode('utf-8'), headers=headers, method="POST")
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode('utf-8'))
+            
+        records = data.get("records", [])
+
+        # Fallback to advanced endpoint if simple search returns nothing
+        if not records:
+            adv_url = f"https://api.data.world/v0/search?size={limit}"
+            adv_body = {
+                "query": query,
+                "category": ["dataset"],
+            }
+            adv_req = urllib.request.Request(
+                adv_url,
+                data=json.dumps(adv_body).encode("utf-8"),
+                headers=headers,
+                method="POST",
+            )
+            with urllib.request.urlopen(adv_req) as response:
+                adv_data = json.loads(response.read().decode("utf-8"))
+            records = adv_data.get("records", [])
+
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"data.world discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # dataset_ref is expected to be "dataworld:owner/id"
+        if dataset_ref.startswith("dataworld:"):
+            ref = dataset_ref.split(":", 1)[1]
+        else:
+            ref = dataset_ref
+            
+        parts = ref.split("/")
+        if len(parts) != 2:
+            return {"ok": False, "error": f"Invalid data.world dataset ID format. Expected owner/id, got {ref}"}
+            
+        owner, dataset_id = parts
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_dataworld_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # First, get the dataset metadata to find the files
+        url = f"https://api.data.world/v0/datasets/{owner}/{dataset_id}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json"
+        }
+        
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req) as response:
+            dataset_meta = json.loads(response.read().decode('utf-8'))
+            
+        files = dataset_meta.get("files", [])
+        if not files:
+            return {"ok": False, "error": "No files found in this dataset"}
+            
+        # Find the best file to download (prefer csv, parquet, jsonl)
+        best_file = None
+        for ext in [".parquet", ".csv", ".jsonl", ".json"]:
+            for f in files:
+                if f.get("name", "").lower().endswith(ext):
+                    best_file = f
+                    break
+            if best_file:
+                break
+                
+        if not best_file:
+            best_file = files[0] # Just take the first one if no preferred format
+            
+        filename = best_file.get("name")
+        
+        # Download the file
+        download_url = f"https://api.data.world/v0/file_download/{owner}/{dataset_id}/{urllib.parse.quote(filename)}"
+        
+        file_path = os.path.join(target_dir, filename)
+        
+        download_req = urllib.request.Request(download_url, headers=headers)
+        with urllib.request.urlopen(download_req) as response, open(file_path, 'wb') as out_file:
+            out_file.write(response.read())
+            
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"data.world download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper data.world Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/src/python/kaggle_engine.py

@@ -109,8 +109,28 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
 
     api: KaggleApi = auth["api"]
     try:
-        datasets = api.dataset_list(search=query, page_size=max(1, min(limit, 100)))
-        items = [_dataset_to_dict(ds) for ds in datasets[:limit]]
+        desired = max(1, min(limit, 100))
+
+        try:
+            datasets = api.dataset_list(search=query, page_size=desired)
+            items = [_dataset_to_dict(ds) for ds in datasets[:limit]]
+            return {"ok": True, "results": items, "count": len(items)}
+        except TypeError:
+            pass
+
+        collected = []
+        page = 1
+        while len(collected) < limit:
+            page_items = api.dataset_list(search=query, page=page)
+            if not page_items:
+                break
+
+            collected.extend(page_items)
+            if len(page_items) < 20:
+                break
+            page += 1
+
+        items = [_dataset_to_dict(ds) for ds in collected[:limit]]
         return {"ok": True, "results": items, "count": len(items)}
     except Exception as e:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}

package/src/python/openml_engine.py

@@ -0,0 +1,146 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+from typing import Dict, Any, List
+
+try:
+    import openml
+except ImportError:
+    openml = None
+
+def _ensure_openml() -> Dict[str, Any]:
+    if openml is None:
+        return {"ok": False, "error": "openml package is not installed. Run 'pip install openml'"}
+    return {"ok": True}
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    # OpenML dataset dict from list_datasets
+    did = ds.get("did", "")
+    name = ds.get("name", f"dataset_{did}")
+    version = ds.get("version", "1")
+    status = ds.get("status", "active")
+    format = ds.get("format", "unknown")
+    
+    # Map to Vesper DatasetMetadata format
+    return {
+        "id": f"openml:{did}",
+        "name": name,
+        "source": "openml",
+        "description": f"OpenML Dataset {name} (ID: {did}, Version: {version}, Format: {format}, Status: {status})",
+        "author": "OpenML Community",
+        "license": "Public",
+        "tags": ["openml", format.lower()],
+        "downloads": ds.get("NumberOfDownloads", 0),
+        "likes": ds.get("NumberOfLikes", 0),
+        "created_at": ds.get("upload_date", ""),
+        "updated_at": ds.get("upload_date", ""),
+        "size_bytes": 0, # Not always available in list
+        "quality_score": 0.8, # Default good score for OpenML
+        "domain": "machine_learning",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": 100,
+        "has_readme": False,
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    check = _ensure_openml()
+    if not check.get("ok"):
+        return check
+
+    try:
+        # OpenML list_datasets doesn't have a direct text search in the python API easily exposed without downloading all.
+        # But we can filter by tag or just get a list and filter locally if needed, or use the REST API directly.
+        # Actually, openml.datasets.list_datasets() returns a dict of datasets.
+        # We can fetch a larger batch and filter by name/keyword.
+        
+        # Fetching a batch of datasets
+        datasets = openml.datasets.list_datasets(output_format='dataframe')
+        
+        if query:
+            # Simple case-insensitive search in name
+            mask = datasets['name'].str.contains(query, case=False, na=False)
+            filtered = datasets[mask]
+        else:
+            filtered = datasets
+            
+        # Sort by NumberOfDownloads if available, else just take top
+        if 'NumberOfDownloads' in filtered.columns:
+            filtered = filtered.sort_values('NumberOfDownloads', ascending=False)
+            
+        top_k = filtered.head(limit)
+        
+        # Convert to list of dicts
+        records = top_k.to_dict(orient='records')
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"OpenML discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    check = _ensure_openml()
+    if not check.get("ok"):
+        return check
+
+    try:
+        # dataset_ref is expected to be "openml:ID"
+        if dataset_ref.startswith("openml:"):
+            did_str = dataset_ref.split(":")[1]
+        else:
+            did_str = dataset_ref
+            
+        did = int(did_str)
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_openml_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # Get the dataset
+        dataset = openml.datasets.get_dataset(did, download_data=True, download_qualities=False, download_features_meta_data=False)
+        
+        # Get the pandas dataframe
+        X, y, categorical_indicator, attribute_names = dataset.get_data(
+            dataset_format="dataframe"
+        )
+        
+        # If there's a target column (y), we might want to join it back if it was separated
+        # get_data() can return X and y separately if target is specified, but usually X contains everything if target=None
+        # Let's just get everything
+        df, _, _, _ = dataset.get_data(target=None, dataset_format="dataframe")
+        
+        # Save to parquet in the target directory
+        safe_name = "".join([c if c.isalnum() else "_" for c in dataset.name])
+        file_path = os.path.join(target_dir, f"{safe_name}_{did}.parquet")
+        
+        df.to_parquet(file_path, index=False)
+        
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"OpenML download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper OpenML Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()