@vespermcp/mcp-server 1.2.4 → 1.2.6

package/build/index.js

@@ -131,8 +131,83 @@ function extractRequestedRows(query, requirements) {
         return Math.max(...allNums);
     return undefined;
 }
+const verifiedPythonModules = new Set();
+function getPythonCommand() {
+    return process.platform === "win32" ? "py" : "python";
+}
+function runPythonProcess(args, timeoutMs = 300000) {
+    const pyCmd = getPythonCommand();
+    return new Promise((resolve, reject) => {
+        const proc = spawn(pyCmd, args, {
+            env: {
+                ...process.env,
+                PIP_DISABLE_PIP_VERSION_CHECK: "1",
+                PYTHONUTF8: "1",
+            },
+        });
+        let stdout = "";
+        let stderr = "";
+        proc.stdout.on("data", (d) => (stdout += d.toString()));
+        proc.stderr.on("data", (d) => (stderr += d.toString()));
+        const timer = setTimeout(() => {
+            try {
+                proc.kill();
+            }
+            catch {
+                // no-op
+            }
+            reject(new Error(`Python command timed out after ${timeoutMs}ms: ${args.join(" ")}`));
+        }, timeoutMs);
+        proc.on("close", (code) => {
+            clearTimeout(timer);
+            resolve({ code: code ?? 1, stdout, stderr });
+        });
+        proc.on("error", (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+async function ensurePythonModules(modulePackagePairs) {
+    const missing = [];
+    for (const pair of modulePackagePairs) {
+        if (verifiedPythonModules.has(pair.module)) {
+            continue;
+        }
+        const check = await runPythonProcess([
+            "-c",
+            `import importlib.util,sys; sys.exit(0 if importlib.util.find_spec(${JSON.stringify(pair.module)}) else 1)`
+        ], 20000);
+        if (check.code === 0) {
+            verifiedPythonModules.add(pair.module);
+        }
+        else {
+            missing.push(pair);
+        }
+    }
+    if (missing.length === 0) {
+        return;
+    }
+    const packages = [...new Set(missing.map(m => m.packageName))];
+    console.error(`[Vesper] Installing missing Python packages: ${packages.join(", ")}`);
+    const installArgs = ["-m", "pip", "install", "--disable-pip-version-check", ...packages];
+    let install = await runPythonProcess(installArgs, 600000);
+    if (install.code !== 0) {
+        console.error(`[Vesper] pip install failed (trying --user fallback): ${(install.stderr || "").slice(0, 300)}`);
+        const userInstallArgs = ["-m", "pip", "install", "--disable-pip-version-check", "--user", ...packages];
+        install = await runPythonProcess(userInstallArgs, 600000);
+    }
+    if (install.code !== 0) {
+        const details = (install.stderr || install.stdout || "Unknown pip install error").trim();
+        throw new Error(`Failed to install required Python packages (${packages.join(", ")}). ${details}`);
+    }
+    console.error(`[Vesper] Successfully installed: ${packages.join(", ")}`);
+    for (const pair of missing) {
+        verifiedPythonModules.add(pair.module);
+    }
+}
 function runPythonJson(scriptPath, args) {
-    const pyCmd = process.platform === "win32" ? "py" : "python";
+    const pyCmd = getPythonCommand();
     return new Promise((resolve, reject) => {
         const proc = spawn(pyCmd, [scriptPath, ...args]);
         let stdout = "";
@@ -167,6 +242,25 @@ function syncPythonScripts(appRoot, dataRoot) {
     const pythonDest = path.join(dataRoot, "python");
     if (!fs.existsSync(pythonDest))
         fs.mkdirSync(pythonDest, { recursive: true });
+    const collectPyFiles = (dir) => {
+        if (!fs.existsSync(dir))
+            return [];
+        const out = [];
+        const stack = [dir];
+        while (stack.length > 0) {
+            const cur = stack.pop();
+            for (const entry of fs.readdirSync(cur, { withFileTypes: true })) {
+                const full = path.join(cur, entry.name);
+                if (entry.isDirectory()) {
+                    stack.push(full);
+                }
+                else if (entry.isFile() && full.endsWith(".py")) {
+                    out.push(full);
+                }
+            }
+        }
+        return out;
+    };
     // Sources to check for Python scripts
     const sources = [
         path.join(appRoot, "src", "python"),
@@ -175,25 +269,21 @@ function syncPythonScripts(appRoot, dataRoot) {
     ];
     let syncedCount = 0;
     for (const src of sources) {
-        if (fs.existsSync(src)) {
-            const files = fs.readdirSync(src);
-            for (const file of files) {
-                if (file.endsWith(".py")) {
-                    const srcPath = path.join(src, file);
-                    const destPath = path.join(pythonDest, file);
-                    // Only copy if file doesn't exist or is different size (basic sync)
-                    const srcStat = fs.statSync(srcPath);
-                    let shouldCopy = true;
-                    if (fs.existsSync(destPath)) {
-                        const destStat = fs.statSync(destPath);
-                        if (srcStat.size === destStat.size)
-                            shouldCopy = false;
-                    }
-                    if (shouldCopy) {
-                        fs.copyFileSync(srcPath, destPath);
-                        syncedCount++;
-                    }
-                }
+        const files = collectPyFiles(src);
+        for (const srcPath of files) {
+            const rel = path.relative(src, srcPath);
+            const destPath = path.join(pythonDest, rel);
+            const srcStat = fs.statSync(srcPath);
+            let shouldCopy = true;
+            if (fs.existsSync(destPath)) {
+                const destStat = fs.statSync(destPath);
+                if (srcStat.size === destStat.size)
+                    shouldCopy = false;
+            }
+            if (shouldCopy) {
+                fs.mkdirSync(path.dirname(destPath), { recursive: true });
+                fs.copyFileSync(srcPath, destPath);
+                syncedCount++;
             }
         }
     }
@@ -471,6 +561,25 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                     required: ["source", "dataset_id"],
                 },
             },
+            {
+                name: "vesper_download_assets",
+                description: "Download real image/media assets using smart source-aware recipes (HuggingFace, Kaggle, direct URL).",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        dataset_id: { type: "string", description: "Unique dataset identifier." },
+                        source: { type: "string", enum: ["huggingface", "kaggle", "url"], description: "Asset source type." },
+                        repo_id: { type: "string", description: "Repo ID for HuggingFace (e.g. cifar100)." },
+                        kaggle_ref: { type: "string", description: "Kaggle dataset ref (owner/dataset)." },
+                        urls: { type: "array", items: { type: "string" }, description: "Direct asset URLs." },
+                        output_format: { type: "string", enum: ["webdataset", "imagefolder", "parquet"], description: "Output asset format." },
+                        max_items: { type: "number", description: "Optional cap on number of assets to fetch." },
+                        workers: { type: "number", description: "Parallel worker count (default 8)." },
+                        image_column: { type: "string", description: "Optional explicit image column for HuggingFace datasets." },
+                    },
+                    required: ["dataset_id", "source"],
+                },
+            },
             {
                 name: "configure_kaggle",
                 description: "Optionally store Kaggle API credentials for Kaggle discover/download. Core Vesper works without this.",
@@ -571,6 +680,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                     properties: {
                         query: { type: "string" },
                         requirements: { type: "string" },
+                        download_images: { type: "boolean", description: "When true, enables post-prepare smart asset downloading for image/media datasets." },
                         cleaning_options: { type: "object" },
                         split_config: { type: "object" },
                     },
@@ -813,6 +923,24 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     const hf = new HuggingFaceScraper();
                     results = await hf.scrape(Math.max(1, limit), true, query);
                 }
+                const recipeScript = path.join(dataRoot, "python", "asset_downloader_engine.py");
+                for (const ds of results.slice(0, limit)) {
+                    const info = {
+                        dataset_id: ds.id,
+                        id: ds.id,
+                        source: ds.source,
+                        repo_id: ds.id,
+                        total_images: ds.total_examples || 0,
+                        image_column: undefined,
+                        recipes_dir: path.join(dataRoot, "recipes"),
+                    };
+                    try {
+                        await runPythonJson(recipeScript, ["build_recipe", JSON.stringify(info)]);
+                    }
+                    catch {
+                        // best-effort recipe generation; ignore discovery-time recipe failures
+                    }
+                }
                 const formattedOutput = formatSearchResults(results.slice(0, limit));
                 return {
                     content: [{ type: "text", text: formattedOutput }]
@@ -857,6 +985,85 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 };
             }
         }
+        case "vesper_download_assets": {
+            hydrateExternalKeys();
+            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+            const source = String(request.params.arguments?.source || "").trim().toLowerCase();
+            const repoId = request.params.arguments?.repo_id ? String(request.params.arguments.repo_id) : undefined;
+            const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
+            const urls = Array.isArray(request.params.arguments?.urls)
+                ? (request.params.arguments?.urls).map(v => String(v))
+                : undefined;
+            const outputFormat = String(request.params.arguments?.output_format || "webdataset");
+            const maxItems = request.params.arguments?.max_items ? Number(request.params.arguments.max_items) : undefined;
+            const workers = request.params.arguments?.workers ? Number(request.params.arguments.workers) : 8;
+            const imageColumn = request.params.arguments?.image_column ? String(request.params.arguments.image_column) : undefined;
+            if (!datasetId || !source) {
+                throw new McpError(ErrorCode.InvalidParams, "dataset_id and source are required");
+            }
+            if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
+                return {
+                    content: [{ type: "text", text: "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds)." }],
+                    isError: true,
+                };
+            }
+            const requiredModules = [
+                { module: "aiohttp", packageName: "aiohttp" },
+            ];
+            if (source === "url") {
+                requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
+            }
+            if (source === "huggingface") {
+                requiredModules.push({ module: "datasets", packageName: "datasets" });
+            }
+            if (source === "kaggle") {
+                requiredModules.push({ module: "kaggle", packageName: "kaggle" });
+            }
+            if (outputFormat === "webdataset") {
+                requiredModules.push({ module: "webdataset", packageName: "webdataset" });
+            }
+            try {
+                await ensurePythonModules(requiredModules);
+            }
+            catch (error) {
+                return {
+                    content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
+                    isError: true,
+                };
+            }
+            const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
+            const payload = {
+                dataset_id: datasetId,
+                source,
+                repo_id: repoId,
+                kaggle_ref: kaggleRef,
+                urls,
+                output_format: outputFormat,
+                max_items: maxItems,
+                workers,
+                image_column: imageColumn,
+                output_root: path.join(dataRoot, "data", "assets"),
+                recipes_dir: path.join(dataRoot, "recipes"),
+            };
+            try {
+                const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
+                if (!result?.ok) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: asset download failed: ${result?.error || "Unknown error"}` }],
+                        isError: true,
+                    };
+                }
+                return {
+                    content: [{ type: "text", text: JSON.stringify(result.result, null, 2) }],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [{ type: "text", text: `ERROR: asset downloader execution failed: ${error.message}` }],
+                    isError: true,
+                };
+            }
+        }
         case "configure_kaggle": {
             const username = String(request.params.arguments?.username || "").trim();
             const key = String(request.params.arguments?.key || "").trim();
@@ -1033,7 +1240,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         case "prepare_dataset": {
             const query = String(request.params.arguments?.query);
             const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
-            const job = jobManager.createJob("prepare", 0, { query, requirements });
+            const downloadImages = request.params.arguments?.download_images === true;
+            const job = jobManager.createJob("prepare", 0, { query, requirements, downloadImages });
             return {
                 content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you.` }]
             };

package/build/python/asset_downloader_engine.py

@@ -0,0 +1,73 @@
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+CURRENT_DIR = Path(__file__).resolve().parent
+if str(CURRENT_DIR) not in sys.path:
+    sys.path.insert(0, str(CURRENT_DIR))
+
+from vesper.core.asset_downloader import AssetDownloader
+from vesper.core.download_recipe import build_download_recipe, save_recipe, get_download_recipe
+
+
+def _print(payload: Dict[str, Any]) -> None:
+    print(json.dumps(payload, ensure_ascii=False))
+
+
+async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
+    payload = json.loads(args.payload)
+    output_root = payload.get("output_root") or str(Path.home() / ".vesper" / "data" / "assets")
+    workers = int(payload.get("workers") or 8)
+    recipes_dir = payload.get("recipes_dir")
+
+    downloader = AssetDownloader(output_root=output_root, workers=workers, recipes_dir=recipes_dir)
+
+    result = await downloader.download_assets(
+        dataset_id=str(payload.get("dataset_id")),
+        source=payload.get("source"),
+        repo_id=payload.get("repo_id"),
+        kaggle_ref=payload.get("kaggle_ref"),
+        urls=payload.get("urls"),
+        output_format=payload.get("output_format", "webdataset"),
+        max_items=payload.get("max_items"),
+        image_column=payload.get("image_column"),
+    )
+    return {"ok": True, "result": result}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Vesper Smart Asset Downloader Engine")
+    parser.add_argument("action", choices=["download", "build_recipe", "get_recipe"])
+    parser.add_argument("payload", help="JSON payload")
+    args = parser.parse_args()
+
+    try:
+        if args.action == "download":
+            response = asyncio.run(_run_download(args))
+            _print(response)
+            return
+
+        payload = json.loads(args.payload)
+        if args.action == "build_recipe":
+            recipe = build_download_recipe(payload)
+            saved = save_recipe(recipe, payload.get("recipes_dir"))
+            _print({"ok": True, "recipe": recipe, "saved_to": saved})
+            return
+
+        if args.action == "get_recipe":
+            dataset_id = str(payload.get("dataset_id"))
+            recipe = get_download_recipe(dataset_id, payload.get("recipes_dir"))
+            _print({"ok": True, "recipe": recipe})
+            return
+
+        _print({"ok": False, "error": f"Unknown action: {args.action}"})
+    except Exception as e:
+        _print({"ok": False, "error": str(e)})
+
+
+if __name__ == "__main__":
+    main()

package/build/python/vesper/__init__.py

@@ -0,0 +1 @@
+"""Vesper Python runtime package."""

package/build/python/vesper/core/__init__.py

@@ -0,0 +1 @@
+"""Core data engines for Vesper."""