@vespermcp/mcp-server 1.2.20 → 1.2.21

package/build/index.js

@@ -760,19 +760,19 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
             },
             {
                 name: "vesper_download_assets",
-                description: "Download real image/media assets using smart source-aware recipes (HuggingFace, Kaggle, direct URL).",
+                description: "Download real image/media assets using smart source-aware recipes (HuggingFace, Kaggle, direct URL). Auto-detects image columns from HF feature types, column names, and value patterns. Supports PIL Images, URL-based images, and binary image data.",
                 inputSchema: {
                     type: "object",
                     properties: {
-                        dataset_id: { type: "string", description: "Unique dataset identifier." },
+                        dataset_id: { type: "string", description: "Unique dataset identifier (e.g. 'user/dataset')." },
                         source: { type: "string", enum: ["huggingface", "kaggle", "url"], description: "Asset source type." },
-                        repo_id: { type: "string", description: "Repo ID for HuggingFace (e.g. cifar100)." },
+                        repo_id: { type: "string", description: "Repo ID for HuggingFace (e.g. 'user/dataset'). Auto-inferred from dataset_id if omitted." },
                         kaggle_ref: { type: "string", description: "Kaggle dataset ref (owner/dataset)." },
                         urls: { type: "array", items: { type: "string" }, description: "Direct asset URLs." },
                         output_format: { type: "string", enum: ["webdataset", "imagefolder", "parquet"], description: "Output asset format." },
                         max_items: { type: "number", description: "Optional cap on number of assets to fetch." },
                         workers: { type: "number", description: "Parallel worker count (default 8)." },
-                        image_column: { type: "string", description: "Optional explicit image column for HuggingFace datasets." },
+                        image_column: { type: "string", description: "Explicit image column name. If omitted, auto-detected from HF features, column names, and sample values." },
                     },
                     required: ["dataset_id", "source"],
                 },
@@ -1270,7 +1270,10 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             hydrateExternalKeys();
             const datasetId = String(request.params.arguments?.dataset_id || "").trim();
             const source = String(request.params.arguments?.source || "").trim().toLowerCase();
-            const repoId = request.params.arguments?.repo_id ? String(request.params.arguments.repo_id) : undefined;
+            // Auto-infer repo_id from dataset_id if not provided (common for HuggingFace)
+            const repoId = request.params.arguments?.repo_id
+                ? String(request.params.arguments.repo_id)
+                : (source === "huggingface" && datasetId.includes("/") ? datasetId : undefined);
             const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
             const urls = Array.isArray(request.params.arguments?.urls)
                 ? (request.params.arguments?.urls).map(v => String(v))
@@ -1296,6 +1299,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             }
             if (source === "huggingface") {
                 requiredModules.push({ module: "datasets", packageName: "datasets" });
+                requiredModules.push({ module: "PIL", packageName: "Pillow" });
             }
             if (source === "kaggle") {
                 requiredModules.push({ module: "kaggle", packageName: "kaggle" });
@@ -1326,8 +1330,17 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             try {
                 const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
                 if (!result?.ok) {
+                    const errMsg = result?.error || "Unknown error";
+                    // Enhance error messages for common failures
+                    let hint = "";
+                    if (errMsg.includes("No image column")) {
+                        hint = "\n\nHint: Specify image_column parameter with the column name containing images/URLs.";
+                    }
+                    else if (errMsg.includes("Authentication") || errMsg.includes("401") || errMsg.includes("403")) {
+                        hint = "\n\nHint: Use configure_keys tool to set HF_TOKEN for gated/private datasets.";
+                    }
                     return {
-                        content: [{ type: "text", text: `ERROR: asset download failed: ${result?.error || "Unknown error"}` }],
+                        content: [{ type: "text", text: `ERROR: asset download failed: ${errMsg}${hint}` }],
                         isError: true,
                     };
                 }

package/build/ingestion/hf-downloader.js

@@ -94,8 +94,18 @@ export class HFDownloader {
         }
         catch (error) {
             const msg = String(error?.message || error);
-            if (msg.includes("401") || msg.includes("403") || msg.toLowerCase().includes("unauthorized")) {
-                throw new Error("Hugging Face gated/private dataset requires token. Run 'vespermcp config keys' to set HF token.");
+            if (msg.includes("401") || msg.toLowerCase().includes("unauthorized")) {
+                throw new Error(`Authentication required for dataset '${repoId}'. ` +
+                    `This dataset may be gated or private. ` +
+                    `Use the configure_keys tool to set your HF_TOKEN, then retry.`);
+            }
+            if (msg.includes("403") || msg.toLowerCase().includes("forbidden")) {
+                throw new Error(`Access denied for dataset '${repoId}'. ` +
+                    `You may need to accept the dataset's usage agreement on huggingface.co, ` +
+                    `then set HF_TOKEN via configure_keys tool.`);
+            }
+            if (msg.includes("404") || msg.toLowerCase().includes("not found")) {
+                throw new Error(`Dataset '${repoId}' not found on HuggingFace. Check the dataset ID.`);
             }
             console.error(`[HF] Failed to list files for ${repoId}:`, msg);
             return null;

package/build/ingestion/ingestor.js

@@ -81,13 +81,20 @@ export class DataIngestor {
                     return resolvedPath;
                 }
                 catch (e) {
-                    this.failDownload(datasetId, e.message);
-                    throw e;
+                    const msg = String(e?.message || e);
+                    // If auth error, propagate immediately with helpful message
+                    if (msg.includes("401") || msg.includes("403") || msg.includes("Authentication") || msg.includes("Access denied")) {
+                        this.failDownload(datasetId, msg);
+                        throw e;
+                    }
+                    // For other download errors, try the fallback
+                    onProgress?.(`Direct download failed (${msg}), trying datasets library fallback...`);
                 }
             }
-            else {
-                // Fallback: Use Python datasets library to download and convert
-                onProgress?.("No raw files found. Using HuggingFace datasets library to download...");
+            // Fallback: Use Python datasets library to download and convert
+            // This runs when findBestFile returns null OR when direct download fails (non-auth)
+            if (!fs.existsSync(this.getTargetPath(datasetId, "parquet")) || !this.store.getDownloadStatus(datasetId)?.status?.includes("completed")) {
+                onProgress?.("Using HuggingFace datasets library to download...");
                 const targetPath = this.getTargetPath(datasetId, "parquet");
                 this.store.registerDownload(datasetId, targetPath, "downloading");
                 try {

package/build/python/asset_downloader_engine.py

@@ -3,9 +3,14 @@ import asyncio
 import json
 import os
 import sys
+import warnings
 from pathlib import Path
 from typing import Any, Dict
 
+# Suppress noisy HF warnings
+warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
+warnings.filterwarnings("ignore", message=".*legacy.*")
+
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
     sys.path.insert(0, str(CURRENT_DIR))
@@ -24,6 +29,11 @@ async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
     workers = int(payload.get("workers") or 8)
     recipes_dir = payload.get("recipes_dir")
 
+    # Auto-set HF token from payload if provided
+    token = payload.get("token") or payload.get("hf_token")
+    if token:
+        os.environ["HF_TOKEN"] = str(token)
+
     downloader = AssetDownloader(output_root=output_root, workers=workers, recipes_dir=recipes_dir)
 
     result = await downloader.download_assets(
@@ -66,7 +76,16 @@ def main() -> None:
 
         _print({"ok": False, "error": f"Unknown action: {args.action}"})
     except Exception as e:
-        _print({"ok": False, "error": str(e)})
+        error_msg = str(e)
+        # Provide actionable error messages
+        if "401" in error_msg or "403" in error_msg or "Unauthorized" in error_msg:
+            error_msg = (
+                "Authentication required. This dataset may be gated/private. "
+                "Use configure_keys tool to set HF_TOKEN, then retry."
+            )
+        elif "No image column" in error_msg:
+            error_msg += " Hint: specify image_column parameter with the name of the column containing images."
+        _print({"ok": False, "error": error_msg})
 
 
 if __name__ == "__main__":

package/build/python/hf_fallback.py

@@ -5,6 +5,12 @@ Used when the HF Hub file listing finds no suitable data files
 (e.g. script-based datasets, gated datasets, datasets that use
 the `datasets` library format).
 
+Handles:
+- Legacy script-based datasets (trust_remote_code)
+- Gated/private datasets (token auth)
+- Image datasets (PIL Image columns → stripped for tabular export)
+- Various split formats (DatasetDict, single split)
+
 Usage:
     python hf_fallback.py '{"repo_id": "user/dataset", "output_path": "/path/to/output.parquet", "token": "optional", "max_rows": 100000, "split": "train"}'
 
@@ -15,6 +21,127 @@ Output: JSON to stdout
 import sys
 import json
 import os
+import warnings
+
+# Suppress noisy HF warnings about trust_remote_code etc.
+warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
+warnings.filterwarnings("ignore", message=".*legacy.*")
+
+
+def _detect_image_columns(ds):
+    """Detect columns that contain HF Image features or PIL Image objects."""
+    image_cols = []
+    features = getattr(ds, "features", None)
+    if features:
+        for name, feat in features.items():
+            feat_cls = feat.__class__.__name__.lower()
+            feat_str = str(feat).lower()
+            if feat_cls == "image" or "image(" in feat_str:
+                image_cols.append(name)
+    return image_cols
+
+
+def _strip_image_columns(ds, image_cols):
+    """Remove image columns from dataset so it can be exported to Parquet/CSV.
+    
+    Image columns contain PIL Image objects that can't be serialized to tabular
+    formats. We replace them with a placeholder string indicating the column
+    was an image column.
+    """
+    if not image_cols:
+        return ds
+    
+    # Remove the image columns entirely for tabular export
+    cols_to_keep = [c for c in ds.column_names if c not in image_cols]
+    if not cols_to_keep:
+        # Dataset is ALL image columns — keep them but cast to path strings if possible
+        return ds
+    
+    return ds.select_columns(cols_to_keep)
+
+
+def _load_dataset_robust(repo_id, token, split):
+    """Load a HuggingFace dataset with multiple fallback strategies.
+    
+    Strategy order:
+    1. Normal load with trust_remote_code=True (handles legacy script datasets)
+    2. Load without trust_remote_code (newer datasets that reject it)
+    3. Load with streaming=True then materialize (handles very large datasets)
+    """
+    from datasets import load_dataset, DatasetDict
+
+    errors = []
+    splits_to_try = [split] if split else ["train", "test", "validation", None]
+    
+    # Strategy 1: Normal load with trust_remote_code
+    for s in splits_to_try:
+        try:
+            kwargs = {"path": repo_id, "trust_remote_code": True}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds = load_dataset(**kwargs)
+            return ds, s
+        except (ValueError, KeyError):
+            continue
+        except Exception as e:
+            msg = str(e)
+            # Auth errors should be raised immediately, not retried
+            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
+                raise
+            if "split" in msg.lower() or "key" in msg.lower():
+                continue
+            errors.append(f"trust_remote_code=True, split={s}: {msg}")
+    
+    # Strategy 2: Load WITHOUT trust_remote_code (some repos reject it)
+    for s in splits_to_try:
+        try:
+            kwargs = {"path": repo_id}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds = load_dataset(**kwargs)
+            return ds, s
+        except (ValueError, KeyError):
+            continue
+        except Exception as e:
+            msg = str(e)
+            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
+                raise
+            if "split" in msg.lower() or "key" in msg.lower():
+                continue
+            errors.append(f"trust_remote_code=False, split={s}: {msg}")
+    
+    # Strategy 3: Streaming fallback (for very large / oddly structured datasets)
+    for s in splits_to_try:
+        if s is None:
+            continue  # streaming requires a split
+        try:
+            kwargs = {"path": repo_id, "streaming": True, "trust_remote_code": True}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds_stream = load_dataset(**kwargs)
+            # Materialize from streaming iterator
+            from datasets import Dataset as HFDataset
+            rows = []
+            for i, row in enumerate(ds_stream):
+                if i >= 500000:
+                    break
+                rows.append(row)
+            if rows:
+                ds = HFDataset.from_list(rows)
+                return ds, s
+        except Exception:
+            continue
+    
+    # All strategies failed
+    error_summary = "; ".join(errors[:3]) if errors else "No valid configuration found"
+    return None, error_summary
+
 
 def main():
     if len(sys.argv) < 2:
@@ -53,37 +180,10 @@ def main():
         pl = None
 
     try:
-        # Try loading with streaming first (memory-efficient)
-        # If split is not specified, try common ones
-        splits_to_try = [split] if split else ["train", "test", "validation", None]
-        
-        ds = None
-        used_split = None
-        
-        for s in splits_to_try:
-            try:
-                kwargs = {
-                    "path": repo_id,
-                    "trust_remote_code": True,
-                }
-                if token:
-                    kwargs["token"] = token
-                if s:
-                    kwargs["split"] = s
-
-                ds = load_dataset(**kwargs)
-                used_split = s
-                break
-            except (ValueError, KeyError):
-                # Split doesn't exist, try next
-                continue
-            except Exception as e:
-                if "split" in str(e).lower() or "key" in str(e).lower():
-                    continue
-                raise
+        ds, used_split = _load_dataset_robust(repo_id, token, split)
 
         if ds is None:
-            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}'. No valid splits found."}))
+            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}': {used_split}"}))
             sys.exit(1)
 
         # Handle DatasetDict (when no split specified)
@@ -107,37 +207,88 @@ def main():
             ds = ds.select(range(max_rows))
             total_rows = max_rows
 
+        # Detect and handle image columns (PIL Image objects can't be exported to Parquet)
+        image_cols = _detect_image_columns(ds)
+        has_images = len(image_cols) > 0
+        
+        if has_images:
+            # Strip image columns for tabular export, note them in output
+            export_ds = _strip_image_columns(ds, image_cols)
+        else:
+            export_ds = ds
+
         # Ensure output directory exists
         os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
         # Export to parquet
-        columns = ds.column_names
+        columns = export_ds.column_names
 
-        if output_path.endswith(".parquet"):
-            ds.to_parquet(output_path)
-        elif output_path.endswith(".csv"):
-            ds.to_csv(output_path)
-        else:
-            # Default to parquet
-            if not output_path.endswith(".parquet"):
-                output_path = output_path + ".parquet"
-            ds.to_parquet(output_path)
+        try:
+            if output_path.endswith(".parquet"):
+                export_ds.to_parquet(output_path)
+            elif output_path.endswith(".csv"):
+                export_ds.to_csv(output_path)
+            else:
+                # Default to parquet
+                if not output_path.endswith(".parquet"):
+                    output_path = output_path + ".parquet"
+                export_ds.to_parquet(output_path)
+        except Exception as export_err:
+            # If parquet export fails (e.g. complex nested types), try CSV
+            csv_path = output_path.replace(".parquet", ".csv")
+            try:
+                export_ds.to_csv(csv_path)
+                output_path = csv_path
+            except Exception:
+                raise export_err  # Re-raise original error
 
-        print(json.dumps({
+        result = {
             "ok": True,
             "path": output_path,
             "rows": total_rows,
             "columns": columns,
             "split": used_split
-        }))
+        }
+        
+        if has_images:
+            result["image_columns"] = image_cols
+            result["note"] = (
+                f"This dataset contains image columns ({', '.join(image_cols)}). "
+                "Image data was stripped for tabular export. "
+                "Use vesper_download_assets with source='huggingface' to download the actual images."
+            )
+
+        print(json.dumps(result))
 
     except Exception as e:
         error_msg = str(e)
-        # Provide helpful hints
-        if "401" in error_msg or "403" in error_msg or "gated" in error_msg.lower():
-            error_msg += " (This dataset may be gated/private. Set HF_TOKEN via configure_keys tool.)"
-        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower():
+        # Provide helpful, actionable hints
+        if "401" in error_msg or "Unauthorized" in error_msg:
+            error_msg = (
+                f"Authentication required for dataset '{repo_id}'. "
+                "This dataset may be gated or private. "
+                "Use the configure_keys tool to set your HF_TOKEN, then retry."
+            )
+        elif "403" in error_msg or "Forbidden" in error_msg:
+            error_msg = (
+                f"Access denied for dataset '{repo_id}'. "
+                "You may need to accept the dataset's usage agreement on huggingface.co, "
+                "then set HF_TOKEN via configure_keys tool."
+            )
+        elif "gated" in error_msg.lower():
+            error_msg = (
+                f"Dataset '{repo_id}' is gated. "
+                "Visit https://huggingface.co/datasets/{repo_id} to request access, "
+                "then set HF_TOKEN via configure_keys tool."
+            ).format(repo_id=repo_id)
+        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower() or "doesn't exist" in error_msg.lower():
             error_msg = f"Dataset '{repo_id}' not found on HuggingFace. Check the dataset ID."
+        elif "script" in error_msg.lower() and "no longer supported" in error_msg.lower():
+            error_msg = (
+                f"Dataset '{repo_id}' uses a legacy loading script that is no longer supported "
+                "by the current version of the datasets library. "
+                "Try: pip install datasets --upgrade, or use an older datasets version."
+            )
 
         print(json.dumps({"ok": False, "error": error_msg}))
         sys.exit(1)