@vespermcp/mcp-server 1.2.20 → 1.2.22

package/src/python/hf_fallback.py

@@ -5,6 +5,12 @@ Used when the HF Hub file listing finds no suitable data files
 (e.g. script-based datasets, gated datasets, datasets that use
 the `datasets` library format).
 
+Handles:
+- Legacy script-based datasets (trust_remote_code)
+- Gated/private datasets (token auth)
+- Image datasets (PIL Image columns → stripped for tabular export)
+- Various split formats (DatasetDict, single split)
+
 Usage:
     python hf_fallback.py '{"repo_id": "user/dataset", "output_path": "/path/to/output.parquet", "token": "optional", "max_rows": 100000, "split": "train"}'
 
@@ -15,6 +21,127 @@ Output: JSON to stdout
 import sys
 import json
 import os
+import warnings
+
+# Suppress noisy HF warnings about trust_remote_code etc.
+warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
+warnings.filterwarnings("ignore", message=".*legacy.*")
+
+
+def _detect_image_columns(ds):
+    """Detect columns that contain HF Image features or PIL Image objects."""
+    image_cols = []
+    features = getattr(ds, "features", None)
+    if features:
+        for name, feat in features.items():
+            feat_cls = feat.__class__.__name__.lower()
+            feat_str = str(feat).lower()
+            if feat_cls == "image" or "image(" in feat_str:
+                image_cols.append(name)
+    return image_cols
+
+
+def _strip_image_columns(ds, image_cols):
+    """Remove image columns from dataset so it can be exported to Parquet/CSV.
+    
+    Image columns contain PIL Image objects that can't be serialized to tabular
+    formats. We replace them with a placeholder string indicating the column
+    was an image column.
+    """
+    if not image_cols:
+        return ds
+    
+    # Remove the image columns entirely for tabular export
+    cols_to_keep = [c for c in ds.column_names if c not in image_cols]
+    if not cols_to_keep:
+        # Dataset is ALL image columns — keep them but cast to path strings if possible
+        return ds
+    
+    return ds.select_columns(cols_to_keep)
+
+
+def _load_dataset_robust(repo_id, token, split):
+    """Load a HuggingFace dataset with multiple fallback strategies.
+    
+    Strategy order:
+    1. Normal load with trust_remote_code=True (handles legacy script datasets)
+    2. Load without trust_remote_code (newer datasets that reject it)
+    3. Load with streaming=True then materialize (handles very large datasets)
+    """
+    from datasets import load_dataset, DatasetDict
+
+    errors = []
+    splits_to_try = [split] if split else ["train", "test", "validation", None]
+    
+    # Strategy 1: Normal load with trust_remote_code
+    for s in splits_to_try:
+        try:
+            kwargs = {"path": repo_id, "trust_remote_code": True}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds = load_dataset(**kwargs)
+            return ds, s
+        except (ValueError, KeyError):
+            continue
+        except Exception as e:
+            msg = str(e)
+            # Auth errors should be raised immediately, not retried
+            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
+                raise
+            if "split" in msg.lower() or "key" in msg.lower():
+                continue
+            errors.append(f"trust_remote_code=True, split={s}: {msg}")
+    
+    # Strategy 2: Load WITHOUT trust_remote_code (some repos reject it)
+    for s in splits_to_try:
+        try:
+            kwargs = {"path": repo_id}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds = load_dataset(**kwargs)
+            return ds, s
+        except (ValueError, KeyError):
+            continue
+        except Exception as e:
+            msg = str(e)
+            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
+                raise
+            if "split" in msg.lower() or "key" in msg.lower():
+                continue
+            errors.append(f"trust_remote_code=False, split={s}: {msg}")
+    
+    # Strategy 3: Streaming fallback (for very large / oddly structured datasets)
+    for s in splits_to_try:
+        if s is None:
+            continue  # streaming requires a split
+        try:
+            kwargs = {"path": repo_id, "streaming": True, "trust_remote_code": True}
+            if token:
+                kwargs["token"] = token
+            if s:
+                kwargs["split"] = s
+            ds_stream = load_dataset(**kwargs)
+            # Materialize from streaming iterator
+            from datasets import Dataset as HFDataset
+            rows = []
+            for i, row in enumerate(ds_stream):
+                if i >= 500000:
+                    break
+                rows.append(row)
+            if rows:
+                ds = HFDataset.from_list(rows)
+                return ds, s
+        except Exception:
+            continue
+    
+    # All strategies failed
+    error_summary = "; ".join(errors[:3]) if errors else "No valid configuration found"
+    return None, error_summary
+
 
 def main():
     if len(sys.argv) < 2:
@@ -53,37 +180,10 @@ def main():
         pl = None
 
     try:
-        # Try loading with streaming first (memory-efficient)
-        # If split is not specified, try common ones
-        splits_to_try = [split] if split else ["train", "test", "validation", None]
-        
-        ds = None
-        used_split = None
-        
-        for s in splits_to_try:
-            try:
-                kwargs = {
-                    "path": repo_id,
-                    "trust_remote_code": True,
-                }
-                if token:
-                    kwargs["token"] = token
-                if s:
-                    kwargs["split"] = s
-
-                ds = load_dataset(**kwargs)
-                used_split = s
-                break
-            except (ValueError, KeyError):
-                # Split doesn't exist, try next
-                continue
-            except Exception as e:
-                if "split" in str(e).lower() or "key" in str(e).lower():
-                    continue
-                raise
+        ds, used_split = _load_dataset_robust(repo_id, token, split)
 
         if ds is None:
-            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}'. No valid splits found."}))
+            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}': {used_split}"}))
             sys.exit(1)
 
         # Handle DatasetDict (when no split specified)
@@ -107,37 +207,88 @@ def main():
             ds = ds.select(range(max_rows))
             total_rows = max_rows
 
+        # Detect and handle image columns (PIL Image objects can't be exported to Parquet)
+        image_cols = _detect_image_columns(ds)
+        has_images = len(image_cols) > 0
+        
+        if has_images:
+            # Strip image columns for tabular export, note them in output
+            export_ds = _strip_image_columns(ds, image_cols)
+        else:
+            export_ds = ds
+
         # Ensure output directory exists
         os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
         # Export to parquet
-        columns = ds.column_names
+        columns = export_ds.column_names
 
-        if output_path.endswith(".parquet"):
-            ds.to_parquet(output_path)
-        elif output_path.endswith(".csv"):
-            ds.to_csv(output_path)
-        else:
-            # Default to parquet
-            if not output_path.endswith(".parquet"):
-                output_path = output_path + ".parquet"
-            ds.to_parquet(output_path)
+        try:
+            if output_path.endswith(".parquet"):
+                export_ds.to_parquet(output_path)
+            elif output_path.endswith(".csv"):
+                export_ds.to_csv(output_path)
+            else:
+                # Default to parquet
+                if not output_path.endswith(".parquet"):
+                    output_path = output_path + ".parquet"
+                export_ds.to_parquet(output_path)
+        except Exception as export_err:
+            # If parquet export fails (e.g. complex nested types), try CSV
+            csv_path = output_path.replace(".parquet", ".csv")
+            try:
+                export_ds.to_csv(csv_path)
+                output_path = csv_path
+            except Exception:
+                raise export_err  # Re-raise original error
 
-        print(json.dumps({
+        result = {
             "ok": True,
             "path": output_path,
             "rows": total_rows,
             "columns": columns,
             "split": used_split
-        }))
+        }
+        
+        if has_images:
+            result["image_columns"] = image_cols
+            result["note"] = (
+                f"This dataset contains image columns ({', '.join(image_cols)}). "
+                "Image data was stripped for tabular export. "
+                "Use vesper_download_assets with source='huggingface' to download the actual images."
+            )
+
+        print(json.dumps(result))
 
     except Exception as e:
         error_msg = str(e)
-        # Provide helpful hints
-        if "401" in error_msg or "403" in error_msg or "gated" in error_msg.lower():
-            error_msg += " (This dataset may be gated/private. Set HF_TOKEN via configure_keys tool.)"
-        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower():
+        # Provide helpful, actionable hints
+        if "401" in error_msg or "Unauthorized" in error_msg:
+            error_msg = (
+                f"Authentication required for dataset '{repo_id}'. "
+                "This dataset may be gated or private. "
+                "Use the configure_keys tool to set your HF_TOKEN, then retry."
+            )
+        elif "403" in error_msg or "Forbidden" in error_msg:
+            error_msg = (
+                f"Access denied for dataset '{repo_id}'. "
+                "You may need to accept the dataset's usage agreement on huggingface.co, "
+                "then set HF_TOKEN via configure_keys tool."
+            )
+        elif "gated" in error_msg.lower():
+            error_msg = (
+                f"Dataset '{repo_id}' is gated. "
+                "Visit https://huggingface.co/datasets/{repo_id} to request access, "
+                "then set HF_TOKEN via configure_keys tool."
+            ).format(repo_id=repo_id)
+        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower() or "doesn't exist" in error_msg.lower():
             error_msg = f"Dataset '{repo_id}' not found on HuggingFace. Check the dataset ID."
+        elif "script" in error_msg.lower() and "no longer supported" in error_msg.lower():
+            error_msg = (
+                f"Dataset '{repo_id}' uses a legacy loading script that is no longer supported "
+                "by the current version of the datasets library. "
+                "Try: pip install datasets --upgrade, or use an older datasets version."
+            )
 
         print(json.dumps({"ok": False, "error": error_msg}))
         sys.exit(1)

package/src/python/kaggle_engine.py

@@ -12,6 +12,19 @@ except Exception:
     HAS_KAGGLE = False
 
 
+IMAGE_EXTENSIONS = {
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".webp",
+    ".bmp",
+    ".gif",
+    ".tiff",
+    ".tif",
+    ".svg",
+}
+
+
 def _ensure_auth() -> Dict[str, Any]:
     if not HAS_KAGGLE:
         return {
@@ -135,7 +148,56 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}
 
 
-def _pick_best_file(root: str) -> str:
+def _find_image_files(root: str) -> List[str]:
+    image_files: List[str] = []
+    for base, _, files in os.walk(root):
+        for name in files:
+            full = os.path.join(base, name)
+            if os.path.splitext(name)[1].lower() in IMAGE_EXTENSIONS:
+                image_files.append(full)
+    image_files.sort()
+    return image_files
+
+
+def _infer_image_record(root: str, full_path: str, index: int) -> Dict[str, Any]:
+    relative_path = os.path.relpath(full_path, root).replace("\\", "/")
+    parent_dir = os.path.dirname(relative_path)
+    parts = [part for part in parent_dir.split("/") if part and part != "."]
+
+    split = None
+    label = None
+    if parts:
+        first = parts[0].lower()
+        if first in {"train", "test", "val", "valid", "validation"}:
+            split = parts[0]
+            if len(parts) > 1:
+                label = parts[-1]
+        else:
+            label = parts[-1]
+
+    record: Dict[str, Any] = {
+        "id": index,
+        "image_path": os.path.abspath(full_path),
+        "relative_path": relative_path,
+        "file_name": os.path.basename(full_path),
+        "extension": os.path.splitext(full_path)[1].lower().lstrip("."),
+    }
+    if split:
+        record["split"] = split
+    if label:
+        record["label"] = label
+    return record
+
+
+def _write_image_manifest(root: str, image_files: List[str]) -> str:
+    manifest_path = os.path.join(root, "_vesper_image_manifest.jsonl")
+    with open(manifest_path, "w", encoding="utf-8") as handle:
+        for index, full_path in enumerate(image_files):
+            handle.write(json.dumps(_infer_image_record(root, full_path, index), ensure_ascii=False) + "\n")
+    return manifest_path
+
+
+def _pick_best_file(root: str) -> Dict[str, Any]:
     candidates: List[str] = []
     for base, _, files in os.walk(root):
         for name in files:
@@ -145,6 +207,14 @@ def _pick_best_file(root: str) -> str:
                 candidates.append(full)
 
     if not candidates:
+        image_files = _find_image_files(root)
+        if image_files:
+            manifest_path = _write_image_manifest(root, image_files)
+            return {
+                "local_path": manifest_path,
+                "dataset_kind": "image-manifest",
+                "image_count": len(image_files),
+            }
         raise RuntimeError("No suitable data file found after download")
 
     # prioritize common tabular formats
@@ -152,8 +222,8 @@ def _pick_best_file(root: str) -> str:
     for ext in priorities:
         for c in candidates:
             if c.lower().endswith(ext):
-                return c
-    return candidates[0]
+                return {"local_path": c, "dataset_kind": "tabular", "image_count": 0}
+    return {"local_path": candidates[0], "dataset_kind": "tabular", "image_count": 0}
 
 
 def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
@@ -174,12 +244,14 @@ def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
 
         # unzip in place, remove zip for convenience
         api.dataset_download_files(dataset_ref, path=target_dir, unzip=True, quiet=True)
-        best_file = _pick_best_file(target_dir)
+        artifact = _pick_best_file(target_dir)
         return {
             "ok": True,
             "dataset_id": dataset_ref,
             "target_dir": target_dir,
-            "local_path": best_file,
+            "local_path": artifact["local_path"],
+            "dataset_kind": artifact["dataset_kind"],
+            "image_count": artifact.get("image_count", 0),
         }
     except Exception as e:
         msg = str(e)

package/src/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/src/python/requirements.txt

@@ -0,0 +1,12 @@
+polars==1.2.0
+pandas==2.2.0
+numpy==1.26.0
+scikit-learn==1.4.0
+# Optional source/download extras:
+kaggle>=1.6.17
+aiohttp>=3.9.0
+aiofiles>=24.1.0
+datasets>=2.20.0
+webdataset>=0.2.86
+# Optional for secure key storage (preferred over file fallback):
+# keyring>=24.0.0