@vespermcp/mcp-server 1.2.21 → 1.2.24

package/src/python/kaggle_engine.py

@@ -12,6 +12,19 @@ except Exception:
     HAS_KAGGLE = False
 
 
+IMAGE_EXTENSIONS = {
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".webp",
+    ".bmp",
+    ".gif",
+    ".tiff",
+    ".tif",
+    ".svg",
+}
+
+
 def _ensure_auth() -> Dict[str, Any]:
     if not HAS_KAGGLE:
         return {
@@ -135,7 +148,56 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}
 
 
-def _pick_best_file(root: str) -> str:
+def _find_image_files(root: str) -> List[str]:
+    image_files: List[str] = []
+    for base, _, files in os.walk(root):
+        for name in files:
+            full = os.path.join(base, name)
+            if os.path.splitext(name)[1].lower() in IMAGE_EXTENSIONS:
+                image_files.append(full)
+    image_files.sort()
+    return image_files
+
+
+def _infer_image_record(root: str, full_path: str, index: int) -> Dict[str, Any]:
+    relative_path = os.path.relpath(full_path, root).replace("\\", "/")
+    parent_dir = os.path.dirname(relative_path)
+    parts = [part for part in parent_dir.split("/") if part and part != "."]
+
+    split = None
+    label = None
+    if parts:
+        first = parts[0].lower()
+        if first in {"train", "test", "val", "valid", "validation"}:
+            split = parts[0]
+            if len(parts) > 1:
+                label = parts[-1]
+        else:
+            label = parts[-1]
+
+    record: Dict[str, Any] = {
+        "id": index,
+        "image_path": os.path.abspath(full_path),
+        "relative_path": relative_path,
+        "file_name": os.path.basename(full_path),
+        "extension": os.path.splitext(full_path)[1].lower().lstrip("."),
+    }
+    if split:
+        record["split"] = split
+    if label:
+        record["label"] = label
+    return record
+
+
+def _write_image_manifest(root: str, image_files: List[str]) -> str:
+    manifest_path = os.path.join(root, "_vesper_image_manifest.jsonl")
+    with open(manifest_path, "w", encoding="utf-8") as handle:
+        for index, full_path in enumerate(image_files):
+            handle.write(json.dumps(_infer_image_record(root, full_path, index), ensure_ascii=False) + "\n")
+    return manifest_path
+
+
+def _pick_best_file(root: str) -> Dict[str, Any]:
     candidates: List[str] = []
     for base, _, files in os.walk(root):
         for name in files:
@@ -145,6 +207,14 @@ def _pick_best_file(root: str) -> str:
                 candidates.append(full)
 
     if not candidates:
+        image_files = _find_image_files(root)
+        if image_files:
+            manifest_path = _write_image_manifest(root, image_files)
+            return {
+                "local_path": manifest_path,
+                "dataset_kind": "image-manifest",
+                "image_count": len(image_files),
+            }
         raise RuntimeError("No suitable data file found after download")
 
     # prioritize common tabular formats
@@ -152,8 +222,8 @@ def _pick_best_file(root: str) -> str:
     for ext in priorities:
         for c in candidates:
             if c.lower().endswith(ext):
-                return c
-    return candidates[0]
+                return {"local_path": c, "dataset_kind": "tabular", "image_count": 0}
+    return {"local_path": candidates[0], "dataset_kind": "tabular", "image_count": 0}
 
 
 def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
@@ -174,12 +244,14 @@ def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
 
         # unzip in place, remove zip for convenience
         api.dataset_download_files(dataset_ref, path=target_dir, unzip=True, quiet=True)
-        best_file = _pick_best_file(target_dir)
+        artifact = _pick_best_file(target_dir)
         return {
             "ok": True,
             "dataset_id": dataset_ref,
             "target_dir": target_dir,
-            "local_path": best_file,
+            "local_path": artifact["local_path"],
+            "dataset_kind": artifact["dataset_kind"],
+            "image_count": artifact.get("image_count", 0),
         }
     except Exception as e:
         msg = str(e)

package/src/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/src/python/requirements.txt

@@ -0,0 +1,12 @@
+polars==1.2.0
+pandas==2.2.0
+numpy==1.26.0
+scikit-learn==1.4.0
+# Optional source/download extras:
+kaggle>=1.6.17
+aiohttp>=3.9.0
+aiofiles>=24.1.0
+datasets>=2.20.0
+webdataset>=0.2.86
+# Optional for secure key storage (preferred over file fallback):
+# keyring>=24.0.0

package/src/python/vesper/core/asset_downloader.py

@@ -191,6 +191,7 @@ class AssetDownloader:
         kaggle_ref: Optional[str] = None,
         urls: Optional[List[str]] = None,
         output_format: str = "webdataset",
+        output_dir: Optional[str] = None,
         max_items: Optional[int] = None,
         image_column: Optional[str] = None,
     ) -> Dict[str, Any]:
@@ -231,7 +232,10 @@ class AssetDownloader:
                 raise ValueError("urls are required for source=url")
 
         # --- Now safe to create directories ---
-        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
+        if output_dir:
+            dataset_dir = Path(output_dir).expanduser().resolve()
+        else:
+            dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
         images_dir = dataset_dir / "images"
         dataset_dir.mkdir(parents=True, exist_ok=True)
         images_dir.mkdir(parents=True, exist_ok=True)

package/wizard.cjs

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require('./scripts/wizard.cjs');