@vespermcp/mcp-server 1.2.21 → 1.2.22

package/build/ingestion/ingestor.js

@@ -46,6 +46,9 @@ export class DataIngestor {
     getKaggleCredentialError() {
         return "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds), or provide ~/.kaggle/kaggle.json.";
     }
+    toSafeDatasetPath(datasetId) {
+        return datasetId.replace(/[:\/]/g, "_");
+    }
     /**
      * Ensures a dataset is available locally
      */
@@ -115,7 +118,7 @@ export class DataIngestor {
                 this.failDownload(datasetId, errorMsg);
                 throw new Error(errorMsg);
             }
-            const targetDir = path.join(this.rawDataDir, datasetId.replace(/\//g, "_"));
+            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
             this.store.registerDownload(datasetId, targetDir, "downloading");
             try {
                 onProgress?.("Downloading from Kaggle...");
@@ -131,7 +134,7 @@ export class DataIngestor {
             }
         }
         else if (source === "openml") {
-            const targetDir = path.join(this.rawDataDir, datasetId.replace(/:/g, "_"));
+            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
             this.store.registerDownload(datasetId, targetDir, "downloading");
             try {
                 onProgress?.("Downloading from OpenML...");
@@ -147,7 +150,7 @@ export class DataIngestor {
             }
         }
         else if (source === "dataworld") {
-            const targetDir = path.join(this.rawDataDir, datasetId.replace(/[:\/]/g, "_"));
+            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
             this.store.registerDownload(datasetId, targetDir, "downloading");
             try {
                 onProgress?.("Downloading from data.world...");
@@ -181,7 +184,7 @@ export class DataIngestor {
      * Generates a safe local filename for a dataset ID
      */
     getTargetPath(datasetId, extension = "parquet") {
-        const safeId = datasetId.replace(/\//g, "_").replace(/:/g, "_");
+        const safeId = this.toSafeDatasetPath(datasetId);
         return path.join(this.rawDataDir, `${safeId}.${extension}`);
     }
     /**

package/build/install/install-service.js

@@ -18,12 +18,15 @@ export class InstallService {
             throw new Error(`Source file not found for installation: ${sourcePath}`);
         }
         const dataset = this.metadataStore.getDataset(datasetId);
-        if (!dataset) {
-            throw new Error(`Dataset metadata not found for ${datasetId}`);
-        }
         // Create target directory
-        const sanitizedName = dataset.name.replace(/[^a-z0-9]/gi, "_").toLowerCase();
-        const installDir = targetDir || path.join(this.projectRoot, "datasets", sanitizedName);
+        const installLabel = dataset?.name || datasetId;
+        const sanitizedName = installLabel.replace(/[^a-z0-9]/gi, "_").toLowerCase();
+        // If caller specified a target dir, use it directly
+        // Otherwise use the current working directory
+        const installDir = targetDir
+            ? path.resolve(targetDir)
+            : path.resolve(process.cwd(), sanitizedName);
+        console.error(`[InstallService] Resolved install directory: ${installDir}`);
         if (!fs.existsSync(installDir)) {
             fs.mkdirSync(installDir, { recursive: true });
         }
@@ -34,7 +37,9 @@ export class InstallService {
         fs.copyFileSync(sourcePath, targetPath);
         // Update metadata
         const absolutePath = path.resolve(targetPath);
-        this.metadataStore.updateInstallPath(datasetId, absolutePath);
+        if (dataset) {
+            this.metadataStore.updateInstallPath(datasetId, absolutePath);
+        }
         console.error(`[InstallService] Dataset ${datasetId} installed to ${absolutePath}`);
         return absolutePath;
     }

package/build/lib/supabase.js

@@ -0,0 +1,3 @@
+import { createClient } from '@supabase/supabase-js';
+export const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY // for MCP, use service_role, not anon
+);

package/build/metadata/scraper.js

@@ -3,22 +3,29 @@ import { categorizeLicense } from "./license.js";
 import { calculateQualityScore } from "./quality.js";
 import { classifyDomain } from "./domain.js";
 import { retryWithBackoff, delayBetweenRequests } from "./rate-limiter.js";
+import { analyzeDatasetQuery, buildIntentSearchQuery, buildHuggingFaceFilterTags, scoreDatasetAgainstIntent, shouldExcludeByLanguage } from "../search/query-intent.js";
 export class HuggingFaceScraper {
     /**
      * Bulk discovery: Fetch many datasets quickly without deep details.
      * Hits the 25k target in minutes.
      */
-    async scrapeBulk(limit = 1000, query) {
+    async scrapeBulk(limit = 1000, queryOrIntent) {
+        const intent = typeof queryOrIntent === "string"
+            ? await analyzeDatasetQuery(queryOrIntent)
+            : queryOrIntent;
+        const query = typeof queryOrIntent === "string" ? queryOrIntent : intent?.searchQuery;
+        const hfQuery = intent ? buildIntentSearchQuery(intent) : query;
         const filterMsg = query ? `, query: ${query}` : "";
         console.error(`[Bulk Scraper] Fetching datasets (target limit: ${limit}${filterMsg})...`);
         const results = [];
         let processed = 0;
         try {
             const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
+            const hfFilterTags = intent ? buildHuggingFaceFilterTags(intent) : [];
             for await (const ds of listDatasets({
                 limit: limit,
                 additionalFields: ["description", "tags", "downloadsAllTime", "createdAt"],
-                search: { query: query },
+                search: { query: hfQuery, tags: hfFilterTags.length > 0 ? hfFilterTags : undefined },
                 ...(hfToken ? { accessToken: hfToken } : {})
             })) {
                 if (results.length >= limit)
@@ -78,6 +85,9 @@ export class HuggingFaceScraper {
                     has_readme: false,
                     is_incomplete: true // Flag for Phase 2
                 };
+                // Hard language exclusion
+                if (intent && shouldExcludeByLanguage(metadata, intent))
+                    continue;
                 results.push(metadata);
             }
         }
@@ -86,8 +96,12 @@ export class HuggingFaceScraper {
         }
         return results;
     }
-    async scrape(limit = 100, applyMVPFilters = true, query // Use as general search query
-    ) {
+    async scrape(limit = 100, applyMVPFilters = true, queryOrIntent) {
+        const intent = typeof queryOrIntent === "string"
+            ? await analyzeDatasetQuery(queryOrIntent)
+            : queryOrIntent;
+        const query = typeof queryOrIntent === "string" ? queryOrIntent : intent?.searchQuery;
+        const hfQuery = intent ? buildIntentSearchQuery(intent) : query;
         const filterMsg = query ? `, query: ${query}` : "";
         console.error(`Fetching datasets (target limit: ${limit}, MVP filters: ${applyMVPFilters}${filterMsg})...`);
         const results = [];
@@ -110,10 +124,11 @@ export class HuggingFaceScraper {
             }
             // Add delay between batches to avoid rate limits
             const BATCH_DELAY = hfToken ? 500 : 2000;
+            const hfFilterTags = intent ? buildHuggingFaceFilterTags(intent) : [];
             for await (const ds of listDatasets({
                 limit: fetchLimit,
                 additionalFields: ["description", "tags"],
-                search: { query: query },
+                search: { query: hfQuery, tags: hfFilterTags.length > 0 ? hfFilterTags : undefined },
                 ...(hfToken ? { accessToken: hfToken } : {})
             })) {
                 if (results.length >= limit)
@@ -150,18 +165,61 @@ export class HuggingFaceScraper {
                             initialDelay: 2000, // Start with 2 seconds for HF API
                             maxDelay: 30000 // Max 30 seconds
                         });
-                        const splits = fullInfo.splits?.map((s) => ({
+                        const cardData = fullInfo.cardData || {};
+                        // Extract splits from cardData.dataset_info (where HF actually stores them)
+                        // cardData.dataset_info can be an object (single config) or array (multi-config)
+                        let rawSplits = [];
+                        const datasetInfoField = cardData.dataset_info;
+                        if (datasetInfoField) {
+                            const configs = Array.isArray(datasetInfoField) ? datasetInfoField : [datasetInfoField];
+                            for (const config of configs) {
+                                if (config?.splits && Array.isArray(config.splits)) {
+                                    rawSplits = rawSplits.concat(config.splits);
+                                }
+                            }
+                        }
+                        // Fallback: try top-level splits from the SDK (rarely populated)
+                        if (rawSplits.length === 0 && fullInfo.splits) {
+                            rawSplits = fullInfo.splits;
+                        }
+                        const splits = rawSplits.map((s) => ({
                             name: s.name,
-                            num_examples: s.numExamples || 0,
-                            size_bytes: s.sizeBytes
-                        })) || [];
-                        const totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
+                            num_examples: s.num_examples || s.numExamples || 0,
+                            size_bytes: s.num_bytes || s.sizeBytes || 0
+                        }));
+                        let totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
                         const totalSizeBytes = splits.reduce((sum, s) => sum + (s.size_bytes || 0), 0);
+                        // Fallback: estimate from size_categories when splits give 0
+                        if (totalExamples === 0) {
+                            const sizeCategories = cardData.size_categories;
+                            if (Array.isArray(sizeCategories) && sizeCategories.length > 0) {
+                                const cat = sizeCategories[0];
+                                const rangeMatch = cat.match(/([\d.]+[KMB]?)\s*<\s*n\s*<\s*([\d.]+[KMB]?)/i);
+                                if (rangeMatch) {
+                                    const parseHumanNum = (s) => {
+                                        const m = s.match(/^([\d.]+)([KMB])?$/i);
+                                        if (!m)
+                                            return 0;
+                                        const base = parseFloat(m[1]);
+                                        const suffix = (m[2] || '').toUpperCase();
+                                        if (suffix === 'K')
+                                            return base * 1000;
+                                        if (suffix === 'M')
+                                            return base * 1_000_000;
+                                        if (suffix === 'B')
+                                            return base * 1_000_000_000;
+                                        return base;
+                                    };
+                                    const lo = parseHumanNum(rangeMatch[1]);
+                                    const hi = parseHumanNum(rangeMatch[2]);
+                                    totalExamples = Math.round((lo + hi) / 2);
+                                }
+                            }
+                        }
                         const totalSizeMB = totalSizeBytes ? Math.round(totalSizeBytes / (1024 * 1024) * 100) / 100 : undefined;
                         const hasValidationSplit = splits.some((s) => s.name === "validation" || s.name === "val");
                         const licenseTag = tags.find(t => t.startsWith("license:"));
                         const licenseId = licenseTag ? licenseTag.replace("license:", "") : fullInfo.license;
-                        const cardData = fullInfo.cardData || {};
                         const licenseUrl = cardData.license?.[0]?.link || cardData.license_link;
                         const license = categorizeLicense(licenseId, licenseUrl);
                         if (license.category === "restricted") {
@@ -247,7 +305,16 @@ export class HuggingFaceScraper {
                             description_length: description.length,
                             has_readme: !!(cardData.readme || cardData.readme_content)
                         };
-                        results.push(metadata);
+                        // Hard language exclusion — drop bilingual/multilingual for single-language queries
+                        if (intent && shouldExcludeByLanguage(metadata, intent)) {
+                            // skip — do not push
+                        }
+                        else {
+                            if (intent) {
+                                metadata.intent_score = scoreDatasetAgainstIntent(metadata, intent);
+                            }
+                            results.push(metadata);
+                        }
                     }
                     catch (e) {
                         // Track all errors for user feedback
@@ -297,8 +364,12 @@ export class HuggingFaceScraper {
         if (otherErrors > 0) {
             console.error(`[HF Scraper] ⚠️ ${otherErrors} datasets skipped due to errors`);
         }
-        // Sort by downloads descending
-        return results.sort((a, b) => b.downloads - a.downloads);
+        return results.sort((a, b) => {
+            const intentDelta = Number(b.intent_score || 0) - Number(a.intent_score || 0);
+            if (intentDelta !== 0)
+                return intentDelta;
+            return b.downloads - a.downloads;
+        });
     }
     extractTask(tags) {
         const taskTags = [

package/build/python/asset_downloader_engine.py

@@ -26,6 +26,7 @@ def _print(payload: Dict[str, Any]) -> None:
 async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
     payload = json.loads(args.payload)
     output_root = payload.get("output_root") or str(Path.home() / ".vesper" / "data" / "assets")
+    output_dir = payload.get("output_dir")
     workers = int(payload.get("workers") or 8)
     recipes_dir = payload.get("recipes_dir")
 
@@ -43,6 +44,7 @@ async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
         kaggle_ref=payload.get("kaggle_ref"),
         urls=payload.get("urls"),
         output_format=payload.get("output_format", "webdataset"),
+        output_dir=str(output_dir) if output_dir else None,
         max_items=payload.get("max_items"),
         image_column=payload.get("image_column"),
     )

package/build/python/convert_engine.py

@@ -0,0 +1,92 @@
+"""
+Convert a dataset file between formats (CSV, Parquet, JSON, JSONL).
+Usage: convert_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N, "columns": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required. Install with: pip install polars"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def _write(df: pl.DataFrame, dst: str) -> None:
+    ext = os.path.splitext(dst)[1].lower()
+    os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
+    if ext in (".parquet", ".pq"):
+        df.write_parquet(dst)
+    elif ext == ".csv":
+        df.write_csv(dst)
+    elif ext == ".json":
+        df.write_json(dst, row_oriented=True)
+    elif ext in (".jsonl", ".ndjson"):
+        df.write_ndjson(dst)
+    else:
+        raise ValueError(f"Unsupported output format: {ext}")
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: convert_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        df = _load(input_path)
+        _write(df, output_path)
+        size_mb = round(os.path.getsize(output_path) / (1024 * 1024), 2)
+        print(json.dumps({
+            "ok": True,
+            "output_path": output_path,
+            "rows": df.height,
+            "columns": df.width,
+            "size_mb": size_mb,
+        }))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/build/python/export_engine.py

@@ -50,6 +50,51 @@ def _load(file_path: str, options: dict) -> pl.DataFrame:
         df = pl.read_ipc(file_path)
     elif ext == ".jsonl":
         df = pl.read_ndjson(file_path)
+    elif ext == ".json":
+        # Auto-detect: array-of-objects vs NDJSON vs nested structures
+        try:
+            import json as _json
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                raw_text = fh.read(512)  # peek
+            stripped = raw_text.lstrip()
+            if stripped.startswith("["):
+                # Array of objects — standard JSON
+                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                    data = _json.load(fh)
+                if isinstance(data, list) and len(data) > 0:
+                    df = pl.DataFrame(data)
+                else:
+                    raise ValueError("JSON file is empty or not an array of objects")
+            elif stripped.startswith("{"):
+                # Could be NDJSON or a single object wrapping rows
+                try:
+                    df = pl.read_ndjson(file_path)
+                except Exception:
+                    with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                        data = _json.load(fh)
+                    # Try common wrapper patterns: {"data": [...]}, {"rows": [...]}, etc.
+                    rows = None
+                    if isinstance(data, dict):
+                        for key in ("data", "rows", "records", "items", "results", "entries"):
+                            if key in data and isinstance(data[key], list):
+                                rows = data[key]
+                                break
+                        if rows is None:
+                            # Last resort: try to use the dict values
+                            rows = [data]
+                    if rows and len(rows) > 0:
+                        df = pl.DataFrame(rows)
+                    else:
+                        raise ValueError("Could not parse JSON structure into tabular data")
+            else:
+                raise ValueError("JSON file does not start with [ or {")
+        except pl.exceptions.ComputeError as ce:
+            raise ValueError(f"Failed to parse JSON: {ce}")
+    elif ext == ".xlsx":
+        try:
+            df = pl.read_excel(file_path)
+        except Exception as e:
+            raise ValueError(f"Failed to read Excel file: {e}")
     else:
         raise ValueError(f"Unsupported input format: {ext}")
 

package/build/python/kaggle_engine.py

@@ -12,6 +12,19 @@ except Exception:
     HAS_KAGGLE = False
 
 
+IMAGE_EXTENSIONS = {
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".webp",
+    ".bmp",
+    ".gif",
+    ".tiff",
+    ".tif",
+    ".svg",
+}
+
+
 def _ensure_auth() -> Dict[str, Any]:
     if not HAS_KAGGLE:
         return {
@@ -135,7 +148,56 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}
 
 
-def _pick_best_file(root: str) -> str:
+def _find_image_files(root: str) -> List[str]:
+    image_files: List[str] = []
+    for base, _, files in os.walk(root):
+        for name in files:
+            full = os.path.join(base, name)
+            if os.path.splitext(name)[1].lower() in IMAGE_EXTENSIONS:
+                image_files.append(full)
+    image_files.sort()
+    return image_files
+
+
+def _infer_image_record(root: str, full_path: str, index: int) -> Dict[str, Any]:
+    relative_path = os.path.relpath(full_path, root).replace("\\", "/")
+    parent_dir = os.path.dirname(relative_path)
+    parts = [part for part in parent_dir.split("/") if part and part != "."]
+
+    split = None
+    label = None
+    if parts:
+        first = parts[0].lower()
+        if first in {"train", "test", "val", "valid", "validation"}:
+            split = parts[0]
+            if len(parts) > 1:
+                label = parts[-1]
+        else:
+            label = parts[-1]
+
+    record: Dict[str, Any] = {
+        "id": index,
+        "image_path": os.path.abspath(full_path),
+        "relative_path": relative_path,
+        "file_name": os.path.basename(full_path),
+        "extension": os.path.splitext(full_path)[1].lower().lstrip("."),
+    }
+    if split:
+        record["split"] = split
+    if label:
+        record["label"] = label
+    return record
+
+
+def _write_image_manifest(root: str, image_files: List[str]) -> str:
+    manifest_path = os.path.join(root, "_vesper_image_manifest.jsonl")
+    with open(manifest_path, "w", encoding="utf-8") as handle:
+        for index, full_path in enumerate(image_files):
+            handle.write(json.dumps(_infer_image_record(root, full_path, index), ensure_ascii=False) + "\n")
+    return manifest_path
+
+
+def _pick_best_file(root: str) -> Dict[str, Any]:
     candidates: List[str] = []
     for base, _, files in os.walk(root):
         for name in files:
@@ -145,6 +207,14 @@ def _pick_best_file(root: str) -> str:
                 candidates.append(full)
 
     if not candidates:
+        image_files = _find_image_files(root)
+        if image_files:
+            manifest_path = _write_image_manifest(root, image_files)
+            return {
+                "local_path": manifest_path,
+                "dataset_kind": "image-manifest",
+                "image_count": len(image_files),
+            }
         raise RuntimeError("No suitable data file found after download")
 
     # prioritize common tabular formats
@@ -152,8 +222,8 @@ def _pick_best_file(root: str) -> str:
     for ext in priorities:
         for c in candidates:
             if c.lower().endswith(ext):
-                return c
-    return candidates[0]
+                return {"local_path": c, "dataset_kind": "tabular", "image_count": 0}
+    return {"local_path": candidates[0], "dataset_kind": "tabular", "image_count": 0}
 
 
 def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
@@ -174,12 +244,14 @@ def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
 
         # unzip in place, remove zip for convenience
         api.dataset_download_files(dataset_ref, path=target_dir, unzip=True, quiet=True)
-        best_file = _pick_best_file(target_dir)
+        artifact = _pick_best_file(target_dir)
         return {
             "ok": True,
             "dataset_id": dataset_ref,
             "target_dir": target_dir,
-            "local_path": best_file,
+            "local_path": artifact["local_path"],
+            "dataset_kind": artifact["dataset_kind"],
+            "image_count": artifact.get("image_count", 0),
         }
     except Exception as e:
         msg = str(e)

package/build/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/build/python/vesper/core/asset_downloader.py

@@ -191,6 +191,7 @@ class AssetDownloader:
         kaggle_ref: Optional[str] = None,
         urls: Optional[List[str]] = None,
         output_format: str = "webdataset",
+        output_dir: Optional[str] = None,
         max_items: Optional[int] = None,
         image_column: Optional[str] = None,
     ) -> Dict[str, Any]:
@@ -231,7 +232,10 @@ class AssetDownloader:
                 raise ValueError("urls are required for source=url")
 
         # --- Now safe to create directories ---
-        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
+        if output_dir:
+            dataset_dir = Path(output_dir).expanduser().resolve()
+        else:
+            dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
         images_dir = dataset_dir / "images"
         dataset_dir.mkdir(parents=True, exist_ok=True)
         images_dir.mkdir(parents=True, exist_ok=True)