@vespermcp/mcp-server 1.0.4 → 1.0.6

package/build/metadata/store.js

@@ -17,7 +17,7 @@ export class MetadataStore {
         // Add install_path if missing
         try {
             this.db.exec("ALTER TABLE datasets ADD COLUMN install_path TEXT");
-            console.log("[MetadataStore] Migrated: Added install_path column");
+            console.error("[MetadataStore] Migrated: Added install_path column");
         }
         catch (e) {
             // Probably already exists
@@ -315,10 +315,25 @@ export class MetadataStore {
      * Perform database maintenance (VACUUM, ANALYZE).
      */
     optimize() {
-        console.log("[MetadataStore] Running database optimization (VACUUM, ANALYZE)...");
+        console.error("[MetadataStore] Running database optimization (VACUUM, ANALYZE)...");
         this.db.exec("VACUUM");
         this.db.exec("ANALYZE");
     }
+    /**
+     * Delete all datasets from a specific source
+     */
+    deleteBySource(source) {
+        const info = this.db.prepare("DELETE FROM datasets WHERE source = ?").run(source);
+        console.error(`[MetadataStore] Deleted ${info.changes} datasets from source: ${source}`);
+        return info.changes;
+    }
+    /**
+     * Get all dataset IDs from a specific source
+     */
+    getDatasetIdsBySource(source) {
+        const rows = this.db.prepare("SELECT id FROM datasets WHERE source = ?").all(source);
+        return rows.map(r => r.id);
+    }
     close() {
         this.db.close();
     }

package/build/monitoring/observability.js

@@ -2,11 +2,11 @@ export class MockErrorTracker {
     exceptions = [];
     messages = [];
     captureException(error, context) {
-        console.log(`[ErrorTracker] Exception captured: ${error.message}`);
+        console.error(`[ErrorTracker] Exception captured: ${error.message}`);
         this.exceptions.push({ error, context });
     }
     captureMessage(message, level = "info") {
-        console.log(`[ErrorTracker] Message captured (${level}): ${message}`);
+        console.error(`[ErrorTracker] Message captured (${level}): ${message}`);
         this.messages.push({ message, level });
     }
 }

package/build/preparation/target-detector.js

@@ -0,0 +1,75 @@
+import { spawn } from "child_process";
+import path from "path";
+import fs from "fs";
+export class TargetDetector {
+    pythonPath = "python";
+    scriptPath;
+    constructor(buildDir = process.cwd()) {
+        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        // Use same robust path resolution as other services
+        const scriptPath0 = path.resolve(dataRoot, "python", "target_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "target_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "target_engine.py");
+        const scriptPath3 = path.resolve(buildDir, "..", "python", "target_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else if (fs.existsSync(scriptPath3)) {
+            this.scriptPath = scriptPath3;
+        }
+        else {
+            this.scriptPath = scriptPath0;
+        }
+        if (process.platform === "win32") {
+            this.pythonPath = "py";
+        }
+    }
+    /**
+     * Detect probable target columns in a dataset
+     */
+    async detectTarget(filePath) {
+        return this.runPython("detect", [filePath]);
+    }
+    /**
+     * Validate a specific column as a target
+     */
+    async validateTarget(filePath, targetColumn) {
+        return this.runPython("validate", [filePath, targetColumn]);
+    }
+    async runPython(action, args) {
+        return new Promise((resolve, reject) => {
+            const childProcess = spawn(this.pythonPath, [this.scriptPath, action, ...args], {
+                env: { ...process.env, PYTHONIOENCODING: 'utf-8' }
+            });
+            let stdout = "";
+            let stderr = "";
+            childProcess.stdout.on("data", (data) => stdout += data.toString());
+            childProcess.stderr.on("data", (data) => stderr += data.toString());
+            childProcess.on("close", (code) => {
+                if (code !== 0) {
+                    reject(new Error(`Target Detector (${action}) failed: ${stderr}`));
+                    return;
+                }
+                try {
+                    const result = JSON.parse(stdout);
+                    if (result.error) {
+                        reject(new Error(result.error));
+                    }
+                    else {
+                        resolve(result);
+                    }
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse target detector output: ${stdout}\nError: ${e}`));
+                }
+            });
+        });
+    }
+}

package/build/python/cleaner.py

@@ -0,0 +1,226 @@
+import sys
+import json
+import polars as pl
+import numpy as np
+
+# --- Operations Library ---
+
+def op_remove_duplicates(df, params):
+    subset = params.get("subset", None) # List of cols or None
+    before = len(df)
+    if subset:
+        df = df.unique(subset=subset)
+    else:
+        df = df.unique()
+    return df, {"rows_removed": before - len(df)}
+
+def op_drop_columns(df, params):
+    cols = params.get("columns", [])
+    before = len(df.columns)
+    # Filter only existing cols to avoid errors
+    cols_to_drop = [c for c in cols if c in df.columns]
+    df = df.drop(cols_to_drop)
+    return df, {"columns_dropped": len(cols_to_drop)}
+
+def op_fill_missing(df, params):
+    col = params["column"]
+    method = params.get("method", "mean") # mean, median, mode, constant
+    value = params.get("value", None)
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    affected = df[col].null_count()
+    
+    if method == "constant":
+        df = df.with_columns(pl.col(col).fill_null(value))
+    elif method == "mean":
+        mean_val = df[col].mean()
+        df = df.with_columns(pl.col(col).fill_null(mean_val))
+    elif method == "median":
+        median_val = df[col].median()
+        df = df.with_columns(pl.col(col).fill_null(median_val))
+    
+    return df, {"rows_imputed": affected}
+
+def op_fix_types(df, params):
+    col = params["column"]
+    target_type = params["type"] # "int", "float", "string", "date"
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    try:
+        if target_type == "int":
+            df = df.with_columns(pl.col(col).cast(pl.Int64, strict=False))
+        elif target_type == "float":
+            df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
+        elif target_type == "string":
+            df = df.with_columns(pl.col(col).cast(pl.Utf8))
+        elif target_type == "date":
+             df = df.with_columns(pl.col(col).str.to_date(strict=False))
+        
+        return df, {"status": "Converted"}
+    except Exception as e:
+        return df, {"error": str(e)}
+
+def op_remove_outliers(df, params):
+    col = params["column"]
+    method = params.get("method", "iqr")
+    threshold = params.get("threshold", 1.5)
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    before = len(df)
+    
+    if method == "iqr":
+        q1 = df[col].quantile(0.25)
+        q3 = df[col].quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - (threshold * iqr)
+        upper = q3 + (threshold * iqr)
+        
+        df = df.filter((pl.col(col) >= lower) & (pl.col(col) <= upper))
+    
+    return df, {"rows_removed": before - len(df)}
+
+def op_encode_categories(df, params):
+    col = params["column"]
+    method = params.get("method", "label") # label, onehot
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    if method == "label":
+        # Polars dense_rank acts similar to label encoding
+        df = df.with_columns(pl.col(col).rank("dense").alias(f"{col}_encoded"))
+    elif method == "onehot":
+        dummies = df[col].to_dummies()
+        df = pl.concat([df, dummies], how="horizontal")
+    
+    return df, {"status": f"Encoded using {method}"}
+
+# --- Registry ---
+
+OPERATIONS = {
+    "RemoveDuplicates": op_remove_duplicates,
+    "DropColumns": op_drop_columns,
+    "FillMissing": op_fill_missing,
+    "FixTypes": op_fix_types,
+    "RemoveOutliers": op_remove_outliers,
+    "EncodeCategories": op_encode_categories
+}
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"error": "Usage: cleaner.py <file_path> <operations_json>"}), file=sys.stderr)
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+    ops_json = sys.argv[2]
+    
+    try:
+        operations = json.loads(ops_json)
+        
+        # Load Data
+        file_path_lower = file_path.lower()
+        if file_path_lower.endswith(".csv"):
+            df = pl.read_csv(file_path, ignore_errors=True)
+        elif file_path_lower.endswith(".parquet"):
+            df = pl.read_parquet(file_path)
+        elif file_path_lower.endswith(".jsonl") or file_path_lower.endswith(".ndjson"):
+            # Explicit NDJSON
+            df = pl.read_ndjson(file_path)
+        elif file_path_lower.endswith(".json"):
+            # Ambiguous .json
+            try:
+                df = pl.read_json(file_path)
+            except Exception:
+                try:
+                     df = pl.read_ndjson(file_path)
+                except Exception as e:
+                     raise ValueError(f"Failed to read JSON: {str(e)}")
+        else:
+            raise ValueError(f"Unsupported format: {file_path}")
+
+        logs = []
+        total_rows_affected = 0
+
+        # Execute Pipeline
+        for op in operations:
+            op_type = op["type"]
+            params = op.get("params", {})
+            
+            if op_type == "RenameTarget":
+                old_name = params.get("old_name")
+                new_name = params.get("new_name", "target")
+                if old_name and old_name in df.columns:
+                    df = df.rename({old_name: new_name})
+                    logs.append(f"Renamed column '{old_name}' to '{new_name}'")
+                else:
+                    logs.append(f"Failed RenameTarget: Column '{old_name}' not found or not specified.")
+            elif op_type in OPERATIONS:
+                try:
+                    df, stats = OPERATIONS[op_type](df, params)
+                    logs.append(f"Executed {op_type}: {stats}")
+                    total_rows_affected += stats.get("rows_removed", 0)
+                except Exception as e:
+                    logs.append(f"Failed {op_type}: {str(e)}")
+            else:
+                logs.append(f"Unknown operation: {op_type}")
+
+        # Save Result (overwrite or new file)
+        # Save Result (overwrite or new file)
+        output_format = sys.argv[3] if len(sys.argv) > 3 else None
+        
+        if not output_format:
+            # Legacy logic: preserve CSV or default to parquet
+            if file_path_lower.endswith(".csv"):
+                output_format = "csv"
+            else:
+                output_format = "parquet"
+
+        base_name = file_path.rsplit(".", 1)[0]
+        if output_format == "csv":
+            output_path = f"{base_name}_cleaned.csv"
+            # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)
+            for col in df.columns:
+                dtype = df.schema[col]
+                # Only keep simple types; stringify everything else for CSV
+                is_simple = (
+                    dtype.is_numeric() or 
+                    dtype.is_temporal() or 
+                    str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
+                )
+                if not is_simple:
+                    # Use a robust helper for clean JSON serialization
+                    def safe_serialize(val):
+                        try:
+                            # Handle Polars nested types (convert to Python list/dict first)
+                            if hasattr(val, "to_list"):
+                                return json.dumps(val.to_list())
+                            if hasattr(val, "to_dict"):
+                                return json.dumps(val.to_dict())
+                            return json.dumps(val)
+                        except:
+                            return str(val)
+                    df = df.with_columns(pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8))
+            df.write_csv(output_path)
+        else:
+            output_path = f"{base_name}_cleaned.parquet"
+            df.write_parquet(output_path)
+
+        print(json.dumps({
+            "success": True,
+            "output_path": output_path,
+            "rows_affected": total_rows_affected,
+            "logs": logs
+        }, default=str))
+
+    except Exception as e:
+        print(json.dumps({"success": False, "error": str(e)}, default=str))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

package/build/python/export_engine.py

@@ -0,0 +1,131 @@
+import sys
+import json
+import polars as pl
+import os
+
+# Optional TensorFlow import for TFRecord support
+try:
+    import tensorflow as tf
+    HAS_TENSORFLOW = True
+except ImportError:
+    HAS_TENSORFLOW = False
+
+def export_data(file_path, output_path, format, options=None):
+    options = options or {}
+    
+    # Load Data
+    try:
+        if file_path.endswith(".csv"):
+            df = pl.read_csv(file_path, ignore_errors=True)
+        elif file_path.endswith(".parquet"):
+            df = pl.read_parquet(file_path)
+        else:
+            return {"error": f"Unsupported input format: {file_path}"}
+    except Exception as e:
+        return {"error": f"Failed to load input file: {str(e)}"}
+
+    output_dir = os.path.dirname(output_path)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir, exist_ok=True)
+
+    try:
+        # Export Logic
+        if format == "csv":
+            # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)
+            for col in df.columns:
+                dtype = df.schema[col]
+                is_simple = (
+                    dtype.is_numeric() or 
+                    dtype.is_temporal() or 
+                    str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
+                )
+                if not is_simple:
+                    def safe_serialize(val):
+                        try:
+                            if hasattr(val, "to_list"):
+                                return json.dumps(val.to_list())
+                            if hasattr(val, "to_dict"):
+                                return json.dumps(val.to_dict())
+                            return json.dumps(val)
+                        except:
+                            return str(val)
+                    df = df.with_columns(pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8))
+            df.write_csv(output_path)
+            
+        elif format == "parquet":
+            compression = options.get("compression", "snappy")
+            df.write_parquet(output_path, compression=compression)
+            
+        elif format == "jsonl":
+            df.write_ndjson(output_path)
+            
+        elif format == "arrow" or format == "ipc":
+            compression = options.get("compression", "uncompressed")
+            if compression == "uncompressed": compression = None
+            df.write_ipc(output_path, compression=compression)
+
+        elif format == "tfrecord":
+            if not HAS_TENSORFLOW:
+                return {"error": "TensorFlow is not installed. Cannot export to TFRecord."}
+            
+            # TFRecord Export Logic (using TensorFlow)
+            with tf.io.TFRecordWriter(output_path) as writer:
+                # Convert Polars -> Pandas for iteration (simpler for now)
+                # TODO: Optimize with Arrow -> Tensor conversion if needed for massive data
+                pdf = df.to_pandas()
+                for _, row in pdf.iterrows():
+                    feature = {}
+                    for col, value in row.items():
+                        if value is None:
+                            continue
+                        
+                        # Type inference for TFRecord features
+                        if isinstance(value, int):
+                            feature[col] = tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+                        elif isinstance(value, float):
+                            feature[col] = tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+                        elif isinstance(value, str):
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode('utf-8')]))
+                        elif isinstance(value, bytes):
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+                        else:
+                            # Fallback to string for unknown types
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value).encode('utf-8')]))
+                            
+                    example = tf.train.Example(features=tf.train.Features(feature=feature))
+                    writer.write(example.SerializeToString())
+
+        else:
+            return {"error": f"Unknown export format: {format}"}
+
+        return {
+            "success": True,
+            "output_path": output_path,
+            "rows": len(df),
+            "format": format
+        }
+
+    except Exception as e:
+        return {"error": f"Export failed: {str(e)}"}
+
+def main():
+    if len(sys.argv) < 4:
+        print(json.dumps({"error": "Usage: export_engine.py <input_file> <output_file> <format> [options_json]"}), file=sys.stderr)
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    fmt = sys.argv[3]
+    
+    options = {}
+    if len(sys.argv) > 4:
+        try:
+            options = json.loads(sys.argv[4])
+        except:
+            pass
+
+    result = export_data(input_file, output_file, fmt, options)
+    print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/build/python/framework_adapters.py

@@ -0,0 +1,100 @@
+import os
+import json
+import warnings
+
+# --- PyTorch Adapter ---
+try:
+    import torch
+    from torch.utils.data import Dataset
+    import polars as pl
+    
+    class VesperPyTorchDataset(Dataset):
+        """
+        PyTorch Dataset wrapper for Vesper exported files (Parquet/CSV/Arrow).
+        Efficiently loads data using Polars and converts to Tensors on demand.
+        """
+        def __init__(self, file_path, target_col=None, transform=None):
+            self.file_path = file_path
+            self.target_col = target_col
+            self.transform = transform
+            
+            # Auto-detect format
+            if file_path.endswith(".parquet"):
+                self.df = pl.read_parquet(file_path)
+            elif file_path.endswith(".csv"):
+                self.df = pl.read_csv(file_path, ignore_errors=True)
+            elif file_path.endswith(".arrow"):
+                self.df = pl.read_ipc(file_path)
+            else:
+                raise ValueError(f"Unsupported file format for PyTorch loader: {file_path}")
+                
+            self.data = self.df.to_pandas() # Convert to pandas for easier row access in __getitem__ (Polars slice can be slow row-wise)
+
+        def __len__(self):
+            return len(self.data)
+
+        def __getitem__(self, idx):
+            row = self.data.iloc[idx]
+            
+            # Simple assumption: all numeric columns except target are features
+            # In production, metadata would tell us which columns are features
+            if self.target_col and self.target_col in row:
+                y = row[self.target_col]
+                x = row.drop(self.target_col).values
+                
+                # Convert to tensors
+                x = torch.tensor(x, dtype=torch.float32)
+                # Auto-detect target type (scalar vs class index)
+                if isinstance(y, (int, float)):
+                     y = torch.tensor(y, dtype=torch.float32) # Regression/Binary
+                else:
+                    # TODO: Label encoding if string
+                    pass
+                
+                sample = (x, y)
+            else:
+                # Unsupervised
+                x = torch.tensor(row.values, dtype=torch.float32)
+                sample = x
+
+            if self.transform:
+                sample = self.transform(sample)
+
+            return sample
+
+except ImportError:
+    class VesperPyTorchDataset:
+        def __init__(self, *args, **kwargs):
+            raise ImportError("PyTorch or Polars not installed.")
+
+# --- HuggingFace Adapter ---
+try:
+    from datasets import load_dataset as hf_load_dataset
+    
+    def load_vesper_dataset(file_path):
+        """
+        Loads a Vesper export into a Hugging Face Dataset.
+        Supported: Parquet, CSV, JSONL, Arrow.
+        """
+        output_format = "parquet" # Default fallback
+        if file_path.endswith(".csv"): output_format = "csv"
+        elif file_path.endswith(".jsonl"): output_format = "json"
+        elif file_path.endswith(".arrow"): output_format = "arrow"
+        
+        # 'arrow' format in HF might need custom script, but usually parquet/csv/json are native
+        if output_format == "arrow":
+             # Use pandas/polars to read then convert to HF dataset
+             import polars as pl
+             from datasets import Dataset
+             df = pl.read_ipc(file_path).to_pandas()
+             return Dataset.from_pandas(df)
+        
+        return hf_load_dataset(output_format, data_files=file_path, split="train")
+
+except ImportError:
+    def load_vesper_dataset(*args, **kwargs):
+        raise ImportError("HuggingFace 'datasets' library not installed.")
+
+if __name__ == "__main__":
+    print("Vesper Framework Adapters Library")
+    print("Usage: import this module in your training script.")

package/build/python/github_adapter.py

@@ -0,0 +1,106 @@
+import sys
+import json
+import argparse
+import urllib.request
+import urllib.parse
+import os
+from datetime import datetime
+
+GITHUB_API_URL = "https://api.github.com/search/repositories"
+
+def search_github(query: str, limit: int = 10):
+    """
+    Search GitHub for dataset repositories.
+    """
+    try:
+        # Construct refined query:
+        # User query + (topic:dataset OR topic:data)
+        # We also filter for repositories with > 5 stars to ensure some relevance
+        refined_query = f"{query} topic:dataset stars:>5"
+        
+        params = {
+            "q": refined_query,
+            "sort": "stars",
+            "order": "desc",
+            "per_page": limit
+        }
+        
+        query_string = urllib.parse.urlencode(params)
+        url = f"{GITHUB_API_URL}?{query_string}"
+        
+        req = urllib.request.Request(url)
+        
+        # Add User-Agent (Required by GitHub)
+        req.add_header("User-Agent", "Vesper-Dataset-Search")
+        
+        # Add Authorization if token exists
+        token = os.environ.get("GITHUB_TOKEN")
+        if token:
+            req.add_header("Authorization", f"token {token}")
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.load(response)
+        
+        items = data.get('items', [])
+        results = []
+        
+        for item in items:
+            # Map GitHub fields to Vesper schema
+            # repo: owner/name
+            repo_id = item.get("full_name")
+            
+            # Simple licensing map
+            license_info = item.get("license") or {}
+            license_key = license_info.get("key", "unknown")
+            license_category = "safe" if license_key in ["mit", "apache-2.0", "cc0-1.0", "bsd-3-clause"] else "unknown"
+            
+            metadata = {
+                "id": f"github:{repo_id}", 
+                "source": "github",
+                "name": item.get("name"),
+                "description": item.get("description") or "No description provided.",
+                "downloads": item.get("forks_count") * 10, # Proxy estimation
+                "likes": item.get("stargazers_count"),
+                "stars": item.get("stargazers_count"),
+                "last_updated": item.get("updated_at"),
+                "quality_score": min(100, 50 + (item.get("stargazers_count", 0))), # Rough heuristic
+                "license": {
+                    "id": license_key,
+                    "name": license_info.get("name", "Unknown"),
+                    "category": license_category,
+                    "usage_restrictions": [],
+                    "warnings": []
+                },
+                "tags": item.get("topics", []),
+                "total_examples": 0, # Unknown without drilling deeper
+                "is_safe_source": True, # GitHub is generally safe code, content varies
+                "is_structured": False, # Often contains code + data
+                "metadata_url": item.get("html_url"),
+                "domain": "general"
+            }
+            
+            results.append(metadata)
+            
+        return results
+
+    except urllib.error.HTTPError as e:
+        if e.code == 403:
+            return {"error": "Rate limit exceeded. Set GITHUB_TOKEN environment variable."}
+        return {"error": f"HTTP Error {e.code}: {e.reason}"}
+    except Exception as e:
+        return {"error": str(e)}
+
+def main():
+    parser = argparse.ArgumentParser(description="GitHub Adapter")
+    parser.add_argument("--action", required=True, choices=["search"])
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--limit", type=int, default=10)
+    
+    args = parser.parse_args()
+    
+    if args.action == "search":
+        results = search_github(args.query, args.limit)
+        print(json.dumps(results))
+
+if __name__ == "__main__":
+    main()