@vespermcp/mcp-server 1.1.2 → 1.2.0

package/build/index.js

@@ -5,10 +5,13 @@ import { CallToolRequestSchema, ListToolsRequestSchema, ErrorCode, McpError, } f
 import { fileURLToPath } from "url";
 import path from "path";
 import fs from "fs";
+import { spawn } from "child_process";
 import { MetadataStore } from "./metadata/store.js";
 import { VectorStore } from "./search/vector-store.js";
 import { Embedder } from "./search/embedder.js";
 import { SearchEngine } from "./search/engine.js";
+import { HuggingFaceScraper } from "./metadata/scraper.js";
+import { KaggleSource } from "./metadata/kaggle-source.js";
 import { formatSearchResults, formatDatasetInfo, formatJobStatus } from "./tools/formatter.js";
 import { JobManager } from "./jobs/manager.js";
 import { QualityAnalyzer } from "./quality/analyzer.js";
@@ -17,6 +20,7 @@ import { DataCleaner } from "./cleaning/cleaner.js";
 import { PipelineExecutor } from "./cleaning/executor.js";
 import { DataSplitter } from "./splitting/splitter.js";
 import { DataExporter } from "./export/exporter.js";
+import { DataFusionEngine } from "./fusion/engine.js";
 import { DataIngestor } from "./ingestion/ingestor.js";
 import { InstallService } from "./install/install-service.js";
 import { CacheService, MockRedisProvider } from "./cache/service.js";
@@ -24,6 +28,8 @@ import { ImageAnalyzer } from "./quality/image-analyzer.js";
 import { MediaAnalyzer } from "./quality/media-analyzer.js";
 import { QualityOrchestrator } from "./quality/quality-orchestrator.js";
 import { ConfigManager } from "./config/config-manager.js";
+import { SecureKeysManager } from "./config/secure-keys.js";
+import readline from "readline";
 import os from "os";
 // Determine absolute paths relative to the compiled script
 const __filename = fileURLToPath(import.meta.url);
@@ -49,6 +55,50 @@ function logError(err, context) {
     fs.appendFileSync(errorLogPath, msg);
     console.error(`[Vesper] Critical error logged to ${errorLogPath}`);
 }
+function extractRequestedRows(query, requirements) {
+    const text = `${query || ""} ${requirements || ""}`.toLowerCase();
+    const explicit = text.match(/(\d[\d,\s]{1,12})\s*(samples?|rows?|records?)/i);
+    if (explicit) {
+        const n = Number(explicit[1].replace(/[\s,]/g, ""));
+        if (Number.isFinite(n) && n > 0)
+            return n;
+    }
+    const allNums = [...text.matchAll(/\b\d{4,9}\b/g)]
+        .map(m => Number(m[0]))
+        .filter(n => Number.isFinite(n) && n > 0);
+    if (allNums.length > 0)
+        return Math.max(...allNums);
+    return undefined;
+}
+function runPythonJson(scriptPath, args) {
+    const pyCmd = process.platform === "win32" ? "py" : "python";
+    return new Promise((resolve, reject) => {
+        const proc = spawn(pyCmd, [scriptPath, ...args]);
+        let stdout = "";
+        let stderr = "";
+        proc.stdout.on("data", (d) => (stdout += d.toString()));
+        proc.stderr.on("data", (d) => (stderr += d.toString()));
+        proc.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(stderr || stdout || `Python exited with ${code}`));
+                return;
+            }
+            try {
+                resolve(JSON.parse(stdout));
+            }
+            catch {
+                reject(new Error(`Invalid JSON from python helper: ${stdout}`));
+            }
+        });
+    });
+}
+async function countRows(filePath) {
+    const scriptPath = path.join(dataRoot, "python", "row_count.py");
+    const result = await runPythonJson(scriptPath, [filePath]);
+    if (!result.ok)
+        throw new Error(result.error || "Failed to count rows");
+    return Number(result.rows || 0);
+}
 /**
  * Sync Python scripts from the application package to the stable data directory (~/.vesper/python)
  */
@@ -105,6 +155,21 @@ const dataCleaner = new DataCleaner(__dirname);
 const pipelineExecutor = new PipelineExecutor(dataRoot, __dirname);
 const dataSplitter = new DataSplitter(__dirname);
 const dataExporter = new DataExporter(__dirname);
+const fusionEngine = new DataFusionEngine(__dirname);
+const kaggleSource = new KaggleSource(__dirname);
+const secureKeys = new SecureKeysManager(__dirname);
+function hydrateExternalKeys() {
+    const keys = secureKeys.getAll();
+    if (!process.env.HF_TOKEN && !process.env.HUGGINGFACE_TOKEN && keys.hf_token) {
+        process.env.HF_TOKEN = String(keys.hf_token);
+    }
+    if (!process.env.KAGGLE_USERNAME && keys.kaggle_username) {
+        process.env.KAGGLE_USERNAME = String(keys.kaggle_username);
+    }
+    if (!process.env.KAGGLE_KEY && keys.kaggle_key) {
+        process.env.KAGGLE_KEY = String(keys.kaggle_key);
+    }
+}
 // CRITICAL FIX: Pass __dirname (build directory) to analyzers
 // Python scripts are in build/python/, so analyzers should look relative to build/
 // NOT relative to project root (appRoot)
@@ -136,7 +201,7 @@ jobManager.on("processJob", async (job, execute) => {
         console.error(`[Vesper] Starting prepareDatasetTask for job ${job.id}...`);
         const metadata = job.metadata ? JSON.parse(job.metadata) : {};
         switch (job.type) {
-            case "prepare": return await handlePrepareJob(job.id, metadata.query);
+            case "prepare": return await handlePrepareJob(job.id, metadata.query, metadata.requirements);
             case "clean": return await handleCleanJob(job.id, metadata.datasetId, metadata.ops);
             default: throw new Error(`Unhandled job type: ${job.type}`);
         }
@@ -154,8 +219,9 @@ jobManager.on("processJob", async (job, execute) => {
 /**
  * Logic for preparing a dataset (Search + Ingest + Process)
  */
-async function handlePrepareJob(jobId, query) {
+async function handlePrepareJob(jobId, query, requirements) {
     const update = (updates) => jobManager.updateJob(jobId, updates);
+    const requestedRows = extractRequestedRows(query, requirements);
     update({ progress: 10, status_text: "Searching for best dataset matching query..." });
     const results = await searchEngine.search(query, { limit: 1 });
     if (results.length === 0) {
@@ -176,9 +242,59 @@ async function handlePrepareJob(jobId, query) {
     }
     update({ progress: 30, status_text: `Starting download from ${source}...` });
     // ensureData handles download and returns path to the raw file
-    const rawFilePath = await dataIngestor.ensureData(topDataset.id, source, (msg, prog) => {
+    let rawFilePath = await dataIngestor.ensureData(topDataset.id, source, (msg, prog) => {
         update({ status_text: msg, progress: 30 + (prog ? Math.floor(prog * 0.4) : 0) });
     });
+    if (requestedRows && requestedRows > 0) {
+        update({ progress: 62, status_text: `Validating requested sample count (${requestedRows.toLocaleString()})...` });
+        let currentRows = await countRows(rawFilePath);
+        if (currentRows < requestedRows) {
+            update({ progress: 64, status_text: `Only ${currentRows.toLocaleString()} rows found. Fetching more matching datasets...` });
+            const additional = await searchEngine.search(query, { limit: 8 });
+            const sourceFiles = [rawFilePath];
+            let totalRows = currentRows;
+            for (const ds of additional) {
+                if (ds.id === topDataset.id)
+                    continue;
+                try {
+                    const dsSource = ds.source;
+                    if (dsSource === "kaggle" && !dataIngestor.hasKaggleCredentials())
+                        continue;
+                    const p = await dataIngestor.ensureData(ds.id, dsSource, () => undefined);
+                    const r = await countRows(p);
+                    if (r <= 0)
+                        continue;
+                    sourceFiles.push(p);
+                    totalRows += r;
+                    if (totalRows >= requestedRows)
+                        break;
+                }
+                catch {
+                    // ignore candidate failures and continue trying
+                }
+            }
+            if (sourceFiles.length > 1) {
+                update({ progress: 67, status_text: `Fusing ${sourceFiles.length} datasets to meet row target...` });
+                const fusedPath = path.join(dataRoot, "fusion", `prepare_fused_${Date.now()}.feather`);
+                const fusionResult = await fusionEngine.fuse(sourceFiles, fusedPath, {
+                    strategy: "concat",
+                    dedup: true,
+                    run_quality_after: false,
+                    leakage_check: false,
+                    output_format: "feather",
+                    compression: "lz4",
+                    preview: true,
+                });
+                rawFilePath = fusionResult.output_path;
+                currentRows = await countRows(rawFilePath);
+            }
+            if (currentRows < requestedRows) {
+                throw new Error(`Requested ${requestedRows.toLocaleString()} samples, but only ${currentRows.toLocaleString()} available across current matches. ` +
+                    `Try broader query or enable additional sources.`);
+            }
+            update({ progress: 69, status_text: `✅ Sample target met: ${currentRows.toLocaleString()} rows` });
+        }
+    }
     update({ progress: 70, status_text: "Analyzing dataset quality..." });
     const report = await qualityAnalyzer.analyze(rawFilePath);
     // Update local metadata with quality info
@@ -234,10 +350,84 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             type: "string",
                             description: "The search query. Use -term to exclude keywords.",
                         },
+                        enable_jit: {
+                            type: "boolean",
+                            description: "Enable live JIT search when local library results are insufficient (default: false).",
+                        },
+                    },
+                    required: ["query"],
+                },
+            },
+            {
+                name: "discover_datasets",
+                description: "Discover datasets from a specific source. Kaggle is optional and requires user-provided API key.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        query: {
+                            type: "string",
+                            description: "Search query, e.g. 'credit risk'.",
+                        },
+                        source: {
+                            type: "string",
+                            enum: ["huggingface", "kaggle"],
+                            description: "Data source to discover from.",
+                        },
+                        limit: {
+                            type: "number",
+                            description: "Max results to return (default: 10).",
+                        },
                     },
                     required: ["query"],
                 },
             },
+            {
+                name: "download_dataset",
+                description: "Download a dataset by source and ID/slug into local Vesper storage. Kaggle requires optional API key.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        source: {
+                            type: "string",
+                            enum: ["huggingface", "kaggle"],
+                            description: "Dataset source.",
+                        },
+                        dataset_id: {
+                            type: "string",
+                            description: "Dataset ID/slug (e.g. user/dataset for Kaggle or HF).",
+                        },
+                        target_dir: {
+                            type: "string",
+                            description: "Optional target directory for downloaded files.",
+                        }
+                    },
+                    required: ["source", "dataset_id"],
+                },
+            },
+            {
+                name: "configure_kaggle",
+                description: "Optionally store Kaggle API credentials for Kaggle discover/download. Core Vesper works without this.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        username: { type: "string", description: "Kaggle username" },
+                        key: { type: "string", description: "Kaggle API key" }
+                    },
+                    required: ["username", "key"],
+                },
+            },
+            {
+                name: "configure_keys",
+                description: "One-time optional key setup for external sources (Kaggle + gated HF). Core tools do not require keys.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        hf_token: { type: "string", description: "Optional Hugging Face token for gated/private datasets" },
+                        kaggle_username: { type: "string", description: "Optional Kaggle username" },
+                        kaggle_key: { type: "string", description: "Optional Kaggle API key" }
+                    },
+                },
+            },
             {
                 name: "get_dataset_info",
                 description: "Get detailed metadata for a specific dataset by its ID. Returns comprehensive information including license, safety flags, and data characteristics.",
@@ -346,7 +536,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
             },
             {
                 name: "export_dataset",
-                description: "Export an ingested or prepared dataset to a specific local directory.",
+                description: "Export a dataset to a local directory. Use format='feather' (default) for 5-10× faster writes than CSV. Add fast=true to skip quality/cleaning steps.",
                 inputSchema: {
                     type: "object",
                     properties: {
@@ -360,13 +550,93 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                         },
                         format: {
                             type: "string",
-                            enum: ["csv", "parquet"],
-                            description: "Desired output format (default: csv).",
+                            enum: ["feather", "parquet", "csv", "jsonl", "arrow"],
+                            description: "Output format. feather (fastest), parquet (best compression), csv (human-readable). Default: feather.",
+                        },
+                        compression: {
+                            type: "string",
+                            enum: ["lz4", "zstd", "snappy", "gzip", "uncompressed"],
+                            description: "Compression algorithm. Default: lz4 for feather, snappy for parquet, none for csv.",
+                        },
+                        fast: {
+                            type: "boolean",
+                            description: "Skip quality analysis and cleaning – raw export only. Much faster. Default: false.",
+                        },
+                        preview: {
+                            type: "boolean",
+                            description: "Generate a small 500-row CSV preview alongside binary exports. Default: false.",
+                        },
+                        sample_rows: {
+                            type: "number",
+                            description: "Export only this many random rows (faster for huge datasets).",
+                        },
+                        columns: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "Export only these columns (faster for wide datasets).",
                         },
                     },
                     required: ["dataset_id"],
                 },
             },
+            {
+                name: "fuse_datasets",
+                description: "Fuse/combine multiple datasets via concat or join. Optionally runs quality & leakage checks afterward.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        sources: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "List of dataset IDs and/or local file paths to fuse.",
+                        },
+                        strategy: {
+                            type: "string",
+                            enum: ["concat", "join"],
+                            description: "Fusion strategy. concat appends rows; join merges on key(s).",
+                        },
+                        join_on: {
+                            oneOf: [
+                                { type: "string" },
+                                { type: "array", items: { type: "string" } }
+                            ],
+                            description: "Join key(s). Required when strategy='join'.",
+                        },
+                        how: {
+                            type: "string",
+                            enum: ["inner", "left", "outer"],
+                            description: "Join mode (only for strategy='join').",
+                        },
+                        dedup: {
+                            type: "boolean",
+                            description: "Drop exact duplicate rows after fusion.",
+                        },
+                        run_quality_after: {
+                            type: "boolean",
+                            description: "Run quality analysis on the fused output.",
+                        },
+                        leakage_check: {
+                            type: "boolean",
+                            description: "Run leakage/overlap checks across fused sources.",
+                        },
+                        output_format: {
+                            type: "string",
+                            enum: ["feather", "parquet", "csv", "jsonl", "arrow"],
+                            description: "Output format (default: feather).",
+                        },
+                        compression: {
+                            type: "string",
+                            enum: ["lz4", "zstd", "snappy", "gzip", "uncompressed"],
+                            description: "Compression algorithm for binary outputs.",
+                        },
+                        preview: {
+                            type: "boolean",
+                            description: "Generate a small preview CSV of fused output.",
+                        },
+                    },
+                    required: ["sources"],
+                },
+            },
             {
                 name: "analyze_image_quality",
                 description: "Analyze image quality (resolution, blur, corruption) for a folder or single image.",
@@ -423,10 +693,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             const query = String(request.params.arguments?.query);
             const limit = 5;
             const safeOnly = true; // Enable safe filter by default
+            const enableJIT = request.params.arguments?.enable_jit === true;
             if (!query) {
                 throw new McpError(ErrorCode.InvalidParams, "Query is required");
             }
-            const results = await searchEngine.search(query, { limit, safeOnly });
+            const results = await searchEngine.search(query, { limit, safeOnly, enableJIT });
             const formattedOutput = formatSearchResults(results);
             return {
                 content: [
@@ -437,6 +708,123 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 ],
             };
         }
+        case "discover_datasets": {
+            hydrateExternalKeys();
+            const query = String(request.params.arguments?.query || "").trim();
+            const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
+            const limit = Number(request.params.arguments?.limit || 10);
+            if (!query) {
+                throw new McpError(ErrorCode.InvalidParams, "query is required");
+            }
+            try {
+                let results = [];
+                if (source === "kaggle") {
+                    if (!dataIngestor.hasKaggleCredentials()) {
+                        return {
+                            content: [{ type: "text", text: `Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds).` }],
+                            isError: true,
+                        };
+                    }
+                    results = await kaggleSource.discover(query, limit);
+                }
+                else {
+                    const hf = new HuggingFaceScraper();
+                    results = await hf.scrape(Math.max(1, limit), true, query);
+                }
+                const formattedOutput = formatSearchResults(results.slice(0, limit));
+                return {
+                    content: [{ type: "text", text: formattedOutput }]
+                };
+            }
+            catch (error) {
+                return {
+                    content: [{ type: "text", text: `ERROR: discover failed: ${error.message}` }],
+                    isError: true,
+                };
+            }
+        }
+        case "download_dataset": {
+            hydrateExternalKeys();
+            const source = String(request.params.arguments?.source || "").toLowerCase();
+            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+            if (!source || !datasetId) {
+                throw new McpError(ErrorCode.InvalidParams, "source and dataset_id are required");
+            }
+            if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
+                return {
+                    content: [{ type: "text", text: `Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds).` }],
+                    isError: true,
+                };
+            }
+            try {
+                const localPath = await dataIngestor.ensureData(datasetId, source, () => undefined);
+                return {
+                    content: [{ type: "text", text: `✅ Download complete: ${localPath}` }]
+                };
+            }
+            catch (error) {
+                return {
+                    content: [{ type: "text", text: `ERROR: download failed: ${error.message}` }],
+                    isError: true,
+                };
+            }
+        }
+        case "configure_kaggle": {
+            const username = String(request.params.arguments?.username || "").trim();
+            const key = String(request.params.arguments?.key || "").trim();
+            if (!username || !key) {
+                throw new McpError(ErrorCode.InvalidParams, "username and key are required");
+            }
+            const r1 = secureKeys.set("kaggle_username", username);
+            const r2 = secureKeys.set("kaggle_key", key);
+            process.env.KAGGLE_USERNAME = username;
+            process.env.KAGGLE_KEY = key;
+            return {
+                content: [{ type: "text", text: `✅ Kaggle credentials saved securely (${r1.method || "store"}/${r2.method || "store"}). You can now use Kaggle datasets.` }]
+            };
+        }
+        case "configure_keys": {
+            const hfToken = String(request.params.arguments?.hf_token || "").trim();
+            const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
+            const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
+            const saved = [];
+            const methods = [];
+            if (hfToken) {
+                const r = secureKeys.set("hf_token", hfToken);
+                if (r.ok) {
+                    process.env.HF_TOKEN = hfToken;
+                    saved.push("HF token");
+                    if (r.method)
+                        methods.push(r.method);
+                }
+            }
+            if (kaggleUsername) {
+                const r = secureKeys.set("kaggle_username", kaggleUsername);
+                if (r.ok) {
+                    process.env.KAGGLE_USERNAME = kaggleUsername;
+                    saved.push("Kaggle username");
+                    if (r.method)
+                        methods.push(r.method);
+                }
+            }
+            if (kaggleKey) {
+                const r = secureKeys.set("kaggle_key", kaggleKey);
+                if (r.ok) {
+                    process.env.KAGGLE_KEY = kaggleKey;
+                    saved.push("Kaggle key");
+                    if (r.method)
+                        methods.push(r.method);
+                }
+            }
+            if (saved.length === 0) {
+                return {
+                    content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
+                };
+            }
+            return {
+                content: [{ type: "text", text: `✅ Key saved securely. Updated: ${saved.join(", ")}.` }]
+            };
+        }
         case "get_dataset_info": {
             const datasetId = String(request.params.arguments?.dataset_id);
             if (!datasetId) {
@@ -546,7 +934,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
         case "prepare_dataset": {
             const query = String(request.params.arguments?.query);
-            const job = jobManager.createJob("prepare", 0, { query });
+            const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
+            const job = jobManager.createJob("prepare", 0, { query, requirements });
             return {
                 content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you.` }]
             };
@@ -577,7 +966,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         case "export_dataset": {
             const datasetId = String(request.params.arguments?.dataset_id);
             const targetDir = request.params.arguments?.target_dir ? String(request.params.arguments?.target_dir) : undefined;
-            const requestedFormat = request.params.arguments?.format || "csv";
+            const requestedFormat = String(request.params.arguments?.format || "feather");
+            const fastMode = request.params.arguments?.fast === true;
+            const preview = request.params.arguments?.preview === true;
+            const sampleRows = request.params.arguments?.sample_rows ? Number(request.params.arguments.sample_rows) : undefined;
+            const columns = request.params.arguments?.columns;
+            const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
             const dataset = metadataStore.getDataset(datasetId);
             if (!dataset) {
                 throw new McpError(ErrorCode.InvalidParams, `Dataset not found: ${datasetId}`);
@@ -591,30 +985,153 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 };
             }
             let sourcePath = downloadStatus.local_path;
-            // Check if we need conversion
-            const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
-            if (currentExt !== requestedFormat) {
-                console.error(`[Export] Format mismatch (${currentExt} vs ${requestedFormat}). Converting...`);
-                try {
-                    const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, requestedFormat);
-                    sourcePath = pipelineResult.final_output_path;
-                }
-                catch (err) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: Failed to convert dataset to ${requestedFormat}: ${err.message}` }],
-                        isError: true
-                    };
+            // If NOT fast mode, run quality/cleaning pipeline first (only for csv/parquet compat)
+            if (!fastMode) {
+                const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
+                const pipelineFmt = (requestedFormat === "csv" || requestedFormat === "parquet") ? requestedFormat : "csv";
+                if (currentExt !== pipelineFmt) {
+                    console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
+                    try {
+                        const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
+                        if (pipelineResult.final_output_path) {
+                            sourcePath = pipelineResult.final_output_path;
+                        }
+                    }
+                    catch (err) {
+                        console.error(`[Export] Pipeline warning: ${err.message}. Continuing with raw file.`);
+                    }
                 }
             }
+            else {
+                console.error(`[Export] ⚡ Fast mode – skipping quality analysis and cleaning`);
+            }
+            // Build export options
+            const exportOpts = {};
+            if (compression)
+                exportOpts.compression = compression;
+            if (preview)
+                exportOpts.preview = true;
+            if (sampleRows)
+                exportOpts.sample_rows = sampleRows;
+            if (columns)
+                exportOpts.columns = columns;
             try {
-                const finalPath = await installService.install(datasetId, sourcePath, targetDir);
+                // Determine output file name
+                const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow", tfrecord: ".tfrecord" };
+                const ext = extMap[requestedFormat] || ".feather";
+                const safeName = datasetId.replace(/\//g, "_");
+                const outDir = targetDir || path.join(dataRoot, "exports");
+                if (!fs.existsSync(outDir))
+                    fs.mkdirSync(outDir, { recursive: true });
+                const outputFile = path.join(outDir, `${safeName}${ext}`);
+                const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
+                // Build rich response
+                let msg = `✅ **Export complete**\n`;
+                msg += `- **File**: ${result.output_path}\n`;
+                msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
+                msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
+                if (result.file_size_mb !== undefined)
+                    msg += `- **Size**: ${result.file_size_mb} MB\n`;
+                if (result.elapsed_seconds !== undefined)
+                    msg += `- **Time**: ${result.elapsed_seconds}s\n`;
+                if (result.preview_path)
+                    msg += `- **Preview**: ${result.preview_path}\n`;
+                msg += `\n`;
+                if (requestedFormat === "feather") {
+                    msg += `💡 **Inspect with:**\n`;
+                    msg += `  Python: \`pd.read_feather('${result.output_path}').head()\`\n`;
+                    msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                }
+                else if (requestedFormat === "parquet") {
+                    msg += `💡 **Inspect with:**\n`;
+                    msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
+                    msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                }
+                return { content: [{ type: "text", text: msg }] };
+            }
+            catch (error) {
                 return {
-                    content: [{ type: "text", text: `✅ Dataset ${datasetId} exported to: ${finalPath}` }]
+                    content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
+                    isError: true
                 };
             }
+        }
+        case "fuse_datasets": {
+            const rawSources = request.params.arguments?.sources;
+            if (!rawSources || !Array.isArray(rawSources) || rawSources.length < 2) {
+                throw new McpError(ErrorCode.InvalidParams, "sources must contain at least 2 dataset IDs/paths");
+            }
+            const strategy = request.params.arguments?.strategy || "concat";
+            const joinOn = request.params.arguments?.join_on;
+            const how = request.params.arguments?.how || "inner";
+            const dedup = request.params.arguments?.dedup !== false;
+            const runQualityAfter = request.params.arguments?.run_quality_after !== false;
+            const leakageCheck = request.params.arguments?.leakage_check !== false;
+            const outputFormat = request.params.arguments?.output_format || "feather";
+            const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
+            const preview = request.params.arguments?.preview !== false;
+            const resolvedPaths = [];
+            const unresolved = [];
+            for (const src of rawSources) {
+                if (fs.existsSync(src)) {
+                    resolvedPaths.push(src);
+                    continue;
+                }
+                const status = metadataStore.getDownloadStatus(src);
+                if (status?.local_path && fs.existsSync(status.local_path)) {
+                    resolvedPaths.push(status.local_path);
+                    continue;
+                }
+                unresolved.push(src);
+            }
+            if (unresolved.length > 0) {
+                return {
+                    content: [{
+                            type: "text",
+                            text: `ERROR: Could not resolve these sources to local files: ${unresolved.join(", ")}. Provide local paths or run prepare_dataset first.`
+                        }],
+                    isError: true
+                };
+            }
+            try {
+                const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow" };
+                const ext = extMap[outputFormat] || ".feather";
+                const outDir = path.join(dataRoot, "fusion");
+                if (!fs.existsSync(outDir))
+                    fs.mkdirSync(outDir, { recursive: true });
+                const outputPath = path.join(outDir, `fused_${Date.now()}${ext}`);
+                const result = await fusionEngine.fuse(resolvedPaths, outputPath, {
+                    strategy,
+                    join_on: joinOn,
+                    how,
+                    dedup,
+                    run_quality_after: runQualityAfter,
+                    leakage_check: leakageCheck,
+                    output_format: outputFormat,
+                    compression: compression,
+                    preview,
+                });
+                const nullDelta = result.stats.null_delta;
+                const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
+                let msg = `✅ Fused ${result.stats.sources_count} sources → ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
+                msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
+                msg += `- Null change: ${nullText}\n`;
+                msg += `- Output: ${result.output_path}\n`;
+                if (result.preview_path)
+                    msg += `- Preview: ${result.preview_path}\n`;
+                if (result.leakage_report) {
+                    msg += `- Leakage: ${result.leakage_report.leakage_detected ? "detected" : "none"}`;
+                    if (result.leakage_report.leakage_count) {
+                        msg += ` (${result.leakage_report.leakage_count})`;
+                    }
+                    msg += "\n";
+                }
+                msg += `\nNext: run split_dataset/export_dataset on fused output.`;
+                return { content: [{ type: "text", text: msg }] };
+            }
             catch (error) {
                 return {
-                    content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
+                    content: [{ type: "text", text: `ERROR: Fusion failed: ${error.message}` }],
                     isError: true
                 };
             }
@@ -761,8 +1278,29 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 });
 async function main() {
     const args = process.argv.slice(2);
+    hydrateExternalKeys();
+    const isFuse = args.includes("fuse");
+    const isDiscover = args.includes("discover");
+    const isDownload = args.includes("download");
+    const isConfig = args.includes("config");
     const isSetup = args.includes("--setup") || args.includes("setup");
     const isSilent = args.includes("--silent");
+    if (isFuse) {
+        await runFuseCli(args);
+        return;
+    }
+    if (isConfig) {
+        await runConfigCli(args);
+        return;
+    }
+    if (isDiscover) {
+        await runDiscoverCli(args);
+        return;
+    }
+    if (isDownload) {
+        await runDownloadCli(args);
+        return;
+    }
     // If run in setup mode OR in a terminal without args (human call), show setup wizard
     if (isSetup || (process.stdin.isTTY && args.length === 0)) {
         await runSetupWizard(isSilent);
@@ -778,6 +1316,256 @@ async function main() {
     console.error("Tip: To configure Vesper for your IDE, run: npx @vespermcp/mcp-server --setup");
     console.log("[Vesper] Main loop finished");
 }
+async function runConfigCli(args) {
+    const isKeys = args.includes("keys");
+    const isKaggle = args.includes("kaggle");
+    if (!(isKeys || isKaggle) || args.includes("--help")) {
+        console.log("Usage: vespermcp config keys");
+        console.log("       vespermcp config kaggle --username <name> --key <api_key>");
+        console.log("Core Vesper tools work with zero API keys.");
+        return;
+    }
+    const getArgValue = (name) => {
+        const idx = args.findIndex(a => a === name);
+        if (idx >= 0 && idx + 1 < args.length)
+            return args[idx + 1];
+        return undefined;
+    };
+    if (isKeys) {
+        console.log("\n🔐 Vesper Optional Keys Setup");
+        console.log("(Press Enter to skip any field)\n");
+        const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+        const ask = (q) => new Promise(resolve => rl.question(q, resolve));
+        const current = secureKeys.getAll();
+        const hfToken = (await ask(`Hugging Face token [${current.hf_token ? "saved" : "empty"}]: `)).trim();
+        const kaggleUsername = (await ask(`Kaggle username [${current.kaggle_username ? "saved" : "empty"}]: `)).trim();
+        const kaggleKey = (await ask(`Kaggle key [${current.kaggle_key ? "saved" : "empty"}]: `)).trim();
+        rl.close();
+        const saved = [];
+        if (hfToken) {
+            const res = secureKeys.set("hf_token", hfToken);
+            if (res.ok) {
+                process.env.HF_TOKEN = hfToken;
+                saved.push("HF token");
+            }
+        }
+        if (kaggleUsername) {
+            const res = secureKeys.set("kaggle_username", kaggleUsername);
+            if (res.ok) {
+                process.env.KAGGLE_USERNAME = kaggleUsername;
+                saved.push("Kaggle username");
+            }
+        }
+        if (kaggleKey) {
+            const res = secureKeys.set("kaggle_key", kaggleKey);
+            if (res.ok) {
+                process.env.KAGGLE_KEY = kaggleKey;
+                saved.push("Kaggle key");
+            }
+        }
+        if (saved.length === 0) {
+            console.log("No new keys saved (all skipped). Core tools continue to work without keys.");
+            return;
+        }
+        console.log(`✅ Key(s) saved securely: ${saved.join(", ")}`);
+        console.log("You can now use Kaggle and gated Hugging Face datasets.");
+        return;
+    }
+    // Backward-compatible Kaggle-specific path
+    let username = getArgValue("--username") || "";
+    let key = getArgValue("--key") || "";
+    if (!username || !key) {
+        const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+        const ask = (q) => new Promise(resolve => rl.question(q, resolve));
+        if (!username)
+            username = (await ask("Kaggle username: ")).trim();
+        if (!key)
+            key = (await ask("Kaggle key: ")).trim();
+        rl.close();
+    }
+    if (!username || !key) {
+        console.error("Missing Kaggle username/key. Aborting.");
+        process.exit(1);
+    }
+    secureKeys.set("kaggle_username", username);
+    secureKeys.set("kaggle_key", key);
+    process.env.KAGGLE_USERNAME = username;
+    process.env.KAGGLE_KEY = key;
+    console.log("✅ Key saved securely. You can now use Kaggle datasets.");
+}
+async function runDiscoverCli(args) {
+    const getArgValue = (name) => {
+        const idx = args.findIndex(a => a === name);
+        if (idx >= 0 && idx + 1 < args.length)
+            return args[idx + 1];
+        return undefined;
+    };
+    const source = (getArgValue("--source") || "huggingface").toLowerCase();
+    const limit = Number(getArgValue("--limit") || "10");
+    const queryParts = [];
+    for (let i = 0; i < args.length; i++) {
+        const token = args[i];
+        if (token === "discover")
+            continue;
+        if (token === "--source" || token === "--limit") {
+            i += 1;
+            continue;
+        }
+        if (token.startsWith("--"))
+            continue;
+        queryParts.push(token);
+    }
+    const query = queryParts.join(" ").trim();
+    if (!query) {
+        console.error("Usage: vespermcp discover --source kaggle \"credit risk\" --limit 10");
+        process.exit(1);
+    }
+    if (source === "kaggle") {
+        if (!dataIngestor.hasKaggleCredentials()) {
+            console.error("Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds).");
+            if (process.stdin.isTTY) {
+                const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+                const answer = await new Promise(resolve => rl.question("Configure Kaggle now? [y/N]: ", resolve));
+                rl.close();
+                if (answer.trim().toLowerCase() === "y") {
+                    await runConfigCli(["config", "kaggle"]);
+                }
+            }
+            if (!dataIngestor.hasKaggleCredentials())
+                process.exit(1);
+        }
+        try {
+            const results = await kaggleSource.discover(query, limit);
+            console.log(formatSearchResults(results));
+        }
+        catch (error) {
+            const msg = String(error?.message || error);
+            if (msg.toLowerCase().includes("kaggle package not installed")) {
+                console.error("Kaggle support is optional and needs the official client: pip install kaggle");
+            }
+            else {
+                console.error(`Kaggle discover failed: ${msg}`);
+            }
+            process.exit(1);
+        }
+        return;
+    }
+    const hf = new HuggingFaceScraper();
+    const results = await hf.scrape(limit, true, query);
+    console.log(formatSearchResults(results));
+}
+async function runDownloadCli(args) {
+    // Usage: vespermcp download kaggle user/dataset-name [--target-dir C:/path]
+    const targetIdx = args.findIndex(a => a === "--target-dir");
+    const targetDir = targetIdx >= 0 && targetIdx + 1 < args.length ? args[targetIdx + 1] : undefined;
+    const nonFlags = args.filter((a, i) => {
+        if (a.startsWith("--"))
+            return false;
+        if (targetIdx >= 0 && i === targetIdx + 1)
+            return false;
+        return true;
+    });
+    const source = (nonFlags[1] || "").toLowerCase();
+    const datasetId = nonFlags[2] || "";
+    if (!source || !datasetId) {
+        console.error("Usage: vespermcp download kaggle <username/dataset-name> [--target-dir C:/path]");
+        process.exit(1);
+    }
+    if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
+        console.error("Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds).");
+        if (process.stdin.isTTY) {
+            const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+            const answer = await new Promise(resolve => rl.question("Configure Kaggle now? [y/N]: ", resolve));
+            rl.close();
+            if (answer.trim().toLowerCase() === "y") {
+                await runConfigCli(["config", "kaggle"]);
+            }
+        }
+        if (!dataIngestor.hasKaggleCredentials())
+            process.exit(1);
+    }
+    let localPath = "";
+    try {
+        if (source === "kaggle" && targetDir) {
+            const normalized = datasetId.includes("kaggle.com/datasets/")
+                ? datasetId.split("kaggle.com/datasets/")[1].replace(/^\//, "")
+                : datasetId;
+            const dl = await kaggleSource.download(normalized, targetDir);
+            localPath = dl.local_path;
+            const size = fs.existsSync(localPath) ? fs.statSync(localPath).size : 0;
+            metadataStore.registerDownload(normalized, localPath, "completed", size);
+        }
+        else {
+            localPath = await dataIngestor.ensureData(datasetId, source, (msg) => console.log(msg));
+        }
+    }
+    catch (error) {
+        const msg = String(error?.message || error);
+        if (source === "kaggle" && msg.toLowerCase().includes("kaggle package not installed")) {
+            console.error("Kaggle support is optional and needs the official client: pip install kaggle");
+        }
+        else {
+            console.error(`Download failed: ${msg}`);
+        }
+        process.exit(1);
+    }
+    console.log(`✅ Download complete: ${localPath}`);
+}
+async function runFuseCli(args) {
+    const getArgValue = (name) => {
+        const idx = args.findIndex(a => a === name);
+        if (idx >= 0 && idx + 1 < args.length)
+            return args[idx + 1];
+        return undefined;
+    };
+    const collectListAfter = (name) => {
+        const idx = args.findIndex(a => a === name);
+        if (idx < 0)
+            return [];
+        const out = [];
+        for (let i = idx + 1; i < args.length; i++) {
+            if (args[i].startsWith("--"))
+                break;
+            out.push(args[i]);
+        }
+        return out;
+    };
+    const sources = collectListAfter("--sources");
+    if (sources.length < 2) {
+        console.error("Usage: vespermcp fuse --sources <file1> <file2> [more] --strategy concat|join [--on id] [--how inner|left|outer] [--dedup] [--quality] [--leakage] [--format feather|parquet|csv|jsonl|arrow]");
+        process.exit(1);
+    }
+    const strategy = getArgValue("--strategy") || "concat";
+    const onValue = getArgValue("--on");
+    const joinOn = onValue ? onValue.split(",").map(s => s.trim()).filter(Boolean) : undefined;
+    const how = getArgValue("--how") || "inner";
+    const outputFormat = getArgValue("--format") || "feather";
+    const compression = getArgValue("--compression");
+    const outputPath = getArgValue("--output") || path.join(process.cwd(), `fused_${Date.now()}.${outputFormat === "arrow" ? "arrow" : outputFormat}`);
+    const dedup = args.includes("--dedup");
+    const runQualityAfter = args.includes("--quality");
+    const leakageCheck = args.includes("--leakage");
+    const preview = !args.includes("--no-preview");
+    const result = await fusionEngine.fuse(sources, outputPath, {
+        strategy,
+        join_on: joinOn,
+        how,
+        dedup,
+        run_quality_after: runQualityAfter,
+        leakage_check: leakageCheck,
+        output_format: outputFormat,
+        compression,
+        preview,
+    });
+    const nullDelta = result.stats.null_delta;
+    const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
+    console.log(`Fused ${result.stats.sources_count} sources → ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).`);
+    console.log(`Null increase: ${nullText}`);
+    console.log(`Output: ${result.output_path}`);
+    if (result.preview_path)
+        console.log(`Preview saved: ${result.preview_path}`);
+    console.log("Next: run vespermcp split/export on the fused dataset");
+}
 async function runSetupWizard(silent = false) {
     const configManager = new ConfigManager();
     if (!silent) {