@vespermcp/mcp-server 1.3.0 → 1.4.0

package/build/index.js

@@ -468,6 +468,7 @@ import http from "http";
 import https from "https";
 import os from "os";
 import { enforcePlanGateForTool } from "./lib/plan-resolve.js";
+import { recordMcpToolAnalyticsAfterCall } from "./lib/mcp-analytics.js";
 // Determine absolute paths relative to the compiled script
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -2009,1474 +2010,1482 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             // Mark this step as complete
             markStepComplete(String(datasetId), String(step));
         }
-        switch (request.params.name) {
-            case "lineage":
-            case "get_lineage":
-            case "diff_lineage_versions": {
-                const operation = request.params.name === "get_lineage"
-                    ? "get"
-                    : request.params.name === "diff_lineage_versions"
-                        ? "diff"
-                        : String(request.params.arguments?.operation || "get").toLowerCase();
-                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                if (!datasetId) {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-                }
-                if (operation === "get") {
+        const toolResponse = await (async () => {
+            switch (request.params.name) {
+                case "lineage":
+                case "get_lineage":
+                case "diff_lineage_versions": {
+                    const operation = request.params.name === "get_lineage"
+                        ? "get"
+                        : request.params.name === "diff_lineage_versions"
+                            ? "diff"
+                            : String(request.params.arguments?.operation || "get").toLowerCase();
+                    const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                    if (!datasetId) {
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                    }
+                    if (operation === "get") {
+                        const base = toBaseDatasetId(datasetId);
+                        const record = readLineageRecord(base);
+                        if (!record.versions || record.versions.length === 0) {
+                            return {
+                                content: [{ type: "text", text: `No lineage found for '${datasetId}' yet.` }]
+                            };
+                        }
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(record, null, 2) }]
+                        };
+                    }
+                    if (operation !== "diff") {
+                        throw new McpError(ErrorCode.InvalidParams, "operation must be 'get' or 'diff'");
+                    }
+                    const fromVersion = Number(request.params.arguments?.from_version);
+                    const toVersion = Number(request.params.arguments?.to_version);
+                    if (!Number.isInteger(fromVersion) || fromVersion <= 0) {
+                        throw new McpError(ErrorCode.InvalidParams, "from_version must be a positive integer");
+                    }
+                    if (!Number.isInteger(toVersion) || toVersion <= 0) {
+                        throw new McpError(ErrorCode.InvalidParams, "to_version must be a positive integer");
+                    }
                     const base = toBaseDatasetId(datasetId);
                     const record = readLineageRecord(base);
-                    if (!record.versions || record.versions.length === 0) {
+                    const fromV = record.versions.find((v) => v.version === fromVersion);
+                    const toV = record.versions.find((v) => v.version === toVersion);
+                    if (!fromV || !toV) {
                         return {
-                            content: [{ type: "text", text: `No lineage found for '${datasetId}' yet.` }]
+                            content: [{ type: "text", text: `ERROR: Could not find both versions in lineage for '${datasetId}'.` }],
+                            isError: true,
                         };
                     }
+                    const fromSchema = (toV.output?.schema_before && toVersion > fromVersion)
+                        ? fromV.output?.schema_after || fromV.output?.schema_before || {}
+                        : fromV.output?.schema_after || fromV.output?.schema_before || {};
+                    const toSchema = toV.output?.schema_after || toV.output?.schema_before || {};
+                    const fromCols = Array.isArray(fromSchema.columns) ? fromSchema.columns.map((c) => String(c)) : [];
+                    const toCols = Array.isArray(toSchema.columns) ? toSchema.columns.map((c) => String(c)) : [];
+                    const fromDtypes = (fromSchema.dtypes && typeof fromSchema.dtypes === "object") ? fromSchema.dtypes : {};
+                    const toDtypes = (toSchema.dtypes && typeof toSchema.dtypes === "object") ? toSchema.dtypes : {};
+                    const schemaDiff = diffSchemaMaps(fromCols, toCols, fromDtypes, toDtypes);
+                    const fromRows = typeof fromSchema.rows === "number" ? fromSchema.rows : (typeof fromV.output?.rows === "number" ? fromV.output.rows : undefined);
+                    const toRows = typeof toSchema.rows === "number" ? toSchema.rows : (typeof toV.output?.rows === "number" ? toV.output.rows : undefined);
+                    const fromSteps = new Set((fromV.steps || []).map((s) => String(s.step)));
+                    const toSteps = new Set((toV.steps || []).map((s) => String(s.step)));
                     return {
-                        content: [{ type: "text", text: JSON.stringify(record, null, 2) }]
-                    };
-                }
-                if (operation !== "diff") {
-                    throw new McpError(ErrorCode.InvalidParams, "operation must be 'get' or 'diff'");
-                }
-                const fromVersion = Number(request.params.arguments?.from_version);
-                const toVersion = Number(request.params.arguments?.to_version);
-                if (!Number.isInteger(fromVersion) || fromVersion <= 0) {
-                    throw new McpError(ErrorCode.InvalidParams, "from_version must be a positive integer");
-                }
-                if (!Number.isInteger(toVersion) || toVersion <= 0) {
-                    throw new McpError(ErrorCode.InvalidParams, "to_version must be a positive integer");
-                }
-                const base = toBaseDatasetId(datasetId);
-                const record = readLineageRecord(base);
-                const fromV = record.versions.find((v) => v.version === fromVersion);
-                const toV = record.versions.find((v) => v.version === toVersion);
-                if (!fromV || !toV) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: Could not find both versions in lineage for '${datasetId}'.` }],
-                        isError: true,
-                    };
-                }
-                const fromSchema = (toV.output?.schema_before && toVersion > fromVersion)
-                    ? fromV.output?.schema_after || fromV.output?.schema_before || {}
-                    : fromV.output?.schema_after || fromV.output?.schema_before || {};
-                const toSchema = toV.output?.schema_after || toV.output?.schema_before || {};
-                const fromCols = Array.isArray(fromSchema.columns) ? fromSchema.columns.map((c) => String(c)) : [];
-                const toCols = Array.isArray(toSchema.columns) ? toSchema.columns.map((c) => String(c)) : [];
-                const fromDtypes = (fromSchema.dtypes && typeof fromSchema.dtypes === "object") ? fromSchema.dtypes : {};
-                const toDtypes = (toSchema.dtypes && typeof toSchema.dtypes === "object") ? toSchema.dtypes : {};
-                const schemaDiff = diffSchemaMaps(fromCols, toCols, fromDtypes, toDtypes);
-                const fromRows = typeof fromSchema.rows === "number" ? fromSchema.rows : (typeof fromV.output?.rows === "number" ? fromV.output.rows : undefined);
-                const toRows = typeof toSchema.rows === "number" ? toSchema.rows : (typeof toV.output?.rows === "number" ? toV.output.rows : undefined);
-                const fromSteps = new Set((fromV.steps || []).map((s) => String(s.step)));
-                const toSteps = new Set((toV.steps || []).map((s) => String(s.step)));
-                return {
-                    content: [{
-                            type: "text",
-                            text: JSON.stringify({
-                                dataset_id_base: base,
-                                from_version: fromVersion,
-                                to_version: toVersion,
-                                schema_diff: schemaDiff,
-                                row_count_delta: {
-                                    from: fromRows,
-                                    to: toRows,
-                                    delta: (typeof fromRows === "number" && typeof toRows === "number") ? (toRows - fromRows) : undefined,
-                                },
-                                steps_diff: {
-                                    added: Array.from(toSteps).filter((s) => !fromSteps.has(s)),
-                                    removed: Array.from(fromSteps).filter((s) => !toSteps.has(s)),
-                                    from_steps: Array.from(fromSteps),
-                                    to_steps: Array.from(toSteps),
-                                },
-                                actor_diff: {
-                                    changed: String(fromV.triggered_by?.agent_id || "") !== String(toV.triggered_by?.agent_id || "") ||
-                                        String(fromV.triggered_by?.pipeline_id || "") !== String(toV.triggered_by?.pipeline_id || ""),
-                                    from: {
-                                        tool: fromV.triggered_by?.tool,
-                                        agent_id: fromV.triggered_by?.agent_id,
-                                        pipeline_id: fromV.triggered_by?.pipeline_id,
+                        content: [{
+                                type: "text",
+                                text: JSON.stringify({
+                                    dataset_id_base: base,
+                                    from_version: fromVersion,
+                                    to_version: toVersion,
+                                    schema_diff: schemaDiff,
+                                    row_count_delta: {
+                                        from: fromRows,
+                                        to: toRows,
+                                        delta: (typeof fromRows === "number" && typeof toRows === "number") ? (toRows - fromRows) : undefined,
                                     },
-                                    to: {
-                                        tool: toV.triggered_by?.tool,
-                                        agent_id: toV.triggered_by?.agent_id,
-                                        pipeline_id: toV.triggered_by?.pipeline_id,
+                                    steps_diff: {
+                                        added: Array.from(toSteps).filter((s) => !fromSteps.has(s)),
+                                        removed: Array.from(fromSteps).filter((s) => !toSteps.has(s)),
+                                        from_steps: Array.from(fromSteps),
+                                        to_steps: Array.from(toSteps),
                                     },
-                                },
-                            }, null, 2),
-                        }],
-                };
-            }
-            case "vesper_web_find": {
-                hydrateExternalKeys();
-                const query = String(request.params.arguments?.query || "").trim();
-                const limit = Number(request.params.arguments?.limit || 10);
-                const sources = Array.isArray(request.params.arguments?.sources)
-                    ? (request.params.arguments?.sources).map(s => String(s).trim().toLowerCase()).filter(Boolean)
-                    : undefined;
-                try {
-                    const result = await webCoreEngine.find({
-                        query,
-                        sources: sources,
-                        limit,
-                        arxiv_full_text: request.params.arguments?.arxiv_full_text === true,
-                        github_include_readme: request.params.arguments?.github_include_readme === true,
-                    });
-                    try {
-                        appendLineageVersion({
-                            datasetIdBase: `webfind_${query || "query"}`,
-                            tool: "vesper_web_find",
-                            requestArgs: request.params.arguments,
-                            output: {
-                                rows: Array.isArray(result.results) ? result.results.length : undefined,
-                            },
-                            sources: Array.isArray(result.results)
-                                ? result.results.slice(0, 200).map((r) => ({
-                                    source: String(r?.source_type || "unknown"),
-                                    url: typeof r?.source_url === "string" ? r.source_url : undefined,
-                                    at: typeof r?.collected_at === "string" ? r.collected_at : undefined,
-                                }))
-                                : [],
-                            steps: [
-                                { step: "web_find_discover", at: new Date().toISOString(), params: { query, sources, limit } },
-                                { step: "web_find_complete", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
-                            ],
-                        });
-                    }
-                    catch (e) {
-                        console.error(`[Lineage] vesper_web_find append failed: ${e?.message || e}`);
-                    }
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
-                    };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: web_find failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "vesper.fuse": {
-                hydrateExternalKeys();
-                const sources = Array.isArray(request.params.arguments?.sources)
-                    ? request.params.arguments?.sources
-                    : undefined;
-                if (!sources || !Array.isArray(sources)) {
-                    return {
-                        content: [{ type: "text", text: "ERROR: vesper.fuse requires 'sources' array." }],
-                        isError: true,
+                                    actor_diff: {
+                                        changed: String(fromV.triggered_by?.agent_id || "") !== String(toV.triggered_by?.agent_id || "") ||
+                                            String(fromV.triggered_by?.pipeline_id || "") !== String(toV.triggered_by?.pipeline_id || ""),
+                                        from: {
+                                            tool: fromV.triggered_by?.tool,
+                                            agent_id: fromV.triggered_by?.agent_id,
+                                            pipeline_id: fromV.triggered_by?.pipeline_id,
+                                        },
+                                        to: {
+                                            tool: toV.triggered_by?.tool,
+                                            agent_id: toV.triggered_by?.agent_id,
+                                            pipeline_id: toV.triggered_by?.pipeline_id,
+                                        },
+                                    },
+                                }, null, 2),
+                            }],
                     };
                 }
-                try {
-                    const mergeStrategyRaw = request.params.arguments?.merge_strategy
-                        ? String(request.params.arguments?.merge_strategy).toLowerCase()
-                        : undefined;
-                    const dedupRaw = request.params.arguments?.deduplication
-                        ? String(request.params.arguments?.deduplication).toLowerCase()
-                        : undefined;
-                    const merge_strategy = mergeStrategyRaw && ["union", "dedup"].includes(mergeStrategyRaw)
-                        ? mergeStrategyRaw
-                        : undefined;
-                    const deduplication = dedupRaw && ["semantic", "exact", "none"].includes(dedupRaw)
-                        ? dedupRaw
+                case "vesper_web_find": {
+                    hydrateExternalKeys();
+                    const query = String(request.params.arguments?.query || "").trim();
+                    const limit = Number(request.params.arguments?.limit || 10);
+                    const sources = Array.isArray(request.params.arguments?.sources)
+                        ? (request.params.arguments?.sources).map(s => String(s).trim().toLowerCase()).filter(Boolean)
                         : undefined;
-                    const result = await webFusionEngine.fuse({
-                        sources: sources.map((s) => ({
-                            type: String(s?.type || "").trim().toLowerCase(),
-                            query: String(s?.query || "").trim(),
-                            max_results: s?.max_results !== undefined ? Number(s.max_results) : undefined,
-                            min_stars: s?.min_stars !== undefined ? Number(s.min_stars) : undefined,
-                            bucket: s?.bucket !== undefined ? String(s.bucket) : undefined,
-                            path: s?.path !== undefined ? String(s.path) : undefined,
-                            region: s?.region !== undefined ? String(s.region) : undefined,
-                            credentials: s?.credentials ? {
-                                accessKeyId: s.credentials.accessKeyId !== undefined ? String(s.credentials.accessKeyId) : undefined,
-                                secretAccessKey: s.credentials.secretAccessKey !== undefined ? String(s.credentials.secretAccessKey) : undefined,
-                                sessionToken: s.credentials.sessionToken !== undefined ? String(s.credentials.sessionToken) : undefined,
-                                roleArn: s.credentials.roleArn !== undefined ? String(s.credentials.roleArn) : undefined,
-                            } : undefined,
-                        })),
-                        merge_strategy,
-                        deduplication,
-                    });
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
-                    };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: vesper.fuse failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "vesper.extract_web": {
-                hydrateExternalKeys();
-                const url = String(request.params.arguments?.url || "").trim();
-                const mode = request.params.arguments?.mode
-                    ? String(request.params.arguments?.mode).trim().toLowerCase()
-                    : "auto";
-                const schema = request.params.arguments?.schema && typeof request.params.arguments.schema === "object"
-                    ? request.params.arguments.schema
-                    : undefined;
-                if (!url) {
-                    return {
-                        content: [{ type: "text", text: "ERROR: vesper.extract_web requires 'url'." }],
-                        isError: true,
-                    };
-                }
-                try {
-                    const out = await webExtractorEngine.extract({
-                        url,
-                        mode: mode,
-                        strict_schema: request.params.arguments?.strict_schema !== false,
-                        schema: schema,
-                    });
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(out, null, 2) }],
-                    };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: vesper.extract_web failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "unified_dataset_api": {
-                hydrateExternalKeys();
-                const operation = String(request.params.arguments?.operation || "").trim().toLowerCase();
-                const source = String(request.params.arguments?.source || "auto").trim().toLowerCase();
-                const includeUnavailable = request.params.arguments?.include_unavailable === true;
-                const publicOnly = request.params.arguments?.public_only !== false;
-                try {
-                    if (operation === "providers") {
-                        return {
-                            content: [{ type: "text", text: JSON.stringify({ providers: unifiedDatasetGateway.getProviderStatuses(includeUnavailable) }, null, 2) }],
-                        };
-                    }
-                    if (operation === "discover") {
-                        const query = String(request.params.arguments?.query || "").trim();
-                        if (!query) {
-                            throw new McpError(ErrorCode.InvalidParams, "query is required for operation='discover'");
-                        }
-                        const result = await unifiedDatasetGateway.discover({
+                    try {
+                        const result = await webCoreEngine.find({
                             query,
-                            source,
-                            limit: Number(request.params.arguments?.limit || 10),
-                            publicOnly,
+                            sources: sources,
+                            limit,
+                            arxiv_full_text: request.params.arguments?.arxiv_full_text === true,
+                            github_include_readme: request.params.arguments?.github_include_readme === true,
                         });
                         try {
                             appendLineageVersion({
-                                datasetIdBase: `discover_${source}_${query || "query"}`,
-                                tool: "unified_dataset_api.discover",
+                                datasetIdBase: `webfind_${query || "query"}`,
+                                tool: "vesper_web_find",
                                 requestArgs: request.params.arguments,
-                                output: { rows: Array.isArray(result.results) ? result.results.length : undefined },
+                                output: {
+                                    rows: Array.isArray(result.results) ? result.results.length : undefined,
+                                },
                                 sources: Array.isArray(result.results)
                                     ? result.results.slice(0, 200).map((r) => ({
-                                        source: String(r?.source || source || "unknown"),
-                                        url: typeof r?.download_url === "string"
-                                            ? r.download_url
-                                            : (typeof r?.metadata_url === "string" ? r.metadata_url : undefined),
-                                        at: new Date().toISOString(),
+                                        source: String(r?.source_type || "unknown"),
+                                        url: typeof r?.source_url === "string" ? r.source_url : undefined,
+                                        at: typeof r?.collected_at === "string" ? r.collected_at : undefined,
                                     }))
                                     : [],
                                 steps: [
-                                    { step: "discover_requested", at: new Date().toISOString(), params: { query, source, limit: Number(request.params.arguments?.limit || 10), publicOnly } },
-                                    { step: "discover_completed", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
+                                    { step: "web_find_discover", at: new Date().toISOString(), params: { query, sources, limit } },
+                                    { step: "web_find_complete", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
                                 ],
                             });
                         }
                         catch (e) {
-                            console.error(`[Lineage] unified discover append failed: ${e?.message || e}`);
+                            console.error(`[Lineage] vesper_web_find append failed: ${e?.message || e}`);
                         }
                         return {
                             content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
                         };
                     }
-                    if (operation === "download") {
-                        const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                        if (!datasetId) {
-                            throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='download'");
-                        }
-                        const requestedTargetDir = request.params.arguments?.target_dir
-                            ? String(request.params.arguments.target_dir).trim()
-                            : request.params.arguments?.output_dir
-                                ? String(request.params.arguments.output_dir).trim()
-                                : "";
-                        const targetDir = requestedTargetDir || process.cwd();
-                        try {
-                            await ensurePythonModules([{ module: "datasets", packageName: "datasets" }]);
-                        }
-                        catch {
-                            // best effort; non-HF providers do not require this
-                        }
-                        const result = await unifiedDatasetGateway.download({
-                            datasetId,
-                            source,
-                            targetDir,
-                        });
-                        try {
-                            upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
-                        }
-                        catch (e) {
-                            console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
-                        }
-                        try {
-                            const schemaAfter = await getSchemaSnapshot(result.copied_to || result.local_path);
-                            const lineage = appendLineageVersion({
-                                datasetIdBase: result.dataset_id,
-                                tool: "unified_dataset_api.download",
-                                requestArgs: request.params.arguments,
-                                outputPath: result.copied_to || result.local_path,
-                                output: {
-                                    local_path: result.copied_to || result.local_path,
-                                    format: path.extname(result.copied_to || result.local_path).replace(".", ""),
-                                    schema_after: schemaAfter,
-                                },
-                                sources: [{
-                                        source: source,
-                                        url: typeof result.dataset_id === "string" ? result.dataset_id : undefined,
-                                        at: new Date().toISOString(),
-                                    }],
-                                steps: [
-                                    { step: "download_requested", at: new Date().toISOString(), params: { datasetId, source, targetDir } },
-                                    { step: "download_completed", at: new Date().toISOString(), metrics: { local_path: result.copied_to || result.local_path } },
-                                ],
-                            });
-                            try {
-                                upsertRegistry(lineage.datasetVersionId, result.copied_to || result.local_path, "completed");
-                            }
-                            catch { }
-                        }
-                        catch (e) {
-                            console.error(`[Lineage] unified download append failed: ${e?.message || e}`);
-                        }
+                    catch (error) {
                         return {
-                            content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                            content: [{ type: "text", text: `ERROR: web_find failed: ${error.message}` }],
+                            isError: true,
                         };
                     }
-                    if (operation === "info") {
-                        const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                        if (!datasetId) {
-                            throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='info'");
-                        }
-                        const result = await unifiedDatasetGateway.info({
-                            datasetId,
-                            source,
-                            publicOnly,
+                }
+                case "vesper.fuse": {
+                    hydrateExternalKeys();
+                    const sources = Array.isArray(request.params.arguments?.sources)
+                        ? request.params.arguments?.sources
+                        : undefined;
+                    if (!sources || !Array.isArray(sources)) {
+                        return {
+                            content: [{ type: "text", text: "ERROR: vesper.fuse requires 'sources' array." }],
+                            isError: true,
+                        };
+                    }
+                    try {
+                        const mergeStrategyRaw = request.params.arguments?.merge_strategy
+                            ? String(request.params.arguments?.merge_strategy).toLowerCase()
+                            : undefined;
+                        const dedupRaw = request.params.arguments?.deduplication
+                            ? String(request.params.arguments?.deduplication).toLowerCase()
+                            : undefined;
+                        const merge_strategy = mergeStrategyRaw && ["union", "dedup"].includes(mergeStrategyRaw)
+                            ? mergeStrategyRaw
+                            : undefined;
+                        const deduplication = dedupRaw && ["semantic", "exact", "none"].includes(dedupRaw)
+                            ? dedupRaw
+                            : undefined;
+                        const result = await webFusionEngine.fuse({
+                            sources: sources.map((s) => ({
+                                type: String(s?.type || "").trim().toLowerCase(),
+                                query: String(s?.query || "").trim(),
+                                max_results: s?.max_results !== undefined ? Number(s.max_results) : undefined,
+                                min_stars: s?.min_stars !== undefined ? Number(s.min_stars) : undefined,
+                                bucket: s?.bucket !== undefined ? String(s.bucket) : undefined,
+                                path: s?.path !== undefined ? String(s.path) : undefined,
+                                region: s?.region !== undefined ? String(s.region) : undefined,
+                                credentials: s?.credentials ? {
+                                    accessKeyId: s.credentials.accessKeyId !== undefined ? String(s.credentials.accessKeyId) : undefined,
+                                    secretAccessKey: s.credentials.secretAccessKey !== undefined ? String(s.credentials.secretAccessKey) : undefined,
+                                    sessionToken: s.credentials.sessionToken !== undefined ? String(s.credentials.sessionToken) : undefined,
+                                    roleArn: s.credentials.roleArn !== undefined ? String(s.credentials.roleArn) : undefined,
+                                } : undefined,
+                            })),
+                            merge_strategy,
+                            deduplication,
                         });
                         return {
                             content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
                         };
                     }
-                    throw new McpError(ErrorCode.InvalidParams, `Unsupported operation: ${operation}`);
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: unified gateway failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "vesper_search": {
-                const query = String(request.params.arguments?.query);
-                const limit = 5;
-                const safeOnly = true; // Enable safe filter by default
-                const enableJIT = request.params.arguments?.enable_jit === true;
-                if (!query) {
-                    throw new McpError(ErrorCode.InvalidParams, "Query is required");
-                }
-                const results = await searchEngine.search(query, { limit, safeOnly, enableJIT });
-                const formattedOutput = formatSearchResults(results);
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: formattedOutput,
-                        },
-                    ],
-                };
-            }
-            case "discover_datasets": {
-                hydrateExternalKeys();
-                const query = String(request.params.arguments?.query || "").trim();
-                const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
-                const limit = Number(request.params.arguments?.limit || 10);
-                if (!query) {
-                    throw new McpError(ErrorCode.InvalidParams, "query is required");
-                }
-                try {
-                    const gatewayResult = await unifiedDatasetGateway.discover({
-                        query,
-                        source,
-                        limit,
-                        publicOnly: false,
-                    });
-                    const results = gatewayResult.results;
-                    const recipeScript = path.join(dataRoot, "python", "asset_downloader_engine.py");
-                    for (const ds of results.slice(0, limit)) {
-                        const info = {
-                            dataset_id: ds.id,
-                            id: ds.id,
-                            source: ds.source,
-                            repo_id: ds.id,
-                            total_images: ds.total_examples || 0,
-                            image_column: undefined,
-                            recipes_dir: path.join(dataRoot, "recipes"),
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: vesper.fuse failed: ${error.message}` }],
+                            isError: true,
                         };
-                        try {
-                            await runPythonJson(recipeScript, ["build_recipe", JSON.stringify(info)]);
-                        }
-                        catch {
-                            // best-effort recipe generation; ignore discovery-time recipe failures
-                        }
                     }
-                    const formattedOutput = formatSearchResults(results.slice(0, limit));
-                    const noteBlock = gatewayResult.notes.length > 0
-                        ? `\nGateway notes:\n- ${gatewayResult.notes.join("\n- ")}`
-                        : "";
-                    return {
-                        content: [{ type: "text", text: `${formattedOutput}${noteBlock}` }]
-                    };
                 }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: discover failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "download_dataset": {
-                hydrateExternalKeys();
-                const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
-                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                const requestedTargetDir = request.params.arguments?.target_dir
-                    ? String(request.params.arguments.target_dir).trim()
-                    : request.params.arguments?.output_dir
-                        ? String(request.params.arguments.output_dir).trim()
-                        : "";
-                const targetDir = requestedTargetDir || process.cwd();
-                if (!datasetId) {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-                }
-                // Pre-install Python datasets library for HuggingFace fallback
-                if (source === "huggingface") {
-                    try {
-                        await ensurePythonModules([
-                            { module: "datasets", packageName: "datasets" },
-                        ]);
-                    }
-                    catch {
-                        // Continue - direct download may still work
+                case "vesper.extract_web": {
+                    hydrateExternalKeys();
+                    const url = String(request.params.arguments?.url || "").trim();
+                    const mode = request.params.arguments?.mode
+                        ? String(request.params.arguments?.mode).trim().toLowerCase()
+                        : "auto";
+                    const schema = request.params.arguments?.schema && typeof request.params.arguments.schema === "object"
+                        ? request.params.arguments.schema
+                        : undefined;
+                    if (!url) {
+                        return {
+                            content: [{ type: "text", text: "ERROR: vesper.extract_web requires 'url'." }],
+                            isError: true,
+                        };
                     }
-                }
-                try {
-                    const result = await unifiedDatasetGateway.download({
-                        datasetId,
-                        source,
-                        targetDir,
-                    });
                     try {
-                        upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
+                        const out = await webExtractorEngine.extract({
+                            url,
+                            mode: mode,
+                            strict_schema: request.params.arguments?.strict_schema !== false,
+                            schema: schema,
+                        });
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(out, null, 2) }],
+                        };
                     }
-                    catch (e) {
-                        console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: vesper.extract_web failed: ${error.message}` }],
+                            isError: true,
+                        };
                     }
-                    const noteBlock = result.notes.length > 0 ? `\nNotes:\n- ${result.notes.join("\n- ")}` : "";
-                    return {
-                        content: [{ type: "text", text: `Download complete: ${result.copied_to || result.local_path}${noteBlock}` }]
-                    };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: download failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-            }
-            case "vesper_download_assets": {
-                hydrateExternalKeys();
-                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                const source = String(request.params.arguments?.source || "").trim().toLowerCase();
-                // Auto-infer repo_id from dataset_id if not provided (common for HuggingFace)
-                const repoId = request.params.arguments?.repo_id
-                    ? String(request.params.arguments.repo_id)
-                    : (source === "huggingface" && datasetId.includes("/") ? datasetId : undefined);
-                const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
-                const urls = Array.isArray(request.params.arguments?.urls)
-                    ? (request.params.arguments?.urls).map(v => String(v))
-                    : undefined;
-                const outputFormat = String(request.params.arguments?.output_format || "webdataset");
-                const requestedOutputDir = request.params.arguments?.target_dir
-                    ? String(request.params.arguments.target_dir).trim()
-                    : request.params.arguments?.output_dir
-                        ? String(request.params.arguments.output_dir).trim()
-                        : undefined;
-                const maxItems = request.params.arguments?.max_items ? Number(request.params.arguments.max_items) : undefined;
-                const workers = request.params.arguments?.workers ? Number(request.params.arguments.workers) : 8;
-                const imageColumn = request.params.arguments?.image_column ? String(request.params.arguments.image_column) : undefined;
-                if (!datasetId || !source) {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id and source are required");
-                }
-                if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
-                    return {
-                        content: [{ type: "text", text: "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds)." }],
-                        isError: true,
-                    };
-                }
-                const requiredModules = [
-                    { module: "aiohttp", packageName: "aiohttp" },
-                ];
-                if (source === "url") {
-                    requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
-                }
-                if (source === "huggingface") {
-                    requiredModules.push({ module: "datasets", packageName: "datasets" });
-                    requiredModules.push({ module: "PIL", packageName: "Pillow" });
-                }
-                if (source === "kaggle") {
-                    requiredModules.push({ module: "kaggle", packageName: "kaggle" });
-                }
-                try {
-                    await ensurePythonModules(requiredModules);
                 }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
-                        isError: true,
-                    };
-                }
-                const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
-                const payload = {
-                    dataset_id: datasetId,
-                    source,
-                    repo_id: repoId,
-                    kaggle_ref: kaggleRef,
-                    urls,
-                    output_format: outputFormat,
-                    output_dir: requestedOutputDir,
-                    max_items: maxItems,
-                    workers,
-                    image_column: imageColumn,
-                    output_root: requestedOutputDir || process.cwd(),
-                    recipes_dir: path.join(dataRoot, "recipes"),
-                };
-                try {
-                    const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
-                    if (!result?.ok) {
-                        const errMsg = result?.error || "Unknown error";
-                        // Enhance error messages for common failures
-                        let hint = "";
-                        if (errMsg.includes("No image column")) {
-                            hint = "\n\nHint: Specify image_column parameter with the column name containing images/URLs.";
+                case "unified_dataset_api": {
+                    hydrateExternalKeys();
+                    const operation = String(request.params.arguments?.operation || "").trim().toLowerCase();
+                    const source = String(request.params.arguments?.source || "auto").trim().toLowerCase();
+                    const includeUnavailable = request.params.arguments?.include_unavailable === true;
+                    const publicOnly = request.params.arguments?.public_only !== false;
+                    try {
+                        if (operation === "providers") {
+                            return {
+                                content: [{ type: "text", text: JSON.stringify({ providers: unifiedDatasetGateway.getProviderStatuses(includeUnavailable) }, null, 2) }],
+                            };
                         }
-                        else if (errMsg.includes("Authentication") || errMsg.includes("401") || errMsg.includes("403")) {
-                            hint = "\n\nHint: Use configure_keys tool to set HF_TOKEN for gated/private datasets.";
+                        if (operation === "discover") {
+                            const query = String(request.params.arguments?.query || "").trim();
+                            if (!query) {
+                                throw new McpError(ErrorCode.InvalidParams, "query is required for operation='discover'");
+                            }
+                            const result = await unifiedDatasetGateway.discover({
+                                query,
+                                source,
+                                limit: Number(request.params.arguments?.limit || 10),
+                                publicOnly,
+                            });
+                            try {
+                                appendLineageVersion({
+                                    datasetIdBase: `discover_${source}_${query || "query"}`,
+                                    tool: "unified_dataset_api.discover",
+                                    requestArgs: request.params.arguments,
+                                    output: { rows: Array.isArray(result.results) ? result.results.length : undefined },
+                                    sources: Array.isArray(result.results)
+                                        ? result.results.slice(0, 200).map((r) => ({
+                                            source: String(r?.source || source || "unknown"),
+                                            url: typeof r?.download_url === "string"
+                                                ? r.download_url
+                                                : (typeof r?.metadata_url === "string" ? r.metadata_url : undefined),
+                                            at: new Date().toISOString(),
+                                        }))
+                                        : [],
+                                    steps: [
+                                        { step: "discover_requested", at: new Date().toISOString(), params: { query, source, limit: Number(request.params.arguments?.limit || 10), publicOnly } },
+                                        { step: "discover_completed", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
+                                    ],
+                                });
+                            }
+                            catch (e) {
+                                console.error(`[Lineage] unified discover append failed: ${e?.message || e}`);
+                            }
+                            return {
+                                content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                            };
                         }
+                        if (operation === "download") {
+                            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                            if (!datasetId) {
+                                throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='download'");
+                            }
+                            const requestedTargetDir = request.params.arguments?.target_dir
+                                ? String(request.params.arguments.target_dir).trim()
+                                : request.params.arguments?.output_dir
+                                    ? String(request.params.arguments.output_dir).trim()
+                                    : "";
+                            const targetDir = requestedTargetDir || process.cwd();
+                            try {
+                                await ensurePythonModules([{ module: "datasets", packageName: "datasets" }]);
+                            }
+                            catch {
+                                // best effort; non-HF providers do not require this
+                            }
+                            const result = await unifiedDatasetGateway.download({
+                                datasetId,
+                                source,
+                                targetDir,
+                            });
+                            try {
+                                upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
+                            }
+                            catch (e) {
+                                console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
+                            }
+                            try {
+                                const schemaAfter = await getSchemaSnapshot(result.copied_to || result.local_path);
+                                const lineage = appendLineageVersion({
+                                    datasetIdBase: result.dataset_id,
+                                    tool: "unified_dataset_api.download",
+                                    requestArgs: request.params.arguments,
+                                    outputPath: result.copied_to || result.local_path,
+                                    output: {
+                                        local_path: result.copied_to || result.local_path,
+                                        format: path.extname(result.copied_to || result.local_path).replace(".", ""),
+                                        schema_after: schemaAfter,
+                                    },
+                                    sources: [{
+                                            source: source,
+                                            url: typeof result.dataset_id === "string" ? result.dataset_id : undefined,
+                                            at: new Date().toISOString(),
+                                        }],
+                                    steps: [
+                                        { step: "download_requested", at: new Date().toISOString(), params: { datasetId, source, targetDir } },
+                                        { step: "download_completed", at: new Date().toISOString(), metrics: { local_path: result.copied_to || result.local_path } },
+                                    ],
+                                });
+                                try {
+                                    upsertRegistry(lineage.datasetVersionId, result.copied_to || result.local_path, "completed");
+                                }
+                                catch { }
+                            }
+                            catch (e) {
+                                console.error(`[Lineage] unified download append failed: ${e?.message || e}`);
+                            }
+                            return {
+                                content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                            };
+                        }
+                        if (operation === "info") {
+                            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                            if (!datasetId) {
+                                throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='info'");
+                            }
+                            const result = await unifiedDatasetGateway.info({
+                                datasetId,
+                                source,
+                                publicOnly,
+                            });
+                            return {
+                                content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                            };
+                        }
+                        throw new McpError(ErrorCode.InvalidParams, `Unsupported operation: ${operation}`);
+                    }
+                    catch (error) {
                         return {
-                            content: [{ type: "text", text: `ERROR: asset download failed: ${errMsg}${hint}` }],
+                            content: [{ type: "text", text: `ERROR: unified gateway failed: ${error.message}` }],
                             isError: true,
                         };
                     }
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(result.result, null, 2) }],
-                    };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: asset downloader execution failed: ${error.message}` }],
-                        isError: true,
-                    };
                 }
-            }
-            case "configure_keys": {
-                const hfToken = String(request.params.arguments?.hf_token || "").trim();
-                const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
-                const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
-                const dataworldToken = String(request.params.arguments?.dataworld_token || "").trim();
-                const saved = [];
-                const methods = [];
-                if (hfToken) {
-                    const r = secureKeys.set("hf_token", hfToken);
-                    if (r.ok) {
-                        process.env.HF_TOKEN = hfToken;
-                        saved.push("HF token");
-                        if (r.method)
-                            methods.push(r.method);
+                case "vesper_search": {
+                    const query = String(request.params.arguments?.query);
+                    const limit = 5;
+                    const safeOnly = true; // Enable safe filter by default
+                    const enableJIT = request.params.arguments?.enable_jit === true;
+                    if (!query) {
+                        throw new McpError(ErrorCode.InvalidParams, "Query is required");
                     }
-                }
-                if (kaggleUsername) {
-                    const r = secureKeys.set("kaggle_username", kaggleUsername);
-                    if (r.ok) {
-                        process.env.KAGGLE_USERNAME = kaggleUsername;
-                        saved.push("Kaggle username");
-                        if (r.method)
-                            methods.push(r.method);
-                    }
-                }
-                if (kaggleKey) {
-                    const r = secureKeys.set("kaggle_key", kaggleKey);
-                    if (r.ok) {
-                        process.env.KAGGLE_KEY = kaggleKey;
-                        saved.push("Kaggle key");
-                        if (r.method)
-                            methods.push(r.method);
-                    }
-                }
-                if (dataworldToken) {
-                    const r = secureKeys.set("dataworld_token", dataworldToken);
-                    if (r.ok) {
-                        process.env.DW_AUTH_TOKEN = dataworldToken;
-                        saved.push("data.world token");
-                        if (r.method)
-                            methods.push(r.method);
-                    }
-                }
-                if (saved.length === 0) {
+                    const results = await searchEngine.search(query, { limit, safeOnly, enableJIT });
+                    const formattedOutput = formatSearchResults(results);
                     return {
-                        content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
+                        content: [
+                            {
+                                type: "text",
+                                text: formattedOutput,
+                            },
+                        ],
                     };
                 }
-                return {
-                    content: [{ type: "text", text: `Key saved securely. Updated: ${saved.join(", ")}.` }]
-                };
-            }
-            case "get_dataset_info": {
-                const datasetId = String(request.params.arguments?.dataset_id);
-                if (!datasetId) {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-                }
-                const dataset = metadataStore.getDataset(datasetId);
-                if (!dataset) {
-                    // Fallback: check the registry for local path info
-                    const regEntry = getRegistryEntry(datasetId);
-                    const regPath = regEntry?.local_path || regEntry?.path;
-                    if (regEntry) {
-                        const exists = regPath && fs.existsSync(regPath);
-                        return {
-                            content: [{ type: "text", text: `**${datasetId}** (from registry)\n- Local path: ${regPath || "unknown"}\n- Status: ${regEntry.status || "unknown"}${exists ? "" : " (file missing)"}\n\nNote: Full metadata not available in metadata store. Use prepare_dataset to get full details.` }],
-                        };
+                case "discover_datasets": {
+                    hydrateExternalKeys();
+                    const query = String(request.params.arguments?.query || "").trim();
+                    const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
+                    const limit = Number(request.params.arguments?.limit || 10);
+                    if (!query) {
+                        throw new McpError(ErrorCode.InvalidParams, "query is required");
                     }
-                    return {
-                        content: [{ type: "text", text: `ERROR: Dataset not found: ${datasetId}. Use vesper_list_datasets to see available datasets, or prepare_dataset to add new ones.` }],
-                        isError: true,
-                    };
-                }
-                // Enrich: if total_examples is 0-ish, try the HF datasets-server /size API
-                if ((!dataset.total_examples || dataset.total_examples === 0) && dataset.source === "huggingface") {
                     try {
-                        const sizeResp = await fetch(`https://datasets-server.huggingface.co/size?dataset=${encodeURIComponent(dataset.id)}`);
-                        if (sizeResp.ok) {
-                            const sizeData = await sizeResp.json();
-                            const numRows = sizeData?.size?.dataset?.num_rows;
-                            if (numRows && numRows > 0) {
-                                dataset.total_examples = numRows;
-                                // Also backfill splits
-                                if (sizeData?.size?.splits && Array.isArray(sizeData.size.splits)) {
-                                    dataset.splits = sizeData.size.splits.map((s) => ({
-                                        name: s.split,
-                                        num_examples: s.num_rows || 0,
-                                        size_bytes: s.num_bytes_parquet_files || 0,
-                                    }));
-                                    dataset.has_train_split = dataset.splits.some((s) => s.name === "train");
-                                    dataset.has_test_split = dataset.splits.some((s) => s.name === "test");
-                                    dataset.has_validation_split = dataset.splits.some((s) => s.name === "validation" || s.name === "val");
-                                }
-                                // Persist enriched metadata
-                                metadataStore.saveDataset(dataset);
+                        const gatewayResult = await unifiedDatasetGateway.discover({
+                            query,
+                            source,
+                            limit,
+                            publicOnly: false,
+                        });
+                        const results = gatewayResult.results;
+                        const recipeScript = path.join(dataRoot, "python", "asset_downloader_engine.py");
+                        for (const ds of results.slice(0, limit)) {
+                            const info = {
+                                dataset_id: ds.id,
+                                id: ds.id,
+                                source: ds.source,
+                                repo_id: ds.id,
+                                total_images: ds.total_examples || 0,
+                                image_column: undefined,
+                                recipes_dir: path.join(dataRoot, "recipes"),
+                            };
+                            try {
+                                await runPythonJson(recipeScript, ["build_recipe", JSON.stringify(info)]);
+                            }
+                            catch {
+                                // best-effort recipe generation; ignore discovery-time recipe failures
                             }
                         }
+                        const formattedOutput = formatSearchResults(results.slice(0, limit));
+                        const noteBlock = gatewayResult.notes.length > 0
+                            ? `\nGateway notes:\n- ${gatewayResult.notes.join("\n- ")}`
+                            : "";
+                        return {
+                            content: [{ type: "text", text: `${formattedOutput}${noteBlock}` }]
+                        };
                     }
-                    catch {
-                        // Enrichment is best-effort; continue with whatever we have
-                    }
-                }
-                const formattedOutput = formatDatasetInfo(dataset);
-                return { content: [{ type: "text", text: formattedOutput }] };
-            }
-            case "quality_analyze":
-            case "analyze_quality":
-            case "analyze_image_quality":
-            case "analyze_media_quality":
-            case "generate_quality_report": {
-                const resolvedOperation = request.params.name === "analyze_image_quality"
-                    ? "image"
-                    : request.params.name === "analyze_media_quality"
-                        ? "media"
-                        : request.params.name === "generate_quality_report"
-                            ? "report"
-                            : String(request.params.arguments?.operation || "dataset").toLowerCase();
-                if (resolvedOperation === "image") {
-                    const inputPath = String(request.params.arguments?.path || "").trim();
-                    if (!inputPath || !fs.existsSync(inputPath)) {
-                        throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
-                    }
-                    const report = await imageAnalyzer.analyze(inputPath);
-                    return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
-                }
-                if (resolvedOperation === "media") {
-                    const inputPath = String(request.params.arguments?.path || "").trim();
-                    if (!inputPath || !fs.existsSync(inputPath)) {
-                        throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: discover failed: ${error.message}` }],
+                            isError: true,
+                        };
                     }
-                    const report = await mediaAnalyzer.analyze(inputPath);
-                    return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
                 }
-                if (resolvedOperation === "report") {
+                case "download_dataset": {
+                    hydrateExternalKeys();
+                    const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
                     const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                    const datasetPath = String(request.params.arguments?.dataset_path || "").trim();
+                    const requestedTargetDir = request.params.arguments?.target_dir
+                        ? String(request.params.arguments.target_dir).trim()
+                        : request.params.arguments?.output_dir
+                            ? String(request.params.arguments.output_dir).trim()
+                            : "";
+                    const targetDir = requestedTargetDir || process.cwd();
                     if (!datasetId) {
-                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='report'");
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                    }
+                    // Pre-install Python datasets library for HuggingFace fallback
+                    if (source === "huggingface") {
+                        try {
+                            await ensurePythonModules([
+                                { module: "datasets", packageName: "datasets" },
+                            ]);
+                        }
+                        catch {
+                            // Continue - direct download may still work
+                        }
                     }
-                    if (!datasetPath || !fs.existsSync(datasetPath)) {
-                        throw new McpError(ErrorCode.InvalidParams, `Dataset path not found: ${datasetPath}`);
+                    try {
+                        const result = await unifiedDatasetGateway.download({
+                            datasetId,
+                            source,
+                            targetDir,
+                        });
+                        try {
+                            upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
+                        }
+                        catch (e) {
+                            console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
+                        }
+                        const noteBlock = result.notes.length > 0 ? `\nNotes:\n- ${result.notes.join("\n- ")}` : "";
+                        return {
+                            content: [{ type: "text", text: `Download complete: ${result.copied_to || result.local_path}${noteBlock}` }]
+                        };
                     }
-                    const metadata = await metadataStore.getDataset(datasetId);
-                    const textQuality = null;
-                    const report = await qualityOrchestrator.generateReport(datasetId, datasetPath, textQuality);
-                    if (metadata) {
-                        metadata.unified_quality_report = report;
-                        await metadataStore.saveDataset(metadata);
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: download failed: ${error.message}` }],
+                            isError: true,
+                        };
                     }
-                    return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
-                }
-                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                if (!datasetId) {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='dataset'");
                 }
-                const safeId = toSafeDatasetPathFragment(datasetId);
-                const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
-                const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
-                let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
-                // Demo Fallback for easy testing
-                if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                    const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
-                    const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
-                    if (fs.existsSync(demoParquetPath)) {
-                        filePath = demoParquetPath;
-                    }
-                    else if (fs.existsSync(demoCsvPath)) {
-                        filePath = demoCsvPath;
+                case "vesper_download_assets": {
+                    hydrateExternalKeys();
+                    const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                    const source = String(request.params.arguments?.source || "").trim().toLowerCase();
+                    // Auto-infer repo_id from dataset_id if not provided (common for HuggingFace)
+                    const repoId = request.params.arguments?.repo_id
+                        ? String(request.params.arguments.repo_id)
+                        : (source === "huggingface" && datasetId.includes("/") ? datasetId : undefined);
+                    const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
+                    const urls = Array.isArray(request.params.arguments?.urls)
+                        ? (request.params.arguments?.urls).map(v => String(v))
+                        : undefined;
+                    const outputFormat = String(request.params.arguments?.output_format || "webdataset");
+                    const requestedOutputDir = request.params.arguments?.target_dir
+                        ? String(request.params.arguments.target_dir).trim()
+                        : request.params.arguments?.output_dir
+                            ? String(request.params.arguments.output_dir).trim()
+                            : undefined;
+                    const maxItems = request.params.arguments?.max_items ? Number(request.params.arguments.max_items) : undefined;
+                    const workers = request.params.arguments?.workers ? Number(request.params.arguments.workers) : 8;
+                    const imageColumn = request.params.arguments?.image_column ? String(request.params.arguments.image_column) : undefined;
+                    if (!datasetId || !source) {
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id and source are required");
                     }
-                    else if (datasetId !== "demo") {
+                    if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
                         return {
-                            content: [{ type: "text", text: `ERROR: Local data file for ${datasetId} not found. Try running prepare_dataset first, or use 'demo' as the dataset_id.` }],
-                            isError: true
+                            content: [{ type: "text", text: "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds)." }],
+                            isError: true,
                         };
                     }
-                }
-                const report = await qualityAnalyzer.analyze(filePath);
-                return {
-                    content: [{ type: "text", text: JSON.stringify(report, null, 2) }]
-                };
-            }
-            case "preview_cleaning": {
-                const datasetId = String(request.params.arguments?.dataset_id);
-                const safeId = toSafeDatasetPathFragment(datasetId);
-                const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
-                const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
-                let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
-                if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                    const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
-                    const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
-                    if (fs.existsSync(demoParquetPath)) {
-                        filePath = demoParquetPath;
+                    const requiredModules = [
+                        { module: "aiohttp", packageName: "aiohttp" },
+                    ];
+                    if (source === "url") {
+                        requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
                     }
-                    else if (fs.existsSync(demoCsvPath)) {
-                        filePath = demoCsvPath;
+                    if (source === "huggingface") {
+                        requiredModules.push({ module: "datasets", packageName: "datasets" });
+                        requiredModules.push({ module: "PIL", packageName: "Pillow" });
                     }
-                    else {
-                        throw new McpError(ErrorCode.InvalidParams, `Local data file not found for ${datasetId}. Please run prepare_dataset first.`);
+                    if (source === "kaggle") {
+                        requiredModules.push({ module: "kaggle", packageName: "kaggle" });
                     }
-                }
-                const report = await qualityAnalyzer.analyze(filePath);
-                // Phase 1: Target Detection
-                // We use the same TargetDetector instance inside CleaningPlanner now? 
-                // Actually, we instantiated it inside CleaningPlanner, so we just need to pass the file path 
-                // OR let the planner handle it if we update its signature to accept filePath.
-                // Let's check `CleaningPlanner.generatePlan` signature again.
-                // We updated it to accept `targetInfo`.
-                // So we need to run detection HERE and pass it.
-                // But `TargetDetector` is not exposed in `index.ts` scope yet.
-                // Let's create a global instance or use the one inside planner if exposed (it's private).
-                // Better approach: Instantiate TargetDetector here in index.ts for the tool content.
-                // Quick fix: Instantiate local detector or make global.
-                // I'll make a global `targetDetector` constant in index.ts
-                // But wait, I updated `CleaningPlanner` to instantiate its own detector.
-                // Does `CleaningPlanner` use it? No, I commented out the logic because it needed `filePath`.
-                // RETRY STRATEGY: 
-                // 1. Instantiate `targetDetector` in `index.ts`.
-                // 2. Run `detectTarget(filePath)`.
-                // 3. Pass result to `cleaningPlanner.generatePlan(..., targetInfo)`.
-                // I need to add `const targetDetector = new TargetDetector(__dirname);` to imports/init section first.
-                // But since I'm in this tool, I can't look back.
-                // I will assume I can add it, or just do it inside the case for now.
-                // To do it properly, I should have added `targetDetector` to the global scope in previous step.
-                // Let's do that in a separate step if needed. 
-                // For now, I'll instantiate it here.
-                const { TargetDetector } = await import("./preparation/target-detector.js");
-                const detector = new TargetDetector(__dirname);
-                const targetResult = await detector.detectTarget(filePath);
-                const targetInfo = targetResult.target_column ? {
-                    target: targetResult.target_column,
-                    confidence: targetResult.confidence
-                } : undefined;
-                const plan = await cleaningPlanner.generatePlan(datasetId, report, undefined, targetInfo);
-                let explanation = `### Cleaning Plan for ${datasetId}\n\n`;
-                if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
-                    explanation += `**Target Detected**: '${targetInfo.target}' (Confidence: ${targetInfo.confidence.toFixed(2)})\n`;
-                    explanation += `   - **Action**: Renaming to 'target' for consistency.\n\n`;
-                }
-                explanation += `Estimated Quality Improvement: +${plan.estimated_impact.quality_score_improvement} points\n\n`;
-                if (plan.operations.length === 0) {
-                    explanation += "No cleaning operations required.";
-                }
-                else {
-                    plan.operations.forEach((op, i) => {
-                        explanation += `${i + 1}. **${op.type}**: ${op.reason}\n`;
-                    });
-                }
-                return {
-                    content: [{ type: "text", text: explanation }]
-                };
-            }
-            case "custom_clean": {
-                const datasetId = String(request.params.arguments?.dataset_id);
-                const ops = request.params.arguments?.operations;
-                if (!datasetId || datasetId === "undefined") {
-                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-                }
-                if (!ops || !Array.isArray(ops) || ops.length === 0) {
-                    throw new McpError(ErrorCode.InvalidParams, "operations array is required and must not be empty. Supported operations: RemoveDuplicates, DropColumns, FillMissing, FixTypes, RemoveOutliers, EncodeCategories");
-                }
-                // Pre-check: verify dataset file exists before starting the job
-                const cleanRegEntry = getRegistryEntry(datasetId);
-                const cleanRegPath = cleanRegEntry?.local_path || cleanRegEntry?.path;
-                const cleanDlStatus = metadataStore.getDownloadStatus(datasetId);
-                const cleanSafeId = toSafeDatasetPathFragment(datasetId);
-                const cleanDataExists = (cleanRegPath && fs.existsSync(cleanRegPath)) ||
-                    (cleanDlStatus?.local_path && fs.existsSync(cleanDlStatus.local_path)) ||
-                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.parquet`)) ||
-                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.csv`)) ||
-                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.feather`)) ||
-                    fs.existsSync(datasetId);
-                if (!cleanDataExists) {
-                    return {
-                        content: [{ type: "text", text: `Dataset '${datasetId}' not found locally. Download it first using download_dataset or prepare_dataset, then run custom_clean.` }],
-                        isError: true,
-                    };
-                }
-                const job = jobManager.createJob("clean", 0, { datasetId, ops });
-                return {
-                    content: [{ type: "text", text: `Cleaning job started. ID: ${job.id}. Use check_job_status to monitor progress.` }]
-                };
-            }
-            case "prepare_dataset": {
-                hydrateExternalKeys();
-                const query = String(request.params.arguments?.query);
-                const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
-                const downloadImages = request.params.arguments?.download_images === true;
-                const requestedOutputDir = request.params.arguments?.target_dir
-                    ? String(request.params.arguments.target_dir).trim()
-                    : request.params.arguments?.output_dir
-                        ? String(request.params.arguments.output_dir).trim()
-                        : "";
-                const outputDir = requestedOutputDir || process.cwd();
-                if (!query || query === "undefined") {
-                    throw new McpError(ErrorCode.InvalidParams, "query is required - describe the dataset you need or provide a dataset ID");
-                }
-                const job = jobManager.createJob("prepare", 0, { query, requirements, downloadImages, outputDir });
-                return {
-                    content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you. Use check_job_status to monitor progress.` }]
-                };
-            }
-            case "compare_datasets": {
-                const datasetIds = request.params.arguments?.dataset_ids;
-                const datasets = datasetIds.map(id => metadataStore.getDataset(id)).filter(d => !!d);
-                let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
-                comparison += "| :--- | " + datasets.map(() => " :---: ").join("|") + " |\n";
-                comparison += "| Quality Score | " + datasets.map(d => `${d.quality_score}/100`).join(" | ") + " |\n";
-                comparison += "| License | " + datasets.map(d => d.license.category).join(" | ") + " |\n";
-                comparison += "| Downloads | " + datasets.map(d => d.downloads).join(" | ") + " |\n";
-                comparison += "| Domain | " + datasets.map(d => d.domain).join(" | ") + " |\n";
-                return {
-                    content: [{ type: "text", text: comparison }]
-                };
-            }
-            case "check_job_status": {
-                const jobId = String(request.params.arguments?.job_id);
-                const job = metadataStore.getJob(jobId);
-                if (!job) {
-                    throw new McpError(ErrorCode.InvalidParams, `Job not found: ${jobId}`);
-                }
-                const activeStatuses = new Set(["pending", "queued", "running", "retrying"]);
-                const now = Date.now();
-                const last = jobStatusLastPoll[jobId] || 0;
-                const minPollMs = 3000;
-                if (activeStatuses.has(job.status) && (now - last) < minPollMs) {
-                    const waitMs = minPollMs - (now - last);
-                    return {
-                        content: [{ type: "text", text: `No significant status change yet. Please poll again in ~${Math.ceil(waitMs / 1000)}s.` }]
+                    try {
+                        await ensurePythonModules(requiredModules);
+                    }
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
+                            isError: true,
+                        };
+                    }
+                    const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
+                    const payload = {
+                        dataset_id: datasetId,
+                        source,
+                        repo_id: repoId,
+                        kaggle_ref: kaggleRef,
+                        urls,
+                        output_format: outputFormat,
+                        output_dir: requestedOutputDir,
+                        max_items: maxItems,
+                        workers,
+                        image_column: imageColumn,
+                        output_root: requestedOutputDir || process.cwd(),
+                        recipes_dir: path.join(dataRoot, "recipes"),
                     };
-                }
-                jobStatusLastPoll[jobId] = now;
-                if (job.status === "completed") {
                     try {
-                        const meta = job.metadata ? JSON.parse(job.metadata) : {};
-                        const baseId = String(meta?.datasetId || meta?.dataset_id || meta?.query || job.id);
-                        const outPath = typeof job.result_url === "string" ? job.result_url : undefined;
-                        appendLineageVersion({
-                            datasetIdBase: baseId,
-                            tool: `job:${job.type}`,
-                            requestArgs: {
-                                dataset_id: meta?.datasetId || meta?.dataset_id,
-                                query: meta?.query,
-                                pipeline_id: meta?.pipeline_id,
-                                agent_id: meta?.agent_id,
-                            },
-                            outputPath: outPath,
-                            output: {},
-                            steps: [
-                                { step: `${job.type}_started`, at: job.created_at, params: meta || {} },
-                                { step: `${job.type}_completed`, at: job.updated_at || new Date().toISOString(), metrics: { progress: job.progress } },
-                            ],
-                        });
+                        const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
+                        if (!result?.ok) {
+                            const errMsg = result?.error || "Unknown error";
+                            // Enhance error messages for common failures
+                            let hint = "";
+                            if (errMsg.includes("No image column")) {
+                                hint = "\n\nHint: Specify image_column parameter with the column name containing images/URLs.";
+                            }
+                            else if (errMsg.includes("Authentication") || errMsg.includes("401") || errMsg.includes("403")) {
+                                hint = "\n\nHint: Use configure_keys tool to set HF_TOKEN for gated/private datasets.";
+                            }
+                            return {
+                                content: [{ type: "text", text: `ERROR: asset download failed: ${errMsg}${hint}` }],
+                                isError: true,
+                            };
+                        }
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(result.result, null, 2) }],
+                        };
                     }
-                    catch (e) {
-                        console.error(`[Lineage] check_job_status append failed: ${e?.message || e}`);
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: asset downloader execution failed: ${error.message}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                case "configure_keys": {
+                    const hfToken = String(request.params.arguments?.hf_token || "").trim();
+                    const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
+                    const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
+                    const dataworldToken = String(request.params.arguments?.dataworld_token || "").trim();
+                    const saved = [];
+                    const methods = [];
+                    if (hfToken) {
+                        const r = secureKeys.set("hf_token", hfToken);
+                        if (r.ok) {
+                            process.env.HF_TOKEN = hfToken;
+                            saved.push("HF token");
+                            if (r.method)
+                                methods.push(r.method);
+                        }
                     }
-                }
-                return {
-                    content: [{ type: "text", text: formatJobStatus(job) }]
-                };
-            }
-            case "export_dataset": {
-                const datasetId = String(request.params.arguments?.dataset_id);
-                const isDirectLocalInput = isDirectLocalDatasetReference(datasetId);
-                const intermediateArtifacts = new Set();
-                const requestedTargetDir = request.params.arguments?.target_dir
-                    ? String(request.params.arguments?.target_dir).trim()
-                    : request.params.arguments?.output_dir
-                        ? String(request.params.arguments?.output_dir).trim()
-                        : "";
-                const targetDir = path.resolve(requestedTargetDir || process.cwd());
-                const requestedFormat = String(request.params.arguments?.format || "feather");
-                const fastMode = request.params.arguments?.fast === true;
-                const preview = request.params.arguments?.preview === true;
-                const sampleRows = request.params.arguments?.sample_rows ? Number(request.params.arguments.sample_rows) : undefined;
-                const columns = request.params.arguments?.columns;
-                const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
-                // Use Metadata or Registry to find the actual local file
-                const preferredLookupDirs = [targetDir, process.cwd()];
-                let sourcePath = resolveDatasetLocalPath(datasetId, preferredLookupDirs);
-                if (!sourcePath) {
-                    console.error(`[Export] No local data found for ${datasetId}. Attempting to prepare automatically...`);
-                    // Start a prepare job for this dataset id (acts like calling prepare_dataset)
-                    try {
-                        jobManager.createJob("prepare", 0, { query: datasetId, requirements: undefined, downloadImages: false, outputDir: process.cwd() });
+                    if (kaggleUsername) {
+                        const r = secureKeys.set("kaggle_username", kaggleUsername);
+                        if (r.ok) {
+                            process.env.KAGGLE_USERNAME = kaggleUsername;
+                            saved.push("Kaggle username");
+                            if (r.method)
+                                methods.push(r.method);
+                        }
                     }
-                    catch (e) {
-                        console.error(`[Export] Failed to start prepare job for ${datasetId}: ${e?.message || e}`);
+                    if (kaggleKey) {
+                        const r = secureKeys.set("kaggle_key", kaggleKey);
+                        if (r.ok) {
+                            process.env.KAGGLE_KEY = kaggleKey;
+                            saved.push("Kaggle key");
+                            if (r.method)
+                                methods.push(r.method);
+                        }
                     }
-                    // Poll for download status or registry entry until local_path appears or timeout
-                    const wait = (ms) => new Promise(res => setTimeout(res, ms));
-                    const maxWait = 120_000; // 120s
-                    const interval = 2000;
-                    let waited = 0;
-                    while (waited < maxWait) {
-                        const resolved = resolveDatasetLocalPath(datasetId, preferredLookupDirs);
-                        if (resolved) {
-                            sourcePath = resolved;
-                            console.error(`[Export] Local data is now available for ${datasetId}: ${sourcePath}`);
-                            break;
+                    if (dataworldToken) {
+                        const r = secureKeys.set("dataworld_token", dataworldToken);
+                        if (r.ok) {
+                            process.env.DW_AUTH_TOKEN = dataworldToken;
+                            saved.push("data.world token");
+                            if (r.method)
+                                methods.push(r.method);
                         }
-                        await wait(interval);
-                        waited += interval;
                     }
-                    // If still no sourcePath, return helpful error listing prepared datasets
-                    if (!sourcePath) {
-                        const entries = readRegistry();
-                        const listText = entries.length === 0 ? "(no prepared datasets found)" : entries.map(e => `- ${e.dataset_id || e.id || "unknown"}: ${e.local_path || e.path || "unknown"}`).slice(0, 10).join("\n") + (entries.length > 10 ? "\n...and " + (entries.length - 10) + " more" : "");
+                    if (saved.length === 0) {
                         return {
-                            content: [{ type: "text", text: `ERROR: No local data found for ${datasetId} after attempting prepare. Check credentials and try running prepare_dataset manually. Prepared datasets:\n${listText}` }],
-                            isError: true
+                            content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
                         };
                     }
+                    return {
+                        content: [{ type: "text", text: `Key saved securely. Updated: ${saved.join(", ")}.` }]
+                    };
                 }
-                sourcePath = ensureExportableLocalPath(sourcePath);
-                try {
-                    if (!isDirectLocalInput && shouldTrackExportPath(sourcePath)) {
-                        upsertRegistry(datasetId, sourcePath, "completed");
+                case "get_dataset_info": {
+                    const datasetId = String(request.params.arguments?.dataset_id);
+                    if (!datasetId) {
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
                     }
-                }
-                catch (e) {
-                    console.error(`[Registry] Failed to normalize registry path for ${datasetId}: ${e?.message || e}`);
-                }
-                // If NOT fast mode, run quality/cleaning pipeline first (only for csv/parquet compat)
-                if (!fastMode) {
-                    const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
-                    const pipelineFmt = (requestedFormat === "csv" || requestedFormat === "parquet") ? requestedFormat : "parquet";
-                    const pipelineCompatibleInput = currentExt === "csv" || currentExt === "parquet";
-                    if (!pipelineCompatibleInput) {
-                        console.error(`[Export] Skipping pipeline for ${currentExt} input; using direct exporter conversion.`);
+                    const dataset = metadataStore.getDataset(datasetId);
+                    if (!dataset) {
+                        // Fallback: check the registry for local path info
+                        const regEntry = getRegistryEntry(datasetId);
+                        const regPath = regEntry?.local_path || regEntry?.path;
+                        if (regEntry) {
+                            const exists = regPath && fs.existsSync(regPath);
+                            return {
+                                content: [{ type: "text", text: `**${datasetId}** (from registry)\n- Local path: ${regPath || "unknown"}\n- Status: ${regEntry.status || "unknown"}${exists ? "" : " (file missing)"}\n\nNote: Full metadata not available in metadata store. Use prepare_dataset to get full details.` }],
+                            };
+                        }
+                        return {
+                            content: [{ type: "text", text: `ERROR: Dataset not found: ${datasetId}. Use vesper_list_datasets to see available datasets, or prepare_dataset to add new ones.` }],
+                            isError: true,
+                        };
                     }
-                    else if (currentExt !== pipelineFmt) {
-                        console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
+                    // Enrich: if total_examples is 0-ish, try the HF datasets-server /size API
+                    if ((!dataset.total_examples || dataset.total_examples === 0) && dataset.source === "huggingface") {
                         try {
-                            const beforeStagingPath = sourcePath;
-                            sourcePath = ensureLocalPipelineSource(sourcePath, datasetId, targetDir);
-                            if (path.resolve(beforeStagingPath) !== path.resolve(sourcePath)) {
-                                intermediateArtifacts.add(sourcePath);
-                            }
-                            const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
-                            if (pipelineResult.final_output_path) {
-                                if (path.resolve(pipelineResult.final_output_path) !== path.resolve(sourcePath)) {
-                                    intermediateArtifacts.add(pipelineResult.final_output_path);
-                                }
-                                sourcePath = pipelineResult.final_output_path;
-                                try {
-                                    // Update registry to point to pipeline's final output
-                                    if (!isDirectLocalInput && shouldTrackExportPath(sourcePath)) {
-                                        upsertRegistry(datasetId, sourcePath, "completed");
+                            const sizeResp = await fetch(`https://datasets-server.huggingface.co/size?dataset=${encodeURIComponent(dataset.id)}`);
+                            if (sizeResp.ok) {
+                                const sizeData = await sizeResp.json();
+                                const numRows = sizeData?.size?.dataset?.num_rows;
+                                if (numRows && numRows > 0) {
+                                    dataset.total_examples = numRows;
+                                    // Also backfill splits
+                                    if (sizeData?.size?.splits && Array.isArray(sizeData.size.splits)) {
+                                        dataset.splits = sizeData.size.splits.map((s) => ({
+                                            name: s.split,
+                                            num_examples: s.num_rows || 0,
+                                            size_bytes: s.num_bytes_parquet_files || 0,
+                                        }));
+                                        dataset.has_train_split = dataset.splits.some((s) => s.name === "train");
+                                        dataset.has_test_split = dataset.splits.some((s) => s.name === "test");
+                                        dataset.has_validation_split = dataset.splits.some((s) => s.name === "validation" || s.name === "val");
                                     }
-                                }
-                                catch (e) {
-                                    console.error(`[Registry] Failed to update registry for ${datasetId}: ${e?.message || e}`);
+                                    // Persist enriched metadata
+                                    metadataStore.saveDataset(dataset);
                                 }
                             }
                         }
-                        catch (err) {
-                            console.error(`[Export] Pipeline warning: ${err.message}. Continuing with raw file.`);
+                        catch {
+                            // Enrichment is best-effort; continue with whatever we have
                         }
                     }
+                    const formattedOutput = formatDatasetInfo(dataset);
+                    return { content: [{ type: "text", text: formattedOutput }] };
                 }
-                else {
-                    console.error(`[Export] Fast mode - skipping quality analysis and cleaning`);
-                }
-                // Build export options
-                const exportOpts = {};
-                if (compression)
-                    exportOpts.compression = compression;
-                if (preview)
-                    exportOpts.preview = true;
-                if (sampleRows)
-                    exportOpts.sample_rows = sampleRows;
-                if (columns)
-                    exportOpts.columns = columns;
-                try {
-                    // Determine output file name
-                    const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow", tfrecord: ".tfrecord" };
-                    const ext = extMap[requestedFormat] || ".feather";
-                    const safeName = getExportFileStem(datasetId);
-                    const outDir = targetDir;
-                    if (!fs.existsSync(outDir))
-                        fs.mkdirSync(outDir, { recursive: true });
-                    const outputFile = path.join(outDir, `${safeName}${ext}`);
-                    const schemaBefore = await getSchemaSnapshot(sourcePath);
-                    const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
-                    const schemaAfter = await getSchemaSnapshot(result.output_path);
-                    const lineage = appendLineageVersion({
-                        datasetIdBase: datasetId,
-                        tool: "export_dataset",
-                        requestArgs: request.params.arguments,
-                        outputPath: result.output_path,
-                        output: {
-                            rows: result.rows,
-                            columns: result.columns,
-                            format: requestedFormat,
-                            size_mb: result.file_size_mb,
-                            schema_before: schemaBefore,
-                            schema_after: schemaAfter,
-                        },
-                        steps: [
-                            { step: "source_resolved", at: new Date().toISOString(), params: { sourcePath } },
-                            { step: "exported", at: new Date().toISOString(), params: { format: requestedFormat, compression }, metrics: { rows: result.rows, columns: result.columns } },
-                        ],
-                    });
-                    try {
-                        upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                case "quality_analyze":
+                case "analyze_quality":
+                case "analyze_image_quality":
+                case "analyze_media_quality":
+                case "generate_quality_report": {
+                    const resolvedOperation = request.params.name === "analyze_image_quality"
+                        ? "image"
+                        : request.params.name === "analyze_media_quality"
+                            ? "media"
+                            : request.params.name === "generate_quality_report"
+                                ? "report"
+                                : String(request.params.arguments?.operation || "dataset").toLowerCase();
+                    if (resolvedOperation === "image") {
+                        const inputPath = String(request.params.arguments?.path || "").trim();
+                        if (!inputPath || !fs.existsSync(inputPath)) {
+                            throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
+                        }
+                        const report = await imageAnalyzer.analyze(inputPath);
+                        return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
+                    }
+                    if (resolvedOperation === "media") {
+                        const inputPath = String(request.params.arguments?.path || "").trim();
+                        if (!inputPath || !fs.existsSync(inputPath)) {
+                            throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
+                        }
+                        const report = await mediaAnalyzer.analyze(inputPath);
+                        return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
                     }
-                    catch { }
-                    // Build rich response
-                    let msg = `**Export complete**\n`;
-                    msg += `- **File**: ${result.output_path}\n`;
-                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
-                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
-                    msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
-                    msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
-                    if (result.file_size_mb !== undefined)
-                        msg += `- **Size**: ${result.file_size_mb} MB\n`;
-                    if (result.elapsed_seconds !== undefined)
-                        msg += `- **Time**: ${result.elapsed_seconds}s\n`;
-                    if (result.preview_path)
-                        msg += `- **Preview**: ${result.preview_path}\n`;
-                    msg += `\n`;
-                    if (requestedFormat === "feather") {
-                        msg += `**Inspect with:**\n`;
-                        msg += `  Python: \`pd.read_feather('${result.output_path}').head()\`\n`;
-                        msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                    if (resolvedOperation === "report") {
+                        const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                        const datasetPath = String(request.params.arguments?.dataset_path || "").trim();
+                        if (!datasetId) {
+                            throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='report'");
+                        }
+                        if (!datasetPath || !fs.existsSync(datasetPath)) {
+                            throw new McpError(ErrorCode.InvalidParams, `Dataset path not found: ${datasetPath}`);
+                        }
+                        const metadata = await metadataStore.getDataset(datasetId);
+                        const textQuality = null;
+                        const report = await qualityOrchestrator.generateReport(datasetId, datasetPath, textQuality);
+                        if (metadata) {
+                            metadata.unified_quality_report = report;
+                            await metadataStore.saveDataset(metadata);
+                        }
+                        return { content: [{ type: "text", text: JSON.stringify(report, null, 2) }] };
                     }
-                    else if (requestedFormat === "parquet") {
-                        msg += `**Inspect with:**\n`;
-                        msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
-                        msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                    const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                    if (!datasetId) {
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='dataset'");
                     }
-                    cleanupIntermediateArtifacts(intermediateArtifacts, result.output_path);
-                    return { content: [{ type: "text", text: msg }] };
-                }
-                catch (error) {
+                    const safeId = toSafeDatasetPathFragment(datasetId);
+                    const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
+                    const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
+                    let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
+                    // Demo Fallback for easy testing
+                    if (datasetId === "demo" || !fs.existsSync(filePath)) {
+                        const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
+                        const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
+                        if (fs.existsSync(demoParquetPath)) {
+                            filePath = demoParquetPath;
+                        }
+                        else if (fs.existsSync(demoCsvPath)) {
+                            filePath = demoCsvPath;
+                        }
+                        else if (datasetId !== "demo") {
+                            return {
+                                content: [{ type: "text", text: `ERROR: Local data file for ${datasetId} not found. Try running prepare_dataset first, or use 'demo' as the dataset_id.` }],
+                                isError: true
+                            };
+                        }
+                    }
+                    const report = await qualityAnalyzer.analyze(filePath);
                     return {
-                        content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
-                        isError: true
+                        content: [{ type: "text", text: JSON.stringify(report, null, 2) }]
                     };
                 }
-            }
-            case "vesper_list_datasets": {
-                const entries = readRegistry();
-                if (entries.length === 0) {
+                case "preview_cleaning": {
+                    const datasetId = String(request.params.arguments?.dataset_id);
+                    const safeId = toSafeDatasetPathFragment(datasetId);
+                    const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
+                    const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
+                    let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
+                    if (datasetId === "demo" || !fs.existsSync(filePath)) {
+                        const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
+                        const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
+                        if (fs.existsSync(demoParquetPath)) {
+                            filePath = demoParquetPath;
+                        }
+                        else if (fs.existsSync(demoCsvPath)) {
+                            filePath = demoCsvPath;
+                        }
+                        else {
+                            throw new McpError(ErrorCode.InvalidParams, `Local data file not found for ${datasetId}. Please run prepare_dataset first.`);
+                        }
+                    }
+                    const report = await qualityAnalyzer.analyze(filePath);
+                    // Phase 1: Target Detection
+                    // We use the same TargetDetector instance inside CleaningPlanner now? 
+                    // Actually, we instantiated it inside CleaningPlanner, so we just need to pass the file path 
+                    // OR let the planner handle it if we update its signature to accept filePath.
+                    // Let's check `CleaningPlanner.generatePlan` signature again.
+                    // We updated it to accept `targetInfo`.
+                    // So we need to run detection HERE and pass it.
+                    // But `TargetDetector` is not exposed in `index.ts` scope yet.
+                    // Let's create a global instance or use the one inside planner if exposed (it's private).
+                    // Better approach: Instantiate TargetDetector here in index.ts for the tool content.
+                    // Quick fix: Instantiate local detector or make global.
+                    // I'll make a global `targetDetector` constant in index.ts
+                    // But wait, I updated `CleaningPlanner` to instantiate its own detector.
+                    // Does `CleaningPlanner` use it? No, I commented out the logic because it needed `filePath`.
+                    // RETRY STRATEGY: 
+                    // 1. Instantiate `targetDetector` in `index.ts`.
+                    // 2. Run `detectTarget(filePath)`.
+                    // 3. Pass result to `cleaningPlanner.generatePlan(..., targetInfo)`.
+                    // I need to add `const targetDetector = new TargetDetector(__dirname);` to imports/init section first.
+                    // But since I'm in this tool, I can't look back.
+                    // I will assume I can add it, or just do it inside the case for now.
+                    // To do it properly, I should have added `targetDetector` to the global scope in previous step.
+                    // Let's do that in a separate step if needed. 
+                    // For now, I'll instantiate it here.
+                    const { TargetDetector } = await import("./preparation/target-detector.js");
+                    const detector = new TargetDetector(__dirname);
+                    const targetResult = await detector.detectTarget(filePath);
+                    const targetInfo = targetResult.target_column ? {
+                        target: targetResult.target_column,
+                        confidence: targetResult.confidence
+                    } : undefined;
+                    const plan = await cleaningPlanner.generatePlan(datasetId, report, undefined, targetInfo);
+                    let explanation = `### Cleaning Plan for ${datasetId}\n\n`;
+                    if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
+                        explanation += `**Target Detected**: '${targetInfo.target}' (Confidence: ${targetInfo.confidence.toFixed(2)})\n`;
+                        explanation += `   - **Action**: Renaming to 'target' for consistency.\n\n`;
+                    }
+                    explanation += `Estimated Quality Improvement: +${plan.estimated_impact.quality_score_improvement} points\n\n`;
+                    if (plan.operations.length === 0) {
+                        explanation += "No cleaning operations required.";
+                    }
+                    else {
+                        plan.operations.forEach((op, i) => {
+                            explanation += `${i + 1}. **${op.type}**: ${op.reason}\n`;
+                        });
+                    }
                     return {
-                        content: [{ type: "text", text: "No prepared datasets found in the Vesper registry. Use prepare_dataset or download_dataset to add datasets." }]
+                        content: [{ type: "text", text: explanation }]
                     };
                 }
-                const lines = entries.map((e, i) => {
-                    const id = e.dataset_id || e.id || "unknown";
-                    const localPath = e.local_path || e.path || "unknown";
-                    const exists = typeof localPath === "string" && localPath !== "unknown" && fs.existsSync(localPath);
-                    return `${i + 1}. **${id}**\n   Path: ${localPath}\n   Status: ${e.status || "unknown"}${exists ? "" : " (file missing)"}`;
-                });
-                return {
-                    content: [{ type: "text", text: `**Vesper Registry** (${entries.length} dataset${entries.length !== 1 ? "s" : ""}):\n\n${lines.join("\n\n")}` }]
-                };
-            }
-            case "vesper_convert_format": {
-                const filePath = String(request.params.arguments?.file_path || "").trim();
-                const targetFormat = String(request.params.arguments?.target_format || "").trim().toLowerCase();
-                if (!filePath) {
-                    throw new McpError(ErrorCode.InvalidParams, "file_path is required");
-                }
-                if (!["csv", "parquet", "json", "jsonl"].includes(targetFormat)) {
-                    throw new McpError(ErrorCode.InvalidParams, "target_format must be one of: csv, parquet, json, jsonl");
+                case "custom_clean": {
+                    const datasetId = String(request.params.arguments?.dataset_id);
+                    const ops = request.params.arguments?.operations;
+                    if (!datasetId || datasetId === "undefined") {
+                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                    }
+                    if (!ops || !Array.isArray(ops) || ops.length === 0) {
+                        throw new McpError(ErrorCode.InvalidParams, "operations array is required and must not be empty. Supported operations: RemoveDuplicates, DropColumns, FillMissing, FixTypes, RemoveOutliers, EncodeCategories");
+                    }
+                    // Pre-check: verify dataset file exists before starting the job
+                    const cleanRegEntry = getRegistryEntry(datasetId);
+                    const cleanRegPath = cleanRegEntry?.local_path || cleanRegEntry?.path;
+                    const cleanDlStatus = metadataStore.getDownloadStatus(datasetId);
+                    const cleanSafeId = toSafeDatasetPathFragment(datasetId);
+                    const cleanDataExists = (cleanRegPath && fs.existsSync(cleanRegPath)) ||
+                        (cleanDlStatus?.local_path && fs.existsSync(cleanDlStatus.local_path)) ||
+                        fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.parquet`)) ||
+                        fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.csv`)) ||
+                        fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.feather`)) ||
+                        fs.existsSync(datasetId);
+                    if (!cleanDataExists) {
+                        return {
+                            content: [{ type: "text", text: `Dataset '${datasetId}' not found locally. Download it first using download_dataset or prepare_dataset, then run custom_clean.` }],
+                            isError: true,
+                        };
+                    }
+                    const job = jobManager.createJob("clean", 0, { datasetId, ops });
+                    return {
+                        content: [{ type: "text", text: `Cleaning job started. ID: ${job.id}. Use check_job_status to monitor progress.` }]
+                    };
                 }
-                if (!fs.existsSync(filePath)) {
+                case "prepare_dataset": {
+                    hydrateExternalKeys();
+                    const query = String(request.params.arguments?.query);
+                    const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
+                    const downloadImages = request.params.arguments?.download_images === true;
+                    const requestedOutputDir = request.params.arguments?.target_dir
+                        ? String(request.params.arguments.target_dir).trim()
+                        : request.params.arguments?.output_dir
+                            ? String(request.params.arguments.output_dir).trim()
+                            : "";
+                    const outputDir = requestedOutputDir || process.cwd();
+                    if (!query || query === "undefined") {
+                        throw new McpError(ErrorCode.InvalidParams, "query is required - describe the dataset you need or provide a dataset ID");
+                    }
+                    const job = jobManager.createJob("prepare", 0, { query, requirements, downloadImages, outputDir });
                     return {
-                        content: [{ type: "text", text: `ERROR: File not found: ${filePath}` }],
-                        isError: true,
+                        content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you. Use check_job_status to monitor progress.` }]
                     };
                 }
-                const inputExt = path.extname(filePath).toLowerCase();
-                const extMap = { csv: ".csv", parquet: ".parquet", json: ".json", jsonl: ".jsonl" };
-                const outputExt = extMap[targetFormat];
-                if (inputExt === outputExt) {
+                case "compare_datasets": {
+                    const datasetIds = request.params.arguments?.dataset_ids;
+                    const datasets = datasetIds.map(id => metadataStore.getDataset(id)).filter(d => !!d);
+                    let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
+                    comparison += "| :--- | " + datasets.map(() => " :---: ").join("|") + " |\n";
+                    comparison += "| Quality Score | " + datasets.map(d => `${d.quality_score}/100`).join(" | ") + " |\n";
+                    comparison += "| License | " + datasets.map(d => d.license.category).join(" | ") + " |\n";
+                    comparison += "| Downloads | " + datasets.map(d => d.downloads).join(" | ") + " |\n";
+                    comparison += "| Domain | " + datasets.map(d => d.domain).join(" | ") + " |\n";
                     return {
-                        content: [{ type: "text", text: `File is already in ${targetFormat} format: ${filePath}` }],
+                        content: [{ type: "text", text: comparison }]
                     };
                 }
-                const outputPath = filePath.replace(/\.[^.]+$/, outputExt);
-                try {
-                    await ensurePythonModules([{ module: "polars", packageName: "polars" }]);
-                    const convertScript = path.join(dataRoot, "python", "convert_engine.py");
-                    const schemaBefore = await getSchemaSnapshot(filePath);
-                    const result = await runPythonJson(convertScript, [filePath, outputPath]);
-                    const schemaAfter = await getSchemaSnapshot(outputPath);
-                    if (!result.ok) {
+                case "check_job_status": {
+                    const jobId = String(request.params.arguments?.job_id);
+                    const job = metadataStore.getJob(jobId);
+                    if (!job) {
+                        throw new McpError(ErrorCode.InvalidParams, `Job not found: ${jobId}`);
+                    }
+                    const activeStatuses = new Set(["pending", "queued", "running", "retrying"]);
+                    const now = Date.now();
+                    const last = jobStatusLastPoll[jobId] || 0;
+                    const minPollMs = 3000;
+                    if (activeStatuses.has(job.status) && (now - last) < minPollMs) {
+                        const waitMs = minPollMs - (now - last);
                         return {
-                            content: [{ type: "text", text: `ERROR: Conversion failed: ${result.error}` }],
-                            isError: true,
+                            content: [{ type: "text", text: `No significant status change yet. Please poll again in ~${Math.ceil(waitMs / 1000)}s.` }]
                         };
                     }
-                    // Register converted file in the registry
-                    const datasetId = path.basename(outputPath, outputExt);
-                    try {
-                        upsertRegistry(datasetId, outputPath, "completed");
-                    }
-                    catch (e) {
-                        console.error(`[Convert] Registry write failed: ${e?.message || e}`);
-                    }
-                    const lineage = appendLineageVersion({
-                        datasetIdBase: datasetId,
-                        tool: "vesper_convert_format",
-                        requestArgs: request.params.arguments,
-                        outputPath,
-                        output: {
-                            rows: result.rows,
-                            columns: result.columns,
-                            format: targetFormat,
-                            size_mb: result.size_mb,
-                            schema_before: schemaBefore,
-                            schema_after: schemaAfter,
-                        },
-                        steps: [
-                            { step: "converted", at: new Date().toISOString(), params: { from: inputExt, to: outputExt } },
-                        ],
-                    });
-                    try {
-                        upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                    jobStatusLastPoll[jobId] = now;
+                    if (job.status === "completed") {
+                        try {
+                            const meta = job.metadata ? JSON.parse(job.metadata) : {};
+                            const baseId = String(meta?.datasetId || meta?.dataset_id || meta?.query || job.id);
+                            const outPath = typeof job.result_url === "string" ? job.result_url : undefined;
+                            appendLineageVersion({
+                                datasetIdBase: baseId,
+                                tool: `job:${job.type}`,
+                                requestArgs: {
+                                    dataset_id: meta?.datasetId || meta?.dataset_id,
+                                    query: meta?.query,
+                                    pipeline_id: meta?.pipeline_id,
+                                    agent_id: meta?.agent_id,
+                                },
+                                outputPath: outPath,
+                                output: {},
+                                steps: [
+                                    { step: `${job.type}_started`, at: job.created_at, params: meta || {} },
+                                    { step: `${job.type}_completed`, at: job.updated_at || new Date().toISOString(), metrics: { progress: job.progress } },
+                                ],
+                            });
+                        }
+                        catch (e) {
+                            console.error(`[Lineage] check_job_status append failed: ${e?.message || e}`);
+                        }
                     }
-                    catch { }
-                    let msg = `**Conversion complete**\n`;
-                    msg += `- **Input**: ${filePath} (${inputExt.slice(1)})\n`;
-                    msg += `- **Output**: ${result.output_path} (${targetFormat})\n`;
-                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
-                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
-                    msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
-                    if (result.size_mb !== undefined)
-                        msg += `- **Size**: ${result.size_mb} MB\n`;
-                    return { content: [{ type: "text", text: msg }] };
-                }
-                catch (error) {
                     return {
-                        content: [{ type: "text", text: `ERROR: Conversion failed: ${error.message}` }],
-                        isError: true,
+                        content: [{ type: "text", text: formatJobStatus(job) }]
                     };
                 }
-            }
-            case "vesper_normalize_schema": {
-                const filePath = String(request.params.arguments?.file_path || "").trim();
-                const outputFormat = String(request.params.arguments?.output_format || "jsonl").trim().toLowerCase();
-                const outputDirRaw = request.params.arguments?.output_dir ? String(request.params.arguments.output_dir).trim() : "";
-                const flattenMetadataJson = request.params.arguments?.flatten_metadata_json !== false;
-                const maxKeys = Number(request.params.arguments?.max_keys ?? 200);
-                const extrasMode = String(request.params.arguments?.extras_mode || "blob").trim().toLowerCase();
-                if (!filePath) {
-                    throw new McpError(ErrorCode.InvalidParams, "file_path is required");
-                }
-                if (!["jsonl", "json"].includes(outputFormat)) {
-                    throw new McpError(ErrorCode.InvalidParams, "output_format must be one of: jsonl, json");
-                }
-                if (!fs.existsSync(filePath)) {
-                    return { content: [{ type: "text", text: `ERROR: File not found: ${filePath}` }], isError: true };
-                }
-                const outDir = outputDirRaw || path.join(dataRoot, "data", "normalized_schema");
-                if (!fs.existsSync(outDir))
-                    fs.mkdirSync(outDir, { recursive: true });
-                const baseName = path.parse(filePath).name || `normalized_${Date.now()}`;
-                const outputPath = path.join(outDir, `${baseName}.normalized.${outputFormat}`);
-                try {
-                    const scriptPath = path.join(dataRoot, "python", "normalize_schema_engine.py");
-                    const options = {
-                        flatten_metadata_json: !!flattenMetadataJson,
-                        max_keys: Number.isFinite(maxKeys) ? maxKeys : 200,
-                        extras_mode: ["blob", "drop"].includes(extrasMode) ? extrasMode : "blob",
-                    };
-                    const schemaBefore = await getSchemaSnapshot(filePath);
-                    const result = await runPythonJson(scriptPath, [filePath, outputPath, JSON.stringify(options)]);
-                    const schemaAfter = await getSchemaSnapshot(outputPath);
-                    if (!result.ok) {
-                        return { content: [{ type: "text", text: `ERROR: Schema normalization failed: ${result.error}` }], isError: true };
+                case "export_dataset": {
+                    const datasetId = String(request.params.arguments?.dataset_id);
+                    const isDirectLocalInput = isDirectLocalDatasetReference(datasetId);
+                    const intermediateArtifacts = new Set();
+                    const requestedTargetDir = request.params.arguments?.target_dir
+                        ? String(request.params.arguments?.target_dir).trim()
+                        : request.params.arguments?.output_dir
+                            ? String(request.params.arguments?.output_dir).trim()
+                            : "";
+                    const targetDir = path.resolve(requestedTargetDir || process.cwd());
+                    const requestedFormat = String(request.params.arguments?.format || "feather");
+                    const fastMode = request.params.arguments?.fast === true;
+                    const preview = request.params.arguments?.preview === true;
+                    const sampleRows = request.params.arguments?.sample_rows ? Number(request.params.arguments.sample_rows) : undefined;
+                    const columns = request.params.arguments?.columns;
+                    const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
+                    // Use Metadata or Registry to find the actual local file
+                    const preferredLookupDirs = [targetDir, process.cwd()];
+                    let sourcePath = resolveDatasetLocalPath(datasetId, preferredLookupDirs);
+                    if (!sourcePath) {
+                        console.error(`[Export] No local data found for ${datasetId}. Attempting to prepare automatically...`);
+                        // Start a prepare job for this dataset id (acts like calling prepare_dataset)
+                        try {
+                            jobManager.createJob("prepare", 0, { query: datasetId, requirements: undefined, downloadImages: false, outputDir: process.cwd() });
+                        }
+                        catch (e) {
+                            console.error(`[Export] Failed to start prepare job for ${datasetId}: ${e?.message || e}`);
+                        }
+                        // Poll for download status or registry entry until local_path appears or timeout
+                        const wait = (ms) => new Promise(res => setTimeout(res, ms));
+                        const maxWait = 120_000; // 120s
+                        const interval = 2000;
+                        let waited = 0;
+                        while (waited < maxWait) {
+                            const resolved = resolveDatasetLocalPath(datasetId, preferredLookupDirs);
+                            if (resolved) {
+                                sourcePath = resolved;
+                                console.error(`[Export] Local data is now available for ${datasetId}: ${sourcePath}`);
+                                break;
+                            }
+                            await wait(interval);
+                            waited += interval;
+                        }
+                        // If still no sourcePath, return helpful error listing prepared datasets
+                        if (!sourcePath) {
+                            const entries = readRegistry();
+                            const listText = entries.length === 0 ? "(no prepared datasets found)" : entries.map(e => `- ${e.dataset_id || e.id || "unknown"}: ${e.local_path || e.path || "unknown"}`).slice(0, 10).join("\n") + (entries.length > 10 ? "\n...and " + (entries.length - 10) + " more" : "");
+                            return {
+                                content: [{ type: "text", text: `ERROR: No local data found for ${datasetId} after attempting prepare. Check credentials and try running prepare_dataset manually. Prepared datasets:\n${listText}` }],
+                                isError: true
+                            };
+                        }
                     }
-                    // Register normalized file to make follow-up conversion easier.
+                    sourcePath = ensureExportableLocalPath(sourcePath);
                     try {
-                        const datasetId = path.basename(outputPath, path.extname(outputPath));
-                        upsertRegistry(datasetId, outputPath, "completed");
+                        if (!isDirectLocalInput && shouldTrackExportPath(sourcePath)) {
+                            upsertRegistry(datasetId, sourcePath, "completed");
+                        }
                     }
                     catch (e) {
-                        console.error(`[NormalizeSchema] Registry write failed: ${e?.message || e}`);
+                        console.error(`[Registry] Failed to normalize registry path for ${datasetId}: ${e?.message || e}`);
                     }
-                    const lineage = appendLineageVersion({
-                        datasetIdBase: path.basename(outputPath, path.extname(outputPath)),
-                        tool: "vesper_normalize_schema",
-                        requestArgs: request.params.arguments,
-                        outputPath,
-                        output: {
-                            rows: result.rows,
-                            columns: result.columns,
-                            format: outputFormat,
-                            schema_before: schemaBefore,
-                            schema_after: schemaAfter,
-                        },
-                        steps: [
-                            { step: "schema_normalized", at: new Date().toISOString(), params: options, metrics: { flattened_keys: result.flattened_keys } },
-                        ],
-                    });
+                    // If NOT fast mode, run quality/cleaning pipeline first (only for csv/parquet compat)
+                    if (!fastMode) {
+                        const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
+                        const pipelineFmt = (requestedFormat === "csv" || requestedFormat === "parquet") ? requestedFormat : "parquet";
+                        const pipelineCompatibleInput = currentExt === "csv" || currentExt === "parquet";
+                        if (!pipelineCompatibleInput) {
+                            console.error(`[Export] Skipping pipeline for ${currentExt} input; using direct exporter conversion.`);
+                        }
+                        else if (currentExt !== pipelineFmt) {
+                            console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
+                            try {
+                                const beforeStagingPath = sourcePath;
+                                sourcePath = ensureLocalPipelineSource(sourcePath, datasetId, targetDir);
+                                if (path.resolve(beforeStagingPath) !== path.resolve(sourcePath)) {
+                                    intermediateArtifacts.add(sourcePath);
+                                }
+                                const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
+                                if (pipelineResult.final_output_path) {
+                                    if (path.resolve(pipelineResult.final_output_path) !== path.resolve(sourcePath)) {
+                                        intermediateArtifacts.add(pipelineResult.final_output_path);
+                                    }
+                                    sourcePath = pipelineResult.final_output_path;
+                                    try {
+                                        // Update registry to point to pipeline's final output
+                                        if (!isDirectLocalInput && shouldTrackExportPath(sourcePath)) {
+                                            upsertRegistry(datasetId, sourcePath, "completed");
+                                        }
+                                    }
+                                    catch (e) {
+                                        console.error(`[Registry] Failed to update registry for ${datasetId}: ${e?.message || e}`);
+                                    }
+                                }
+                            }
+                            catch (err) {
+                                console.error(`[Export] Pipeline warning: ${err.message}. Continuing with raw file.`);
+                            }
+                        }
+                    }
+                    else {
+                        console.error(`[Export] Fast mode - skipping quality analysis and cleaning`);
+                    }
+                    // Build export options
+                    const exportOpts = {};
+                    if (compression)
+                        exportOpts.compression = compression;
+                    if (preview)
+                        exportOpts.preview = true;
+                    if (sampleRows)
+                        exportOpts.sample_rows = sampleRows;
+                    if (columns)
+                        exportOpts.columns = columns;
                     try {
-                        upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                        // Determine output file name
+                        const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow", tfrecord: ".tfrecord" };
+                        const ext = extMap[requestedFormat] || ".feather";
+                        const safeName = getExportFileStem(datasetId);
+                        const outDir = targetDir;
+                        if (!fs.existsSync(outDir))
+                            fs.mkdirSync(outDir, { recursive: true });
+                        const outputFile = path.join(outDir, `${safeName}${ext}`);
+                        const schemaBefore = await getSchemaSnapshot(sourcePath);
+                        const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
+                        const schemaAfter = await getSchemaSnapshot(result.output_path);
+                        const lineage = appendLineageVersion({
+                            datasetIdBase: datasetId,
+                            tool: "export_dataset",
+                            requestArgs: request.params.arguments,
+                            outputPath: result.output_path,
+                            output: {
+                                rows: result.rows,
+                                columns: result.columns,
+                                format: requestedFormat,
+                                size_mb: result.file_size_mb,
+                                schema_before: schemaBefore,
+                                schema_after: schemaAfter,
+                            },
+                            steps: [
+                                { step: "source_resolved", at: new Date().toISOString(), params: { sourcePath } },
+                                { step: "exported", at: new Date().toISOString(), params: { format: requestedFormat, compression }, metrics: { rows: result.rows, columns: result.columns } },
+                            ],
+                        });
+                        try {
+                            upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                        }
+                        catch { }
+                        // Build rich response
+                        let msg = `**Export complete**\n`;
+                        msg += `- **File**: ${result.output_path}\n`;
+                        msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                        msg += `- **Lineage**: ${lineage.lineagePath}\n`;
+                        msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
+                        msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
+                        if (result.file_size_mb !== undefined)
+                            msg += `- **Size**: ${result.file_size_mb} MB\n`;
+                        if (result.elapsed_seconds !== undefined)
+                            msg += `- **Time**: ${result.elapsed_seconds}s\n`;
+                        if (result.preview_path)
+                            msg += `- **Preview**: ${result.preview_path}\n`;
+                        msg += `\n`;
+                        if (requestedFormat === "feather") {
+                            msg += `**Inspect with:**\n`;
+                            msg += `  Python: \`pd.read_feather('${result.output_path}').head()\`\n`;
+                            msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                        }
+                        else if (requestedFormat === "parquet") {
+                            msg += `**Inspect with:**\n`;
+                            msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
+                            msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                        }
+                        cleanupIntermediateArtifacts(intermediateArtifacts, result.output_path);
+                        return { content: [{ type: "text", text: msg }] };
+                    }
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
+                            isError: true
+                        };
                     }
-                    catch { }
-                    let msg = `**Schema normalization complete**\n`;
-                    msg += `- **Input**: ${filePath}\n`;
-                    msg += `- **Output**: ${result.output_path}\n`;
-                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
-                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
-                    msg += `- **Rows**: ${result.rows?.toLocaleString?.() ?? result.rows}\n`;
-                    msg += `- **Columns**: ${result.columns}\n`;
-                    msg += `- **Flattened keys**: ${result.flattened_keys}\n`;
-                    msg += `- **Extras mode**: ${result.extras_mode}\n`;
-                    if (result.extras_rows !== undefined)
-                        msg += `- **Rows with extras**: ${result.extras_rows}\n`;
-                    msg += `\nNext: run \`vesper_convert_format\` on the output to convert to parquet.\n`;
-                    return { content: [{ type: "text", text: msg }] };
-                }
-                catch (error) {
-                    return { content: [{ type: "text", text: `ERROR: Schema normalization failed: ${error.message}` }], isError: true };
                 }
-            }
-            case "fuse":
-            case "fuse_datasets": {
-                const operation = request.params.name === "fuse_datasets"
-                    ? "tabular"
-                    : String(request.params.arguments?.operation || "tabular").toLowerCase();
-                if (operation === "web") {
-                    hydrateExternalKeys();
-                    const webSources = Array.isArray(request.params.arguments?.sources)
-                        ? request.params.arguments?.sources
-                        : undefined;
-                    if (!webSources || !Array.isArray(webSources)) {
+                case "vesper_list_datasets": {
+                    const entries = readRegistry();
+                    if (entries.length === 0) {
                         return {
-                            content: [{ type: "text", text: "ERROR: fuse(operation='web') requires 'sources' array." }],
-                            isError: true,
+                            content: [{ type: "text", text: "No prepared datasets found in the Vesper registry. Use prepare_dataset or download_dataset to add datasets." }]
                         };
                     }
-                    const mergeStrategyRaw = request.params.arguments?.merge_strategy
-                        ? String(request.params.arguments?.merge_strategy).toLowerCase()
-                        : undefined;
-                    const dedupRaw = request.params.arguments?.deduplication
-                        ? String(request.params.arguments?.deduplication).toLowerCase()
-                        : undefined;
-                    const merge_strategy = mergeStrategyRaw && ["union", "dedup"].includes(mergeStrategyRaw)
-                        ? mergeStrategyRaw
-                        : undefined;
-                    const deduplication = dedupRaw && ["semantic", "exact", "none"].includes(dedupRaw)
-                        ? dedupRaw
-                        : undefined;
-                    const webResult = await webFusionEngine.fuse({
-                        sources: webSources.map((s) => ({
-                            type: String(s?.type || "").trim().toLowerCase(),
-                            query: String(s?.query || "").trim(),
-                            max_results: s?.max_results !== undefined ? Number(s.max_results) : undefined,
-                            min_stars: s?.min_stars !== undefined ? Number(s.min_stars) : undefined,
-                            bucket: s?.bucket !== undefined ? String(s.bucket) : undefined,
-                            path: s?.path !== undefined ? String(s.path) : undefined,
-                            region: s?.region !== undefined ? String(s.region) : undefined,
-                            credentials: s?.credentials ? {
-                                accessKeyId: s.credentials.accessKeyId !== undefined ? String(s.credentials.accessKeyId) : undefined,
-                                secretAccessKey: s.credentials.secretAccessKey !== undefined ? String(s.credentials.secretAccessKey) : undefined,
-                                sessionToken: s.credentials.sessionToken !== undefined ? String(s.credentials.sessionToken) : undefined,
-                                roleArn: s.credentials.roleArn !== undefined ? String(s.credentials.roleArn) : undefined,
-                            } : undefined,
-                        })),
-                        merge_strategy,
-                        deduplication,
+                    const lines = entries.map((e, i) => {
+                        const id = e.dataset_id || e.id || "unknown";
+                        const localPath = e.local_path || e.path || "unknown";
+                        const exists = typeof localPath === "string" && localPath !== "unknown" && fs.existsSync(localPath);
+                        return `${i + 1}. **${id}**\n   Path: ${localPath}\n   Status: ${e.status || "unknown"}${exists ? "" : " (file missing)"}`;
                     });
                     return {
-                        content: [{ type: "text", text: JSON.stringify(webResult, null, 2) }],
+                        content: [{ type: "text", text: `**Vesper Registry** (${entries.length} dataset${entries.length !== 1 ? "s" : ""}):\n\n${lines.join("\n\n")}` }]
                     };
                 }
-                const rawSources = request.params.arguments?.sources;
-                if (!rawSources || !Array.isArray(rawSources) || rawSources.length < 2) {
-                    throw new McpError(ErrorCode.InvalidParams, "sources must contain at least 2 dataset IDs/paths");
-                }
-                const strategy = request.params.arguments?.strategy || "concat";
-                const joinOn = request.params.arguments?.join_on;
-                const how = request.params.arguments?.how || "inner";
-                const dedup = request.params.arguments?.dedup !== false;
-                const runQualityAfter = request.params.arguments?.run_quality_after !== false;
-                const leakageCheck = request.params.arguments?.leakage_check !== false;
-                const outputFormat = request.params.arguments?.output_format || "feather";
-                const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
-                const preview = request.params.arguments?.preview !== false;
-                const resolvedPaths = [];
-                const unresolved = [];
-                for (const src of rawSources) {
-                    if (fs.existsSync(src)) {
-                        resolvedPaths.push(src);
-                        continue;
+                case "vesper_convert_format": {
+                    const filePath = String(request.params.arguments?.file_path || "").trim();
+                    const targetFormat = String(request.params.arguments?.target_format || "").trim().toLowerCase();
+                    if (!filePath) {
+                        throw new McpError(ErrorCode.InvalidParams, "file_path is required");
                     }
-                    const status = metadataStore.getDownloadStatus(src);
-                    if (status?.local_path && fs.existsSync(status.local_path)) {
-                        resolvedPaths.push(status.local_path);
-                        continue;
+                    if (!["csv", "parquet", "json", "jsonl"].includes(targetFormat)) {
+                        throw new McpError(ErrorCode.InvalidParams, "target_format must be one of: csv, parquet, json, jsonl");
+                    }
+                    if (!fs.existsSync(filePath)) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: File not found: ${filePath}` }],
+                            isError: true,
+                        };
+                    }
+                    const inputExt = path.extname(filePath).toLowerCase();
+                    const extMap = { csv: ".csv", parquet: ".parquet", json: ".json", jsonl: ".jsonl" };
+                    const outputExt = extMap[targetFormat];
+                    if (inputExt === outputExt) {
+                        return {
+                            content: [{ type: "text", text: `File is already in ${targetFormat} format: ${filePath}` }],
+                        };
+                    }
+                    const outputPath = filePath.replace(/\.[^.]+$/, outputExt);
+                    try {
+                        await ensurePythonModules([{ module: "polars", packageName: "polars" }]);
+                        const convertScript = path.join(dataRoot, "python", "convert_engine.py");
+                        const schemaBefore = await getSchemaSnapshot(filePath);
+                        const result = await runPythonJson(convertScript, [filePath, outputPath]);
+                        const schemaAfter = await getSchemaSnapshot(outputPath);
+                        if (!result.ok) {
+                            return {
+                                content: [{ type: "text", text: `ERROR: Conversion failed: ${result.error}` }],
+                                isError: true,
+                            };
+                        }
+                        // Register converted file in the registry
+                        const datasetId = path.basename(outputPath, outputExt);
+                        try {
+                            upsertRegistry(datasetId, outputPath, "completed");
+                        }
+                        catch (e) {
+                            console.error(`[Convert] Registry write failed: ${e?.message || e}`);
+                        }
+                        const lineage = appendLineageVersion({
+                            datasetIdBase: datasetId,
+                            tool: "vesper_convert_format",
+                            requestArgs: request.params.arguments,
+                            outputPath,
+                            output: {
+                                rows: result.rows,
+                                columns: result.columns,
+                                format: targetFormat,
+                                size_mb: result.size_mb,
+                                schema_before: schemaBefore,
+                                schema_after: schemaAfter,
+                            },
+                            steps: [
+                                { step: "converted", at: new Date().toISOString(), params: { from: inputExt, to: outputExt } },
+                            ],
+                        });
+                        try {
+                            upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                        }
+                        catch { }
+                        let msg = `**Conversion complete**\n`;
+                        msg += `- **Input**: ${filePath} (${inputExt.slice(1)})\n`;
+                        msg += `- **Output**: ${result.output_path} (${targetFormat})\n`;
+                        msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                        msg += `- **Lineage**: ${lineage.lineagePath}\n`;
+                        msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
+                        if (result.size_mb !== undefined)
+                            msg += `- **Size**: ${result.size_mb} MB\n`;
+                        return { content: [{ type: "text", text: msg }] };
+                    }
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: Conversion failed: ${error.message}` }],
+                            isError: true,
+                        };
                     }
-                    unresolved.push(src);
-                }
-                if (unresolved.length > 0) {
-                    return {
-                        content: [{
-                                type: "text",
-                                text: `ERROR: Could not resolve these sources to local files: ${unresolved.join(", ")}. Provide local paths or run prepare_dataset first.`
-                            }],
-                        isError: true
-                    };
                 }
-                try {
-                    const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow" };
-                    const ext = extMap[outputFormat] || ".feather";
-                    const outDir = process.cwd();
+                case "vesper_normalize_schema": {
+                    const filePath = String(request.params.arguments?.file_path || "").trim();
+                    const outputFormat = String(request.params.arguments?.output_format || "jsonl").trim().toLowerCase();
+                    const outputDirRaw = request.params.arguments?.output_dir ? String(request.params.arguments.output_dir).trim() : "";
+                    const flattenMetadataJson = request.params.arguments?.flatten_metadata_json !== false;
+                    const maxKeys = Number(request.params.arguments?.max_keys ?? 200);
+                    const extrasMode = String(request.params.arguments?.extras_mode || "blob").trim().toLowerCase();
+                    if (!filePath) {
+                        throw new McpError(ErrorCode.InvalidParams, "file_path is required");
+                    }
+                    if (!["jsonl", "json"].includes(outputFormat)) {
+                        throw new McpError(ErrorCode.InvalidParams, "output_format must be one of: jsonl, json");
+                    }
+                    if (!fs.existsSync(filePath)) {
+                        return { content: [{ type: "text", text: `ERROR: File not found: ${filePath}` }], isError: true };
+                    }
+                    const outDir = outputDirRaw || path.join(dataRoot, "data", "normalized_schema");
                     if (!fs.existsSync(outDir))
                         fs.mkdirSync(outDir, { recursive: true });
-                    const outputPath = path.join(outDir, `fused_${Date.now()}${ext}`);
-                    console.error(`[Fusion] Resolved output directory: ${outDir}`);
-                    const result = await fusionEngine.fuse(resolvedPaths, outputPath, {
-                        strategy,
-                        join_on: joinOn,
-                        how,
-                        dedup,
-                        run_quality_after: runQualityAfter,
-                        leakage_check: leakageCheck,
-                        output_format: outputFormat,
-                        compression: compression,
-                        preview,
-                    });
-                    const nullDelta = result.stats.null_delta;
-                    const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
-                    // Register fused dataset under a generated id so users can export it easily
-                    const fusedId = `fused_${Date.now()}`;
+                    const baseName = path.parse(filePath).name || `normalized_${Date.now()}`;
+                    const outputPath = path.join(outDir, `${baseName}.normalized.${outputFormat}`);
                     try {
-                        upsertRegistry(fusedId, result.output_path, "completed");
+                        const scriptPath = path.join(dataRoot, "python", "normalize_schema_engine.py");
+                        const options = {
+                            flatten_metadata_json: !!flattenMetadataJson,
+                            max_keys: Number.isFinite(maxKeys) ? maxKeys : 200,
+                            extras_mode: ["blob", "drop"].includes(extrasMode) ? extrasMode : "blob",
+                        };
+                        const schemaBefore = await getSchemaSnapshot(filePath);
+                        const result = await runPythonJson(scriptPath, [filePath, outputPath, JSON.stringify(options)]);
+                        const schemaAfter = await getSchemaSnapshot(outputPath);
+                        if (!result.ok) {
+                            return { content: [{ type: "text", text: `ERROR: Schema normalization failed: ${result.error}` }], isError: true };
+                        }
+                        // Register normalized file to make follow-up conversion easier.
+                        try {
+                            const datasetId = path.basename(outputPath, path.extname(outputPath));
+                            upsertRegistry(datasetId, outputPath, "completed");
+                        }
+                        catch (e) {
+                            console.error(`[NormalizeSchema] Registry write failed: ${e?.message || e}`);
+                        }
+                        const lineage = appendLineageVersion({
+                            datasetIdBase: path.basename(outputPath, path.extname(outputPath)),
+                            tool: "vesper_normalize_schema",
+                            requestArgs: request.params.arguments,
+                            outputPath,
+                            output: {
+                                rows: result.rows,
+                                columns: result.columns,
+                                format: outputFormat,
+                                schema_before: schemaBefore,
+                                schema_after: schemaAfter,
+                            },
+                            steps: [
+                                { step: "schema_normalized", at: new Date().toISOString(), params: options, metrics: { flattened_keys: result.flattened_keys } },
+                            ],
+                        });
+                        try {
+                            upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                        }
+                        catch { }
+                        let msg = `**Schema normalization complete**\n`;
+                        msg += `- **Input**: ${filePath}\n`;
+                        msg += `- **Output**: ${result.output_path}\n`;
+                        msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                        msg += `- **Lineage**: ${lineage.lineagePath}\n`;
+                        msg += `- **Rows**: ${result.rows?.toLocaleString?.() ?? result.rows}\n`;
+                        msg += `- **Columns**: ${result.columns}\n`;
+                        msg += `- **Flattened keys**: ${result.flattened_keys}\n`;
+                        msg += `- **Extras mode**: ${result.extras_mode}\n`;
+                        if (result.extras_rows !== undefined)
+                            msg += `- **Rows with extras**: ${result.extras_rows}\n`;
+                        msg += `\nNext: run \`vesper_convert_format\` on the output to convert to parquet.\n`;
+                        return { content: [{ type: "text", text: msg }] };
                     }
-                    catch (e) {
-                        console.error(`[Registry] Failed to register fused dataset ${fusedId}: ${e?.message || e}`);
+                    catch (error) {
+                        return { content: [{ type: "text", text: `ERROR: Schema normalization failed: ${error.message}` }], isError: true };
                     }
-                    const inputSchemaSnapshots = await Promise.all(resolvedPaths.map((p) => getSchemaSnapshot(p)));
-                    const schemaBefore = mergeSchemaSnapshots(inputSchemaSnapshots);
-                    const schemaAfter = await getSchemaSnapshot(result.output_path);
-                    const lineage = appendLineageVersion({
-                        datasetIdBase: fusedId,
-                        tool: "fuse_datasets",
-                        requestArgs: request.params.arguments,
-                        outputPath: result.output_path,
-                        output: {
-                            rows: result.stats.rows_after,
-                            format: outputFormat,
-                            schema_before: schemaBefore,
-                            schema_after: schemaAfter,
-                        },
-                        sources: resolvedPaths.map((p) => ({ source: "local", url: p, at: new Date().toISOString() })),
-                        steps: [
-                            { step: "fused", at: new Date().toISOString(), params: { strategy, dedup, how }, metrics: { rows_before: result.stats.rows_before, rows_after: result.stats.rows_after, duplicates_removed: result.stats.duplicates_removed } },
-                        ],
-                    });
-                    try {
-                        upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                }
+                case "fuse":
+                case "fuse_datasets": {
+                    const operation = request.params.name === "fuse_datasets"
+                        ? "tabular"
+                        : String(request.params.arguments?.operation || "tabular").toLowerCase();
+                    if (operation === "web") {
+                        hydrateExternalKeys();
+                        const webSources = Array.isArray(request.params.arguments?.sources)
+                            ? request.params.arguments?.sources
+                            : undefined;
+                        if (!webSources || !Array.isArray(webSources)) {
+                            return {
+                                content: [{ type: "text", text: "ERROR: fuse(operation='web') requires 'sources' array." }],
+                                isError: true,
+                            };
+                        }
+                        const mergeStrategyRaw = request.params.arguments?.merge_strategy
+                            ? String(request.params.arguments?.merge_strategy).toLowerCase()
+                            : undefined;
+                        const dedupRaw = request.params.arguments?.deduplication
+                            ? String(request.params.arguments?.deduplication).toLowerCase()
+                            : undefined;
+                        const merge_strategy = mergeStrategyRaw && ["union", "dedup"].includes(mergeStrategyRaw)
+                            ? mergeStrategyRaw
+                            : undefined;
+                        const deduplication = dedupRaw && ["semantic", "exact", "none"].includes(dedupRaw)
+                            ? dedupRaw
+                            : undefined;
+                        const webResult = await webFusionEngine.fuse({
+                            sources: webSources.map((s) => ({
+                                type: String(s?.type || "").trim().toLowerCase(),
+                                query: String(s?.query || "").trim(),
+                                max_results: s?.max_results !== undefined ? Number(s.max_results) : undefined,
+                                min_stars: s?.min_stars !== undefined ? Number(s.min_stars) : undefined,
+                                bucket: s?.bucket !== undefined ? String(s.bucket) : undefined,
+                                path: s?.path !== undefined ? String(s.path) : undefined,
+                                region: s?.region !== undefined ? String(s.region) : undefined,
+                                credentials: s?.credentials ? {
+                                    accessKeyId: s.credentials.accessKeyId !== undefined ? String(s.credentials.accessKeyId) : undefined,
+                                    secretAccessKey: s.credentials.secretAccessKey !== undefined ? String(s.credentials.secretAccessKey) : undefined,
+                                    sessionToken: s.credentials.sessionToken !== undefined ? String(s.credentials.sessionToken) : undefined,
+                                    roleArn: s.credentials.roleArn !== undefined ? String(s.credentials.roleArn) : undefined,
+                                } : undefined,
+                            })),
+                            merge_strategy,
+                            deduplication,
+                        });
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(webResult, null, 2) }],
+                        };
+                    }
+                    const rawSources = request.params.arguments?.sources;
+                    if (!rawSources || !Array.isArray(rawSources) || rawSources.length < 2) {
+                        throw new McpError(ErrorCode.InvalidParams, "sources must contain at least 2 dataset IDs/paths");
                     }
-                    catch { }
-                    let msg = `Fused ${result.stats.sources_count} sources -> ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
-                    msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
-                    msg += `- Null change: ${nullText}\n`;
-                    msg += `- Output: ${result.output_path}\n`;
-                    msg += `- Version: ${lineage.datasetVersionId}\n`;
-                    msg += `- Lineage: ${lineage.lineagePath}\n`;
-                    if (result.preview_path)
-                        msg += `- Preview: ${result.preview_path}\n`;
-                    if (result.leakage_report) {
-                        msg += `- Leakage: ${result.leakage_report.leakage_detected ? "detected" : "none"}`;
-                        if (result.leakage_report.leakage_count) {
-                            msg += ` (${result.leakage_report.leakage_count})`;
+                    const strategy = request.params.arguments?.strategy || "concat";
+                    const joinOn = request.params.arguments?.join_on;
+                    const how = request.params.arguments?.how || "inner";
+                    const dedup = request.params.arguments?.dedup !== false;
+                    const runQualityAfter = request.params.arguments?.run_quality_after !== false;
+                    const leakageCheck = request.params.arguments?.leakage_check !== false;
+                    const outputFormat = request.params.arguments?.output_format || "feather";
+                    const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
+                    const preview = request.params.arguments?.preview !== false;
+                    const resolvedPaths = [];
+                    const unresolved = [];
+                    for (const src of rawSources) {
+                        if (fs.existsSync(src)) {
+                            resolvedPaths.push(src);
+                            continue;
                         }
-                        msg += "\n";
+                        const status = metadataStore.getDownloadStatus(src);
+                        if (status?.local_path && fs.existsSync(status.local_path)) {
+                            resolvedPaths.push(status.local_path);
+                            continue;
+                        }
+                        unresolved.push(src);
+                    }
+                    if (unresolved.length > 0) {
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `ERROR: Could not resolve these sources to local files: ${unresolved.join(", ")}. Provide local paths or run prepare_dataset first.`
+                                }],
+                            isError: true
+                        };
+                    }
+                    try {
+                        const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow" };
+                        const ext = extMap[outputFormat] || ".feather";
+                        const outDir = process.cwd();
+                        if (!fs.existsSync(outDir))
+                            fs.mkdirSync(outDir, { recursive: true });
+                        const outputPath = path.join(outDir, `fused_${Date.now()}${ext}`);
+                        console.error(`[Fusion] Resolved output directory: ${outDir}`);
+                        const result = await fusionEngine.fuse(resolvedPaths, outputPath, {
+                            strategy,
+                            join_on: joinOn,
+                            how,
+                            dedup,
+                            run_quality_after: runQualityAfter,
+                            leakage_check: leakageCheck,
+                            output_format: outputFormat,
+                            compression: compression,
+                            preview,
+                        });
+                        const nullDelta = result.stats.null_delta;
+                        const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
+                        // Register fused dataset under a generated id so users can export it easily
+                        const fusedId = `fused_${Date.now()}`;
+                        try {
+                            upsertRegistry(fusedId, result.output_path, "completed");
+                        }
+                        catch (e) {
+                            console.error(`[Registry] Failed to register fused dataset ${fusedId}: ${e?.message || e}`);
+                        }
+                        const inputSchemaSnapshots = await Promise.all(resolvedPaths.map((p) => getSchemaSnapshot(p)));
+                        const schemaBefore = mergeSchemaSnapshots(inputSchemaSnapshots);
+                        const schemaAfter = await getSchemaSnapshot(result.output_path);
+                        const lineage = appendLineageVersion({
+                            datasetIdBase: fusedId,
+                            tool: "fuse_datasets",
+                            requestArgs: request.params.arguments,
+                            outputPath: result.output_path,
+                            output: {
+                                rows: result.stats.rows_after,
+                                format: outputFormat,
+                                schema_before: schemaBefore,
+                                schema_after: schemaAfter,
+                            },
+                            sources: resolvedPaths.map((p) => ({ source: "local", url: p, at: new Date().toISOString() })),
+                            steps: [
+                                { step: "fused", at: new Date().toISOString(), params: { strategy, dedup, how }, metrics: { rows_before: result.stats.rows_before, rows_after: result.stats.rows_after, duplicates_removed: result.stats.duplicates_removed } },
+                            ],
+                        });
+                        try {
+                            upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                        }
+                        catch { }
+                        let msg = `Fused ${result.stats.sources_count} sources -> ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
+                        msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
+                        msg += `- Null change: ${nullText}\n`;
+                        msg += `- Output: ${result.output_path}\n`;
+                        msg += `- Version: ${lineage.datasetVersionId}\n`;
+                        msg += `- Lineage: ${lineage.lineagePath}\n`;
+                        if (result.preview_path)
+                            msg += `- Preview: ${result.preview_path}\n`;
+                        if (result.leakage_report) {
+                            msg += `- Leakage: ${result.leakage_report.leakage_detected ? "detected" : "none"}`;
+                            if (result.leakage_report.leakage_count) {
+                                msg += ` (${result.leakage_report.leakage_count})`;
+                            }
+                            msg += "\n";
+                        }
+                        msg += `\nNext: run split_dataset/export_dataset on fused output. Registered fused dataset id: ${fusedId}`;
+                        return { content: [{ type: "text", text: msg }] };
+                    }
+                    catch (error) {
+                        return {
+                            content: [{ type: "text", text: `ERROR: Fusion failed: ${error.message}` }],
+                            isError: true
+                        };
                     }
-                    msg += `\nNext: run split_dataset/export_dataset on fused output. Registered fused dataset id: ${fusedId}`;
-                    return { content: [{ type: "text", text: msg }] };
-                }
-                catch (error) {
-                    return {
-                        content: [{ type: "text", text: `ERROR: Fusion failed: ${error.message}` }],
-                        isError: true
-                    };
                 }
+                default:
+                    throw new McpError(ErrorCode.MethodNotFound, "Tool not found");
             }
-            default:
-                throw new McpError(ErrorCode.MethodNotFound, "Tool not found");
-        }
+        })();
+        void recordMcpToolAnalyticsAfterCall({
+            toolName: String(request.params.name),
+            args: request.params.arguments,
+            result: toolResponse,
+        }).catch((err) => console.error("[mcp-analytics]", err));
+        return toolResponse;
     }); // end requestQueue.enqueue
 });
 async function main() {