@vespermcp/mcp-server 1.2.27 → 1.2.29

package/README.md

@@ -88,6 +88,47 @@ Vesper attempts to auto-configure itself! Restart Claude and check. If not:
 
 - `KAGGLE_USERNAME` & `KAGGLE_KEY`: For Kaggle dataset access
 - `HF_TOKEN`: For private HuggingFace datasets
+- `VESPER_TELEMETRY_ENDPOINT`: Optional HTTP endpoint for lineage telemetry events (`lineage.version.appended`)
+
+### Telemetry Transparency (Opt-in)
+
+Vesper does **not** send telemetry unless `VESPER_TELEMETRY_ENDPOINT` is explicitly set.
+
+When enabled, Vesper sends only lineage event metadata on version append:
+- dataset base/version IDs
+- tool name + actor metadata (`agent_id`, `pipeline_id` when provided)
+- basic output metadata (`local_path`, rows/columns, format)
+- timestamp + host name
+
+It does **not** upload dataset file contents.
+
+### Lineage Receiver (for web dashboard backend)
+
+Vesper includes a tiny ingestion server for lineage telemetry events:
+
+```bash
+npm run telemetry:receiver
+```
+
+Storage backends:
+- **Postgres**: set `DATABASE_URL`
+- **SQLite**: set `SQLITE_PATH` (for lightweight/local deployments)
+
+Optional env vars:
+- `PORT` (default `8787`)
+- `LINEAGE_INGEST_PATH` (default `/vesper/lineage`)
+
+Example for hosted backend:
+- ingest URL: `https://getvesper.dev/vesper/lineage`
+- client env: `VESPER_TELEMETRY_ENDPOINT=https://getvesper.dev/vesper/lineage`
+
+DDL files:
+- `telemetry/sql/lineage_events.postgres.sql`
+- `telemetry/sql/lineage_events.sqlite.sql`
+
+Stats endpoint for web dashboard bootstrap:
+- `GET /vesper/lineage/stats?days=30`
+- Returns JSON: overview, by-tool counts, by-day counts, top datasets, recent activity.
 
 ### Optional Kaggle Setup (Not Required)
 
@@ -121,6 +162,8 @@ vespermcp discover --source kaggle "credit risk" --limit 10
 vespermcp discover --source huggingface "credit risk" --limit 10
 vespermcp download kaggle username/dataset-name
 vespermcp download kaggle https://www.kaggle.com/datasets/username/dataset-name --target-dir ./data
+vespermcp status
+vespermcp status --dir ./some/project --max-depth 3
 ```
 
 ## 🚀 Quick Start

package/build/index.js

@@ -74,18 +74,156 @@ function getRegistryEntry(dataset_id) {
     console.error(`[Registry] Lookup keys: ${aliases.join(", ")}`);
     return readRegistry().find(e => aliases.includes((e.dataset_id || e.id)));
 }
+function getLineageDir() {
+    const p = path.join(dataRoot, "lineage");
+    if (!fs.existsSync(p))
+        fs.mkdirSync(p, { recursive: true });
+    return p;
+}
+function toBaseDatasetId(datasetId) {
+    const safe = normalize_dataset_id(String(datasetId || "").trim());
+    return safe.replace(/_v\d+$/i, "");
+}
+function getLineageRecordPath(datasetIdBase) {
+    return path.join(getLineageDir(), `${toBaseDatasetId(datasetIdBase)}.lineage.json`);
+}
+function readLineageRecord(datasetIdBase) {
+    const p = getLineageRecordPath(datasetIdBase);
+    if (!fs.existsSync(p)) {
+        return {
+            dataset_id_base: toBaseDatasetId(datasetIdBase),
+            latest_version: 0,
+            updated_at: new Date().toISOString(),
+            versions: [],
+        };
+    }
+    try {
+        return JSON.parse(fs.readFileSync(p, "utf-8"));
+    }
+    catch {
+        return {
+            dataset_id_base: toBaseDatasetId(datasetIdBase),
+            latest_version: 0,
+            updated_at: new Date().toISOString(),
+            versions: [],
+        };
+    }
+}
+function writeLineageRecord(record) {
+    const p = getLineageRecordPath(record.dataset_id_base);
+    fs.writeFileSync(p, JSON.stringify(record, null, 2));
+}
+function appendLineageVersion(input) {
+    const base = toBaseDatasetId(input.datasetIdBase);
+    const record = readLineageRecord(base);
+    if (input.outputPath) {
+        const existing = record.versions.find(v => v.output?.local_path === input.outputPath);
+        if (existing) {
+            return { datasetVersionId: existing.dataset_id, version: existing.version, lineagePath: getLineageRecordPath(base) };
+        }
+    }
+    const version = (record.latest_version || 0) + 1;
+    const datasetVersionId = `${base}_v${version}`;
+    const now = new Date().toISOString();
+    const outputPath = input.outputPath;
+    let sidecarPath;
+    if (outputPath && fs.existsSync(outputPath)) {
+        sidecarPath = `${outputPath}.lineage.json`;
+    }
+    const v = {
+        version,
+        dataset_id: datasetVersionId,
+        created_at: now,
+        triggered_by: {
+            tool: input.tool,
+            agent_id: input.requestArgs?.agent_id ? String(input.requestArgs.agent_id) : undefined,
+            pipeline_id: input.requestArgs?.pipeline_id ? String(input.requestArgs.pipeline_id) : undefined,
+            api_key: input.requestArgs?.api_key ? String(input.requestArgs.api_key) : undefined,
+        },
+        input: {
+            dataset_id: input.requestArgs?.dataset_id ? String(input.requestArgs.dataset_id) : undefined,
+            query: input.requestArgs?.query ? String(input.requestArgs.query) : undefined,
+            source_path: input.requestArgs?.file_path ? String(input.requestArgs.file_path) : undefined,
+            source_urls: Array.isArray(input.requestArgs?.source_urls)
+                ? input.requestArgs.source_urls.map((u) => String(u))
+                : undefined,
+        },
+        output: {
+            local_path: outputPath,
+            rows: typeof input.output?.rows === "number" ? input.output.rows : undefined,
+            columns: typeof input.output?.columns === "number" ? input.output.columns : undefined,
+            format: typeof input.output?.format === "string" ? input.output.format : undefined,
+            size_mb: typeof input.output?.size_mb === "number" ? input.output.size_mb : undefined,
+            quality_score: typeof input.output?.quality_score === "number" ? input.output.quality_score : undefined,
+            schema_before: input.output?.schema_before && typeof input.output.schema_before === "object"
+                ? {
+                    rows: typeof input.output.schema_before.rows === "number" ? input.output.schema_before.rows : undefined,
+                    columns: Array.isArray(input.output.schema_before.columns) ? input.output.schema_before.columns.map((c) => String(c)) : undefined,
+                    dtypes: input.output.schema_before.dtypes && typeof input.output.schema_before.dtypes === "object"
+                        ? Object.fromEntries(Object.entries(input.output.schema_before.dtypes).map(([k, v]) => [String(k), String(v)]))
+                        : undefined,
+                }
+                : undefined,
+            schema_after: input.output?.schema_after && typeof input.output.schema_after === "object"
+                ? {
+                    rows: typeof input.output.schema_after.rows === "number" ? input.output.schema_after.rows : undefined,
+                    columns: Array.isArray(input.output.schema_after.columns) ? input.output.schema_after.columns.map((c) => String(c)) : undefined,
+                    dtypes: input.output.schema_after.dtypes && typeof input.output.schema_after.dtypes === "object"
+                        ? Object.fromEntries(Object.entries(input.output.schema_after.dtypes).map(([k, v]) => [String(k), String(v)]))
+                        : undefined,
+                }
+                : undefined,
+        },
+        sources: input.sources || [],
+        steps: input.steps || [{ step: input.tool, at: now, params: input.requestArgs || {} }],
+    };
+    record.latest_version = version;
+    record.updated_at = now;
+    record.versions.push(v);
+    writeLineageRecord(record);
+    const telemetryEndpoint = process.env.VESPER_TELEMETRY_ENDPOINT?.trim();
+    if (telemetryEndpoint) {
+        postJsonNonBlocking(telemetryEndpoint, {
+            event: "lineage.version.appended",
+            sent_at: now,
+            host: os.hostname(),
+            dataset_id_base: base,
+            version: v.version,
+            dataset_id: v.dataset_id,
+            triggered_by: v.triggered_by,
+            output: {
+                local_path: v.output?.local_path,
+                rows: v.output?.rows,
+                columns: v.output?.columns,
+                format: v.output?.format,
+            },
+        });
+    }
+    if (sidecarPath) {
+        try {
+            fs.writeFileSync(sidecarPath, JSON.stringify(v, null, 2));
+        }
+        catch {
+            // best effort sidecar write
+        }
+    }
+    return { datasetVersionId, version, lineagePath: getLineageRecordPath(base) };
+}
 const STRUCTURED_FILE_EXTENSIONS = [".parquet", ".csv", ".jsonl", ".json", ".feather", ".arrow", ".tsv", ".txt"];
 const IMAGE_FILE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tiff", ".tif", ".svg"]);
-function walkFilesRecursive(rootDir) {
+function walkFilesRecursive(rootDir, maxDepth = Number.POSITIVE_INFINITY) {
     const out = [];
-    const stack = [rootDir];
+    const stack = [{ dir: rootDir, depth: 0 }];
     while (stack.length > 0) {
-        const currentDir = stack.pop();
+        const current = stack.pop();
+        const currentDir = current.dir;
         const entries = fs.readdirSync(currentDir, { withFileTypes: true });
         for (const entry of entries) {
             const fullPath = path.join(currentDir, entry.name);
             if (entry.isDirectory()) {
-                stack.push(fullPath);
+                if (current.depth < maxDepth) {
+                    stack.push({ dir: fullPath, depth: current.depth + 1 });
+                }
             }
             else if (entry.isFile()) {
                 out.push(fullPath);
@@ -201,6 +339,36 @@ function ensureLocalPipelineSource(sourcePath, datasetId, targetDir) {
     }
     return stagedPath;
 }
+function cleanupIntermediateArtifacts(artifactPaths, finalOutputPath) {
+    const finalResolved = path.resolve(finalOutputPath);
+    const finalLineage = `${finalResolved}.lineage.json`;
+    for (const candidate of artifactPaths) {
+        if (!candidate)
+            continue;
+        const resolved = path.resolve(candidate);
+        if (resolved === finalResolved || resolved === finalLineage)
+            continue;
+        try {
+            if (fs.existsSync(resolved) && fs.statSync(resolved).isFile()) {
+                fs.unlinkSync(resolved);
+            }
+        }
+        catch {
+            // Best-effort cleanup.
+        }
+        const sidecar = `${resolved}.lineage.json`;
+        if (sidecar === finalLineage)
+            continue;
+        try {
+            if (fs.existsSync(sidecar) && fs.statSync(sidecar).isFile()) {
+                fs.unlinkSync(sidecar);
+            }
+        }
+        catch {
+            // Best-effort cleanup.
+        }
+    }
+}
 function resolveDatasetLocalPath(datasetIdOrPath, preferredDirs = []) {
     if (fs.existsSync(datasetIdOrPath)) {
         return ensureExportableLocalPath(datasetIdOrPath);
@@ -296,6 +464,8 @@ import { QualityOrchestrator } from "./quality/quality-orchestrator.js";
 import { ConfigManager } from "./config/config-manager.js";
 import { SecureKeysManager } from "./config/secure-keys.js";
 import readline from "readline";
+import http from "http";
+import https from "https";
 import os from "os";
 // Determine absolute paths relative to the compiled script
 const __filename = fileURLToPath(import.meta.url);
@@ -321,6 +491,34 @@ function logError(err, context) {
     fs.appendFileSync(errorLogPath, msg);
     console.error(`[Vesper] Critical error logged to ${errorLogPath}`);
 }
+function postJsonNonBlocking(urlRaw, body) {
+    try {
+        const u = new URL(urlRaw);
+        const payload = JSON.stringify(body);
+        const isHttps = u.protocol === "https:";
+        const transport = isHttps ? https : http;
+        const req = transport.request({
+            method: "POST",
+            hostname: u.hostname,
+            port: u.port ? Number(u.port) : (isHttps ? 443 : 80),
+            path: `${u.pathname}${u.search || ""}`,
+            headers: {
+                "content-type": "application/json",
+                "content-length": Buffer.byteLength(payload),
+            },
+            timeout: 3000,
+        }, (res) => {
+            res.resume();
+        });
+        req.on("error", () => { });
+        req.on("timeout", () => req.destroy());
+        req.write(payload);
+        req.end();
+    }
+    catch {
+        // best effort telemetry only
+    }
+}
 // --- Request Queue: serialize all MCP tool calls to prevent crashes ---
 class RequestQueue {
     queue = [];
@@ -535,6 +733,102 @@ function runPythonJson(scriptPath, args) {
         });
     });
 }
+async function getSchemaSnapshot(filePath) {
+    try {
+        if (!filePath || !fs.existsSync(filePath))
+            return undefined;
+        await ensurePythonModules([{ module: "polars", packageName: "polars" }]);
+        const pyCode = [
+            "import json, os, polars as pl",
+            "p = __import__('sys').argv[1]",
+            "ext = os.path.splitext(p)[1].lower()",
+            "if ext == '.csv': df = pl.read_csv(p, infer_schema_length=10000, ignore_errors=True)",
+            "elif ext in ('.jsonl', '.ndjson'): df = pl.read_ndjson(p)",
+            "elif ext == '.json': df = pl.read_json(p)",
+            "elif ext in ('.parquet', '.pq'): df = pl.read_parquet(p)",
+            "elif ext in ('.feather', '.ftr', '.arrow', '.ipc'): df = pl.read_ipc(p)",
+            "else: df = pl.read_csv(p, infer_schema_length=10000, ignore_errors=True)",
+            "print(json.dumps({'rows': int(df.height), 'columns': [str(c) for c in df.columns], 'dtypes': {str(c): str(t) for c,t in zip(df.columns, df.dtypes)}}))",
+        ].join(";");
+        const proc = await runPythonProcess(["-c", pyCode, filePath], 120000);
+        if (proc.code !== 0)
+            return undefined;
+        return JSON.parse((proc.stdout || "{}").trim());
+    }
+    catch {
+        return undefined;
+    }
+}
+function mergeSchemaSnapshots(snapshots) {
+    const valid = snapshots.filter(Boolean);
+    if (valid.length === 0)
+        return undefined;
+    let rows = 0;
+    const colSet = new Set();
+    const dtypeMulti = {};
+    for (const s of valid) {
+        if (typeof s.rows === "number") {
+            rows = (rows ?? 0) + s.rows;
+        }
+        else {
+            rows = undefined;
+        }
+        for (const c of s.columns || [])
+            colSet.add(String(c));
+        for (const [k, v] of Object.entries(s.dtypes || {})) {
+            if (!dtypeMulti[k])
+                dtypeMulti[k] = new Set();
+            dtypeMulti[k].add(String(v));
+        }
+    }
+    const dtypes = {};
+    for (const [k, values] of Object.entries(dtypeMulti)) {
+        const list = Array.from(values);
+        dtypes[k] = list.length <= 1 ? list[0] : `mixed(${list.join("|")})`;
+    }
+    return {
+        rows,
+        columns: Array.from(colSet).sort(),
+        dtypes,
+    };
+}
+function diffSchemaMaps(fromColumns, toColumns, fromDtypes, toDtypes) {
+    const fromSet = new Set(fromColumns);
+    const toSet = new Set(toColumns);
+    const added_columns = toColumns.filter((c) => !fromSet.has(c));
+    const removed_columns = fromColumns.filter((c) => !toSet.has(c));
+    const common = fromColumns.filter((c) => toSet.has(c));
+    const changed_dtypes = common
+        .filter((c) => String(fromDtypes[c] || "") !== String(toDtypes[c] || ""))
+        .map((c) => ({ column: c, from: fromDtypes[c], to: toDtypes[c] }));
+    return { added_columns, removed_columns, changed_dtypes };
+}
+function isLineageRecordShape(value) {
+    return !!value && typeof value === "object" && typeof value.dataset_id_base === "string" && Array.isArray(value.versions);
+}
+function parseErrorLogLines(filePath, withinDays) {
+    try {
+        if (!fs.existsSync(filePath))
+            return [];
+        const text = fs.readFileSync(filePath, "utf-8");
+        const lines = text.split(/\r?\n/).filter(Boolean);
+        const minTs = Date.now() - withinDays * 24 * 60 * 60 * 1000;
+        const out = [];
+        for (const line of lines) {
+            const m = line.match(/^\[(.+?)\]\s+ERROR\s+in\s+(.+)$/);
+            if (!m)
+                continue;
+            const ts = Date.parse(m[1]);
+            if (Number.isFinite(ts) && ts >= minTs) {
+                out.push({ at: m[1], message: m[2] });
+            }
+        }
+        return out.slice(-20);
+    }
+    catch {
+        return [];
+    }
+}
 async function countRows(filePath) {
     const scriptPath = path.join(dataRoot, "python", "row_count.py");
     const result = await runPythonJson(scriptPath, [filePath]);
@@ -1166,6 +1460,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             type: "boolean",
                             description: "When true, operation='providers' includes connectors that are scaffolded but not currently configured.",
                         },
+                        agent_id: {
+                            type: "string",
+                            description: "Strongly recommended: caller agent identity for lineage/audit.",
+                        },
+                        pipeline_id: {
+                            type: "string",
+                            description: "Strongly recommended: workflow/pipeline identifier for lineage/audit.",
+                        },
                     },
                     required: ["operation"],
                 },
@@ -1185,6 +1487,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                         limit: { type: "number", description: "Max documents to return (default 10, max 50)." },
                         arxiv_full_text: { type: "boolean", description: "When true, fetch and parse ArXiv PDFs and return full text as document content (slower)." },
                         github_include_readme: { type: "boolean", description: "When true, fetch and include GitHub README.md text as document content (slower)." },
+                        agent_id: { type: "string", description: "Strongly recommended: caller agent identity for lineage/audit." },
+                        pipeline_id: { type: "string", description: "Strongly recommended: workflow/pipeline identifier for lineage/audit." },
                     },
                     required: ["query"],
                 },
@@ -1232,6 +1536,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             enum: ["semantic", "exact", "none"],
                             description: "How to deduplicate across sources.",
                         },
+                        agent_id: { type: "string", description: "Strongly recommended: caller agent identity for lineage/audit." },
+                        pipeline_id: { type: "string", description: "Strongly recommended: workflow/pipeline identifier for lineage/audit." },
                     },
                     required: ["sources"],
                 },
@@ -1435,6 +1741,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                         download_images: { type: "boolean", description: "When true, enables post-prepare smart asset downloading for image/media datasets." },
                         cleaning_options: { type: "object" },
                         split_config: { type: "object" },
+                        agent_id: { type: "string", description: "Strongly recommended: caller agent identity for lineage/audit." },
+                        pipeline_id: { type: "string", description: "Strongly recommended: workflow/pipeline identifier for lineage/audit." },
                     },
                     required: ["query"],
                 },
@@ -1509,6 +1817,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             items: { type: "string" },
                             description: "Export only these columns (faster for wide datasets).",
                         },
+                        agent_id: {
+                            type: "string",
+                            description: "Strongly recommended: caller agent identity for lineage/audit.",
+                        },
+                        pipeline_id: {
+                            type: "string",
+                            description: "Strongly recommended: workflow/pipeline identifier for lineage/audit.",
+                        },
                     },
                     required: ["dataset_id"],
                 },
@@ -1521,6 +1837,42 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                     properties: {},
                 },
             },
+            {
+                name: "get_lineage",
+                description: "Get version history and full lineage/provenance for a dataset (sources, steps, inputs/outputs, trigger metadata).",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        dataset_id: {
+                            type: "string",
+                            description: "Dataset ID (base or versioned, e.g. my_dataset or my_dataset_v2).",
+                        },
+                    },
+                    required: ["dataset_id"],
+                },
+            },
+            {
+                name: "diff_lineage_versions",
+                description: "Diff two lineage versions for one dataset and return structured changes (schema, rows, steps, actor identity).",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        dataset_id: {
+                            type: "string",
+                            description: "Dataset ID (base or versioned).",
+                        },
+                        from_version: {
+                            type: "number",
+                            description: "Source lineage version number (e.g., 1).",
+                        },
+                        to_version: {
+                            type: "number",
+                            description: "Target lineage version number (e.g., 2).",
+                        },
+                    },
+                    required: ["dataset_id", "from_version", "to_version"],
+                },
+            },
             {
                 name: "vesper_convert_format",
                 description: "Convert a dataset file between formats (CSV, Parquet, JSON, JSONL). Auto-detects input format from extension. Saves output in the same directory with the new extension and registers it in the Vesper registry.",
@@ -1536,6 +1888,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             enum: ["csv", "parquet", "json", "jsonl"],
                             description: "The desired output format.",
                         },
+                        agent_id: {
+                            type: "string",
+                            description: "Strongly recommended: caller agent identity for lineage/audit.",
+                        },
+                        pipeline_id: {
+                            type: "string",
+                            description: "Strongly recommended: workflow/pipeline identifier for lineage/audit.",
+                        },
                     },
                     required: ["file_path", "target_format"],
                 },
@@ -1572,6 +1932,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
                             enum: ["blob", "drop"],
                             description: "How to handle metadata_json keys beyond max_keys. blob keeps them in metadata_json_blob; drop discards them. Default: blob.",
                         },
+                        agent_id: {
+                            type: "string",
+                            description: "Strongly recommended: caller agent identity for lineage/audit.",
+                        },
+                        pipeline_id: {
+                            type: "string",
+                            description: "Strongly recommended: workflow/pipeline identifier for lineage/audit.",
+                        },
                     },
                     required: ["file_path"],
                 },
@@ -1761,6 +2129,30 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         arxiv_full_text: request.params.arguments?.arxiv_full_text === true,
                         github_include_readme: request.params.arguments?.github_include_readme === true,
                     });
+                    try {
+                        appendLineageVersion({
+                            datasetIdBase: `webfind_${query || "query"}`,
+                            tool: "vesper_web_find",
+                            requestArgs: request.params.arguments,
+                            output: {
+                                rows: Array.isArray(result.results) ? result.results.length : undefined,
+                            },
+                            sources: Array.isArray(result.results)
+                                ? result.results.slice(0, 200).map((r) => ({
+                                    source: String(r?.source_type || "unknown"),
+                                    url: typeof r?.source_url === "string" ? r.source_url : undefined,
+                                    at: typeof r?.collected_at === "string" ? r.collected_at : undefined,
+                                }))
+                                : [],
+                            steps: [
+                                { step: "web_find_discover", at: new Date().toISOString(), params: { query, sources, limit } },
+                                { step: "web_find_complete", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
+                            ],
+                        });
+                    }
+                    catch (e) {
+                        console.error(`[Lineage] vesper_web_find append failed: ${e?.message || e}`);
+                    }
                     return {
                         content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
                     };
@@ -1882,6 +2274,30 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                             limit: Number(request.params.arguments?.limit || 10),
                             publicOnly,
                         });
+                        try {
+                            appendLineageVersion({
+                                datasetIdBase: `discover_${source}_${query || "query"}`,
+                                tool: "unified_dataset_api.discover",
+                                requestArgs: request.params.arguments,
+                                output: { rows: Array.isArray(result.results) ? result.results.length : undefined },
+                                sources: Array.isArray(result.results)
+                                    ? result.results.slice(0, 200).map((r) => ({
+                                        source: String(r?.source || source || "unknown"),
+                                        url: typeof r?.download_url === "string"
+                                            ? r.download_url
+                                            : (typeof r?.metadata_url === "string" ? r.metadata_url : undefined),
+                                        at: new Date().toISOString(),
+                                    }))
+                                    : [],
+                                steps: [
+                                    { step: "discover_requested", at: new Date().toISOString(), params: { query, source, limit: Number(request.params.arguments?.limit || 10), publicOnly } },
+                                    { step: "discover_completed", at: new Date().toISOString(), metrics: { result_count: Array.isArray(result.results) ? result.results.length : 0 } },
+                                ],
+                            });
+                        }
+                        catch (e) {
+                            console.error(`[Lineage] unified discover append failed: ${e?.message || e}`);
+                        }
                         return {
                             content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
                         };
@@ -1914,6 +2330,36 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         catch (e) {
                             console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
                         }
+                        try {
+                            const schemaAfter = await getSchemaSnapshot(result.copied_to || result.local_path);
+                            const lineage = appendLineageVersion({
+                                datasetIdBase: result.dataset_id,
+                                tool: "unified_dataset_api.download",
+                                requestArgs: request.params.arguments,
+                                outputPath: result.copied_to || result.local_path,
+                                output: {
+                                    local_path: result.copied_to || result.local_path,
+                                    format: path.extname(result.copied_to || result.local_path).replace(".", ""),
+                                    schema_after: schemaAfter,
+                                },
+                                sources: [{
+                                        source: source,
+                                        url: typeof result.dataset_id === "string" ? result.dataset_id : undefined,
+                                        at: new Date().toISOString(),
+                                    }],
+                                steps: [
+                                    { step: "download_requested", at: new Date().toISOString(), params: { datasetId, source, targetDir } },
+                                    { step: "download_completed", at: new Date().toISOString(), metrics: { local_path: result.copied_to || result.local_path } },
+                                ],
+                            });
+                            try {
+                                upsertRegistry(lineage.datasetVersionId, result.copied_to || result.local_path, "completed");
+                            }
+                            catch { }
+                        }
+                        catch (e) {
+                            console.error(`[Lineage] unified download append failed: ${e?.message || e}`);
+                        }
                         return {
                             content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
                         };
@@ -2449,6 +2895,32 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     };
                 }
                 jobStatusLastPoll[jobId] = now;
+                if (job.status === "completed") {
+                    try {
+                        const meta = job.metadata ? JSON.parse(job.metadata) : {};
+                        const baseId = String(meta?.datasetId || meta?.dataset_id || meta?.query || job.id);
+                        const outPath = typeof job.result_url === "string" ? job.result_url : undefined;
+                        appendLineageVersion({
+                            datasetIdBase: baseId,
+                            tool: `job:${job.type}`,
+                            requestArgs: {
+                                dataset_id: meta?.datasetId || meta?.dataset_id,
+                                query: meta?.query,
+                                pipeline_id: meta?.pipeline_id,
+                                agent_id: meta?.agent_id,
+                            },
+                            outputPath: outPath,
+                            output: {},
+                            steps: [
+                                { step: `${job.type}_started`, at: job.created_at, params: meta || {} },
+                                { step: `${job.type}_completed`, at: job.updated_at || new Date().toISOString(), metrics: { progress: job.progress } },
+                            ],
+                        });
+                    }
+                    catch (e) {
+                        console.error(`[Lineage] check_job_status append failed: ${e?.message || e}`);
+                    }
+                }
                 return {
                     content: [{ type: "text", text: formatJobStatus(job) }]
                 };
@@ -2456,6 +2928,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             case "export_dataset": {
                 const datasetId = String(request.params.arguments?.dataset_id);
                 const isDirectLocalInput = isDirectLocalDatasetReference(datasetId);
+                const intermediateArtifacts = new Set();
                 const requestedTargetDir = request.params.arguments?.target_dir
                     ? String(request.params.arguments?.target_dir).trim()
                     : request.params.arguments?.output_dir
@@ -2525,9 +2998,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     else if (currentExt !== pipelineFmt) {
                         console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
                         try {
+                            const beforeStagingPath = sourcePath;
                             sourcePath = ensureLocalPipelineSource(sourcePath, datasetId, targetDir);
+                            if (path.resolve(beforeStagingPath) !== path.resolve(sourcePath)) {
+                                intermediateArtifacts.add(sourcePath);
+                            }
                             const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
                             if (pipelineResult.final_output_path) {
+                                if (path.resolve(pipelineResult.final_output_path) !== path.resolve(sourcePath)) {
+                                    intermediateArtifacts.add(pipelineResult.final_output_path);
+                                }
                                 sourcePath = pipelineResult.final_output_path;
                                 try {
                                     // Update registry to point to pipeline's final output
@@ -2567,10 +3047,36 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     if (!fs.existsSync(outDir))
                         fs.mkdirSync(outDir, { recursive: true });
                     const outputFile = path.join(outDir, `${safeName}${ext}`);
+                    const schemaBefore = await getSchemaSnapshot(sourcePath);
                     const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
+                    const schemaAfter = await getSchemaSnapshot(result.output_path);
+                    const lineage = appendLineageVersion({
+                        datasetIdBase: datasetId,
+                        tool: "export_dataset",
+                        requestArgs: request.params.arguments,
+                        outputPath: result.output_path,
+                        output: {
+                            rows: result.rows,
+                            columns: result.columns,
+                            format: requestedFormat,
+                            size_mb: result.file_size_mb,
+                            schema_before: schemaBefore,
+                            schema_after: schemaAfter,
+                        },
+                        steps: [
+                            { step: "source_resolved", at: new Date().toISOString(), params: { sourcePath } },
+                            { step: "exported", at: new Date().toISOString(), params: { format: requestedFormat, compression }, metrics: { rows: result.rows, columns: result.columns } },
+                        ],
+                    });
+                    try {
+                        upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                    }
+                    catch { }
                     // Build rich response
                     let msg = `**Export complete**\n`;
                     msg += `- **File**: ${result.output_path}\n`;
+                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
                     msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
                     msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
                     if (result.file_size_mb !== undefined)
@@ -2590,6 +3096,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
                         msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
                     }
+                    cleanupIntermediateArtifacts(intermediateArtifacts, result.output_path);
                     return { content: [{ type: "text", text: msg }] };
                 }
                 catch (error) {
@@ -2616,6 +3123,100 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     content: [{ type: "text", text: `**Vesper Registry** (${entries.length} dataset${entries.length !== 1 ? "s" : ""}):\n\n${lines.join("\n\n")}` }]
                 };
             }
+            case "get_lineage": {
+                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                if (!datasetId) {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                }
+                const base = toBaseDatasetId(datasetId);
+                const record = readLineageRecord(base);
+                if (!record.versions || record.versions.length === 0) {
+                    return {
+                        content: [{ type: "text", text: `No lineage found for '${datasetId}' yet.` }]
+                    };
+                }
+                return {
+                    content: [{ type: "text", text: JSON.stringify(record, null, 2) }]
+                };
+            }
+            case "diff_lineage_versions": {
+                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                const fromVersion = Number(request.params.arguments?.from_version);
+                const toVersion = Number(request.params.arguments?.to_version);
+                if (!datasetId) {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                }
+                if (!Number.isInteger(fromVersion) || fromVersion <= 0) {
+                    throw new McpError(ErrorCode.InvalidParams, "from_version must be a positive integer");
+                }
+                if (!Number.isInteger(toVersion) || toVersion <= 0) {
+                    throw new McpError(ErrorCode.InvalidParams, "to_version must be a positive integer");
+                }
+                const base = toBaseDatasetId(datasetId);
+                const record = readLineageRecord(base);
+                const fromV = record.versions.find((v) => v.version === fromVersion);
+                const toV = record.versions.find((v) => v.version === toVersion);
+                if (!fromV || !toV) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Could not find both versions in lineage for '${datasetId}'.` }],
+                        isError: true,
+                    };
+                }
+                const fromSchema = (toV.output?.schema_before && toVersion > fromVersion)
+                    ? fromV.output?.schema_after || fromV.output?.schema_before || {}
+                    : fromV.output?.schema_after || fromV.output?.schema_before || {};
+                const toSchema = toV.output?.schema_after || toV.output?.schema_before || {};
+                const fromCols = Array.isArray(fromSchema.columns) ? fromSchema.columns.map((c) => String(c)) : [];
+                const toCols = Array.isArray(toSchema.columns) ? toSchema.columns.map((c) => String(c)) : [];
+                const fromDtypes = (fromSchema.dtypes && typeof fromSchema.dtypes === "object") ? fromSchema.dtypes : {};
+                const toDtypes = (toSchema.dtypes && typeof toSchema.dtypes === "object") ? toSchema.dtypes : {};
+                const schemaDiff = diffSchemaMaps(fromCols, toCols, fromDtypes, toDtypes);
+                const fromRows = typeof fromSchema.rows === "number"
+                    ? fromSchema.rows
+                    : (typeof fromV.output?.rows === "number" ? fromV.output.rows : undefined);
+                const toRows = typeof toSchema.rows === "number"
+                    ? toSchema.rows
+                    : (typeof toV.output?.rows === "number" ? toV.output.rows : undefined);
+                const fromSteps = new Set((fromV.steps || []).map((s) => String(s.step)));
+                const toSteps = new Set((toV.steps || []).map((s) => String(s.step)));
+                const addedSteps = Array.from(toSteps).filter((s) => !fromSteps.has(s));
+                const removedSteps = Array.from(fromSteps).filter((s) => !toSteps.has(s));
+                const actorDiff = {
+                    changed: String(fromV.triggered_by?.agent_id || "") !== String(toV.triggered_by?.agent_id || "") ||
+                        String(fromV.triggered_by?.pipeline_id || "") !== String(toV.triggered_by?.pipeline_id || ""),
+                    from: {
+                        tool: fromV.triggered_by?.tool,
+                        agent_id: fromV.triggered_by?.agent_id,
+                        pipeline_id: fromV.triggered_by?.pipeline_id,
+                    },
+                    to: {
+                        tool: toV.triggered_by?.tool,
+                        agent_id: toV.triggered_by?.agent_id,
+                        pipeline_id: toV.triggered_by?.pipeline_id,
+                    },
+                };
+                const diffResult = {
+                    dataset_id_base: base,
+                    from_version: fromVersion,
+                    to_version: toVersion,
+                    schema_diff: schemaDiff,
+                    row_count_delta: {
+                        from: fromRows,
+                        to: toRows,
+                        delta: (typeof fromRows === "number" && typeof toRows === "number") ? (toRows - fromRows) : undefined,
+                    },
+                    steps_diff: {
+                        added: addedSteps,
+                        removed: removedSteps,
+                        from_steps: Array.from(fromSteps),
+                        to_steps: Array.from(toSteps),
+                    },
+                    actor_diff: actorDiff,
+                };
+                return {
+                    content: [{ type: "text", text: JSON.stringify(diffResult, null, 2) }],
+                };
+            }
             case "vesper_convert_format": {
                 const filePath = String(request.params.arguments?.file_path || "").trim();
                 const targetFormat = String(request.params.arguments?.target_format || "").trim().toLowerCase();
@@ -2643,7 +3244,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 try {
                     await ensurePythonModules([{ module: "polars", packageName: "polars" }]);
                     const convertScript = path.join(dataRoot, "python", "convert_engine.py");
+                    const schemaBefore = await getSchemaSnapshot(filePath);
                     const result = await runPythonJson(convertScript, [filePath, outputPath]);
+                    const schemaAfter = await getSchemaSnapshot(outputPath);
                     if (!result.ok) {
                         return {
                             content: [{ type: "text", text: `ERROR: Conversion failed: ${result.error}` }],
@@ -2658,9 +3261,32 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     catch (e) {
                         console.error(`[Convert] Registry write failed: ${e?.message || e}`);
                     }
+                    const lineage = appendLineageVersion({
+                        datasetIdBase: datasetId,
+                        tool: "vesper_convert_format",
+                        requestArgs: request.params.arguments,
+                        outputPath,
+                        output: {
+                            rows: result.rows,
+                            columns: result.columns,
+                            format: targetFormat,
+                            size_mb: result.size_mb,
+                            schema_before: schemaBefore,
+                            schema_after: schemaAfter,
+                        },
+                        steps: [
+                            { step: "converted", at: new Date().toISOString(), params: { from: inputExt, to: outputExt } },
+                        ],
+                    });
+                    try {
+                        upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                    }
+                    catch { }
                     let msg = `**Conversion complete**\n`;
                     msg += `- **Input**: ${filePath} (${inputExt.slice(1)})\n`;
                     msg += `- **Output**: ${result.output_path} (${targetFormat})\n`;
+                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
                     msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
                     if (result.size_mb !== undefined)
                         msg += `- **Size**: ${result.size_mb} MB\n`;
@@ -2701,7 +3327,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         max_keys: Number.isFinite(maxKeys) ? maxKeys : 200,
                         extras_mode: ["blob", "drop"].includes(extrasMode) ? extrasMode : "blob",
                     };
+                    const schemaBefore = await getSchemaSnapshot(filePath);
                     const result = await runPythonJson(scriptPath, [filePath, outputPath, JSON.stringify(options)]);
+                    const schemaAfter = await getSchemaSnapshot(outputPath);
                     if (!result.ok) {
                         return { content: [{ type: "text", text: `ERROR: Schema normalization failed: ${result.error}` }], isError: true };
                     }
@@ -2713,9 +3341,31 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     catch (e) {
                         console.error(`[NormalizeSchema] Registry write failed: ${e?.message || e}`);
                     }
+                    const lineage = appendLineageVersion({
+                        datasetIdBase: path.basename(outputPath, path.extname(outputPath)),
+                        tool: "vesper_normalize_schema",
+                        requestArgs: request.params.arguments,
+                        outputPath,
+                        output: {
+                            rows: result.rows,
+                            columns: result.columns,
+                            format: outputFormat,
+                            schema_before: schemaBefore,
+                            schema_after: schemaAfter,
+                        },
+                        steps: [
+                            { step: "schema_normalized", at: new Date().toISOString(), params: options, metrics: { flattened_keys: result.flattened_keys } },
+                        ],
+                    });
+                    try {
+                        upsertRegistry(lineage.datasetVersionId, outputPath, "completed");
+                    }
+                    catch { }
                     let msg = `**Schema normalization complete**\n`;
                     msg += `- **Input**: ${filePath}\n`;
                     msg += `- **Output**: ${result.output_path}\n`;
+                    msg += `- **Version**: ${lineage.datasetVersionId}\n`;
+                    msg += `- **Lineage**: ${lineage.lineagePath}\n`;
                     msg += `- **Rows**: ${result.rows?.toLocaleString?.() ?? result.rows}\n`;
                     msg += `- **Columns**: ${result.columns}\n`;
                     msg += `- **Flattened keys**: ${result.flattened_keys}\n`;
@@ -2795,10 +3445,35 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     catch (e) {
                         console.error(`[Registry] Failed to register fused dataset ${fusedId}: ${e?.message || e}`);
                     }
+                    const inputSchemaSnapshots = await Promise.all(resolvedPaths.map((p) => getSchemaSnapshot(p)));
+                    const schemaBefore = mergeSchemaSnapshots(inputSchemaSnapshots);
+                    const schemaAfter = await getSchemaSnapshot(result.output_path);
+                    const lineage = appendLineageVersion({
+                        datasetIdBase: fusedId,
+                        tool: "fuse_datasets",
+                        requestArgs: request.params.arguments,
+                        outputPath: result.output_path,
+                        output: {
+                            rows: result.stats.rows_after,
+                            format: outputFormat,
+                            schema_before: schemaBefore,
+                            schema_after: schemaAfter,
+                        },
+                        sources: resolvedPaths.map((p) => ({ source: "local", url: p, at: new Date().toISOString() })),
+                        steps: [
+                            { step: "fused", at: new Date().toISOString(), params: { strategy, dedup, how }, metrics: { rows_before: result.stats.rows_before, rows_after: result.stats.rows_after, duplicates_removed: result.stats.duplicates_removed } },
+                        ],
+                    });
+                    try {
+                        upsertRegistry(lineage.datasetVersionId, result.output_path, "completed");
+                    }
+                    catch { }
                     let msg = `Fused ${result.stats.sources_count} sources -> ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
                     msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
                     msg += `- Null change: ${nullText}\n`;
                     msg += `- Output: ${result.output_path}\n`;
+                    msg += `- Version: ${lineage.datasetVersionId}\n`;
+                    msg += `- Lineage: ${lineage.lineagePath}\n`;
                     if (result.preview_path)
                         msg += `- Preview: ${result.preview_path}\n`;
                     if (result.leakage_report) {
@@ -2966,6 +3641,7 @@ async function main() {
     const isDiscover = args.includes("discover");
     const isDownload = args.includes("download");
     const isExport = args.includes("export");
+    const isStatus = args.includes("status");
     const isConfig = args.includes("config") || args.includes("configure");
     const isSetup = args.includes("--setup") || args.includes("setup");
     const isSilent = args.includes("--silent");
@@ -2992,6 +3668,10 @@ async function main() {
         await runExportCli(args);
         return;
     }
+    if (isStatus) {
+        await runStatusCli(args);
+        return;
+    }
     // If run in explicit setup mode, show setup wizard (do not auto-run on server startup)
     if (isSetup) {
         await runSetupWizard(isSilent);
@@ -3292,6 +3972,7 @@ async function runExportCli(args) {
     const fastMode = args.includes("--fast");
     const preview = args.includes("--preview");
     const isDirectLocalInput = isDirectLocalDatasetReference(datasetId);
+    const intermediateArtifacts = new Set();
     const resolvedTargetDir = path.resolve(targetDir || process.cwd());
     let sourcePath = resolveDatasetLocalPath(datasetId, [resolvedTargetDir, process.cwd()]);
     if (!sourcePath) {
@@ -3313,9 +3994,16 @@ async function runExportCli(args) {
         const pipelineCompatibleInput = currentExt === "csv" || currentExt === "parquet";
         if (pipelineCompatibleInput && currentExt !== pipelineFmt) {
             try {
+                const beforeStagingPath = sourcePath;
                 sourcePath = ensureLocalPipelineSource(sourcePath, datasetId, resolvedTargetDir);
+                if (path.resolve(beforeStagingPath) !== path.resolve(sourcePath)) {
+                    intermediateArtifacts.add(sourcePath);
+                }
                 const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
                 if (pipelineResult.final_output_path) {
+                    if (path.resolve(pipelineResult.final_output_path) !== path.resolve(sourcePath)) {
+                        intermediateArtifacts.add(pipelineResult.final_output_path);
+                    }
                     sourcePath = pipelineResult.final_output_path;
                     if (!isDirectLocalInput && shouldTrackExportPath(sourcePath)) {
                         upsertRegistry(datasetId, sourcePath, "completed");
@@ -3346,6 +4034,7 @@ async function runExportCli(args) {
     console.error(`[Export] Resolved output directory: ${outDir}`);
     console.error(`[Export] Output file: ${outputFile}`);
     const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
+    cleanupIntermediateArtifacts(intermediateArtifacts, result.output_path);
     console.log(`Export complete: ${result.output_path}`);
     console.log(`Format: ${result.format}${result.compression ? ` (${result.compression})` : ""}`);
     if (result.rows !== undefined)
@@ -3412,6 +4101,173 @@ async function runFuseCli(args) {
         console.log(`Preview saved: ${result.preview_path}`);
     console.log("Next: run vespermcp split/export on the fused dataset");
 }
+async function runStatusCli(args) {
+    const [{ default: chalk }, { default: Table }] = await Promise.all([
+        import("chalk"),
+        import("cli-table3"),
+    ]);
+    const getArgValue = (name) => {
+        const idx = args.findIndex(a => a === name);
+        if (idx >= 0 && idx + 1 < args.length)
+            return args[idx + 1];
+        return undefined;
+    };
+    const defaultDir = path.join(dataRoot, "lineage");
+    const scanDirRaw = getArgValue("--dir");
+    const scanDir = path.resolve(scanDirRaw || defaultDir);
+    const maxDepthRaw = getArgValue("--max-depth");
+    const maxDepthParsed = maxDepthRaw !== undefined ? Number(maxDepthRaw) : 4;
+    const maxDepth = Number.isFinite(maxDepthParsed) && maxDepthParsed >= 0
+        ? Math.floor(maxDepthParsed)
+        : 4;
+    if (!fs.existsSync(scanDir)) {
+        console.log(`Lineage directory not found: ${scanDir}`);
+        console.log("Tip: use --dir <path> to scan a custom location.");
+        return;
+    }
+    const lineageFiles = walkFilesRecursive(scanDir, maxDepth).filter((p) => p.toLowerCase().endsWith(".lineage.json"));
+    const records = [];
+    for (const filePath of lineageFiles) {
+        try {
+            const data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+            if (isLineageRecordShape(data)) {
+                records.push(data);
+            }
+        }
+        catch {
+            // ignore malformed files
+        }
+    }
+    if (records.length === 0) {
+        console.log("No lineage records found.");
+        console.log("Tip: default scan is ~/.vesper/lineage. Use --dir <path> for project-local lineage files.");
+        return;
+    }
+    const allVersions = records.flatMap((r) => r.versions || []);
+    const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const operationsLast7d = allVersions.filter((v) => Date.parse(v.created_at || "") >= sevenDaysAgo).length;
+    console.log(chalk.bold.cyan("\nVesper Lineage Status"));
+    console.log(chalk.gray(`Scan dir: ${scanDir}`));
+    console.log(chalk.gray(`Max depth: ${maxDepth}`));
+    console.log(chalk.gray(`Lineage records: ${records.length}`));
+    console.log(chalk.gray(`Total operations: ${allVersions.length} (${operationsLast7d} in last 7 days)\n`));
+    const perDatasetTable = new Table({
+        head: ["Dataset", "Versions", "Last Modified", "Last Actor"],
+        colWidths: [34, 10, 28, 28],
+        wordWrap: true,
+    });
+    for (const record of records.sort((a, b) => (a.dataset_id_base || "").localeCompare(b.dataset_id_base || ""))) {
+        const sorted = [...(record.versions || [])].sort((a, b) => (a.version || 0) - (b.version || 0));
+        const last = sorted[sorted.length - 1];
+        const actor = last?.triggered_by?.agent_id || last?.triggered_by?.pipeline_id || "-";
+        perDatasetTable.push([
+            record.dataset_id_base,
+            String(sorted.length),
+            last?.created_at || "-",
+            actor,
+        ]);
+    }
+    console.log(chalk.bold("Per-dataset summary"));
+    console.log(perDatasetTable.toString());
+    const trendTable = new Table({
+        head: ["Dataset", "Rows Trend", "Details"],
+        colWidths: [34, 14, 52],
+        wordWrap: true,
+    });
+    for (const record of records.sort((a, b) => (a.dataset_id_base || "").localeCompare(b.dataset_id_base || ""))) {
+        const sorted = [...(record.versions || [])].sort((a, b) => (a.version || 0) - (b.version || 0));
+        const series = sorted
+            .map((v) => ({ version: v.version, rows: v.output?.schema_after?.rows ?? v.output?.rows }))
+            .filter((x) => typeof x.rows === "number");
+        if (series.length < 2) {
+            trendTable.push([record.dataset_id_base, "-", "insufficient row snapshots"]);
+            continue;
+        }
+        const first = series[0].rows;
+        const last = series[series.length - 1].rows;
+        const trend = last > first ? chalk.green("growing") : last < first ? chalk.yellow("shrinking") : "flat";
+        const details = series.map((x) => `v${x.version}:${x.rows}`).join(" -> ");
+        trendTable.push([record.dataset_id_base, trend, details]);
+    }
+    console.log(chalk.bold("\nQuality trend (schema_after.rows)"));
+    console.log(trendTable.toString());
+    const dtypeWarnings = [];
+    for (const record of records) {
+        const sorted = [...(record.versions || [])].sort((a, b) => (a.version || 0) - (b.version || 0));
+        for (let i = 1; i < sorted.length; i++) {
+            const prev = sorted[i - 1];
+            const curr = sorted[i];
+            const prevSchema = prev.output?.schema_after || prev.output?.schema_before;
+            const currSchema = curr.output?.schema_after || curr.output?.schema_before;
+            const prevCols = prevSchema?.columns || [];
+            const currCols = currSchema?.columns || [];
+            const prevDtypes = prevSchema?.dtypes || {};
+            const currDtypes = currSchema?.dtypes || {};
+            const diff = diffSchemaMaps(prevCols, currCols, prevDtypes, currDtypes);
+            if (diff.changed_dtypes.length > 0) {
+                dtypeWarnings.push({
+                    dataset: record.dataset_id_base,
+                    from: prev.version,
+                    to: curr.version,
+                    changes: diff.changed_dtypes.slice(0, 4).map((d) => `${d.column}:${d.from}->${d.to}`),
+                });
+            }
+        }
+    }
+    console.log(chalk.bold("\nDtype warnings"));
+    if (dtypeWarnings.length === 0) {
+        console.log(chalk.green("No dtype changes detected across adjacent versions."));
+    }
+    else {
+        const warningTable = new Table({
+            head: ["Dataset", "Versions", "Changed dtypes"],
+            colWidths: [34, 14, 52],
+            wordWrap: true,
+        });
+        for (const w of dtypeWarnings.slice(-20)) {
+            warningTable.push([
+                w.dataset,
+                `v${w.from}->v${w.to}`,
+                w.changes.join(", "),
+            ]);
+        }
+        console.log(warningTable.toString());
+    }
+    const lineageErrors = [];
+    for (const record of records) {
+        for (const v of record.versions || []) {
+            for (const step of v.steps || []) {
+                const errMsg = (typeof step.metrics?.error === "string" && step.metrics.error) ||
+                    (typeof step.params?.error === "string" && step.params.error) ||
+                    undefined;
+                if (errMsg) {
+                    lineageErrors.push({ at: step.at, where: `${record.dataset_id_base}/v${v.version}:${step.step}`, message: errMsg });
+                }
+            }
+        }
+    }
+    const logErrors = parseErrorLogLines(errorLogPath, 7).map((e) => ({
+        at: e.at,
+        where: "vesper_errors.log",
+        message: e.message,
+    }));
+    const recentErrors = [...lineageErrors, ...logErrors].slice(-20);
+    console.log(chalk.bold("\nRecent errors"));
+    if (recentErrors.length === 0) {
+        console.log(chalk.green("No recent lineage-linked errors found."));
+    }
+    else {
+        const errTable = new Table({
+            head: ["At", "Where", "Error"],
+            colWidths: [28, 36, 46],
+            wordWrap: true,
+        });
+        for (const e of recentErrors) {
+            errTable.push([e.at || "-", e.where, e.message]);
+        }
+        console.log(errTable.toString());
+    }
+}
 async function runSetupWizard(silent = false) {
     if (!silent && process.stdin.isTTY) {
         const wizardCandidates = [

package/build/python/cleaner.py

@@ -182,6 +182,8 @@ def main():
                 output_format = "parquet"
 
         base_name = file_path.rsplit(".", 1)[0]
+        if base_name.endswith("_cleaned"):
+            base_name = base_name[:-8]
         if output_format == "csv":
             output_path = f"{base_name}_cleaned.csv"
             # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vespermcp/mcp-server",
-  "version": "1.2.27",
+  "version": "1.2.29",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",
@@ -37,6 +37,7 @@
     "setup": "node build/index.js --setup",
     "setup:silent": "node build/index.js --setup --silent",
     "refresh-index": "node scripts/refresh-index.cjs",
+    "telemetry:receiver": "tsx telemetry/lineage-receiver.ts",
     "test": "vitest",
     "start": "node build/index.js"
   },
@@ -79,9 +80,13 @@
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
     "better-sqlite3": "^12.6.0",
+    "chalk": "^5.6.2",
+    "cli-table3": "^0.6.5",
+    "express": "^5.1.0",
     "inquirer": "^13.3.0",
     "lodash": "^4.17.21",
     "pdf-parse": "^2.4.5",
+    "pg": "^8.16.3",
     "uuid": "^13.0.0",
     "zod": "^4.3.5",
     "zod-to-json-schema": "^3.25.1"

package/src/python/cleaner.py

@@ -182,6 +182,8 @@ def main():
                 output_format = "parquet"
 
         base_name = file_path.rsplit(".", 1)[0]
+        if base_name.endswith("_cleaned"):
+            base_name = base_name[:-8]
         if output_format == "csv":
             output_path = f"{base_name}_cleaned.csv"
             # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)