@vespermcp/mcp-server 1.0.0

package/build/metadata/uci-scraper.js

@@ -0,0 +1,49 @@
+import { spawn } from "child_process";
+import path from "path";
+const PYTHON_SCRIPT_PATH = path.resolve("src", "python", "uci_adapter.py");
+export class UCIScraper {
+    /**
+     * Search UCI repository using the Python adapter
+     */
+    async scrape(query, limit = 10) {
+        return new Promise((resolve, reject) => {
+            const pythonProcess = spawn("python", [
+                PYTHON_SCRIPT_PATH,
+                "--action", "search",
+                "--query", query,
+                "--limit", String(limit)
+            ]);
+            let output = "";
+            let errorOutput = "";
+            pythonProcess.stdout.on("data", (data) => {
+                output += data.toString();
+            });
+            pythonProcess.stderr.on("data", (data) => {
+                errorOutput += data.toString();
+            });
+            pythonProcess.on("close", (code) => {
+                if (code !== 0) {
+                    // It's possible for python to emit stderr warnings but still succeed
+                    // But exit code != 0 is definitely an error
+                    console.error(`[UCIScraper] Process exited with code ${code}: ${errorOutput}`);
+                    resolve([]); // Fail gracefully by returning empty
+                    return;
+                }
+                try {
+                    const results = JSON.parse(output);
+                    if (results.error) {
+                        console.error(`[UCIScraper] Internal error: ${results.error}`);
+                        resolve([]);
+                    }
+                    else {
+                        resolve(results);
+                    }
+                }
+                catch (e) {
+                    console.error(`[UCIScraper] JSON parse error: ${e.message}. Output: ${output.substring(0, 100)}...`);
+                    resolve([]);
+                }
+            });
+        });
+    }
+}

package/build/monitoring/observability.js

@@ -0,0 +1,76 @@
+export class MockErrorTracker {
+    exceptions = [];
+    messages = [];
+    captureException(error, context) {
+        console.log(`[ErrorTracker] Exception captured: ${error.message}`);
+        this.exceptions.push({ error, context });
+    }
+    captureMessage(message, level = "info") {
+        console.log(`[ErrorTracker] Message captured (${level}): ${message}`);
+        this.messages.push({ message, level });
+    }
+}
+export class ObservabilityService {
+    errorTracker;
+    jobSuccessCounter = new Map();
+    jobFailureCounter = new Map();
+    jobDurationHistogram = new Map();
+    constructor(errorTracker = new MockErrorTracker()) {
+        this.errorTracker = errorTracker;
+    }
+    recordJobSuccess(type, durationMs) {
+        // Increment success counter
+        this.jobSuccessCounter.set(type, (this.jobSuccessCounter.get(type) || 0) + 1);
+        // Record duration
+        this.updateMetricRecord(type, durationMs);
+    }
+    recordJobFailure(type, error) {
+        // Increment failure counter
+        this.jobFailureCounter.set(type, (this.jobFailureCounter.get(type) || 0) + 1);
+        // Track error
+        this.errorTracker.captureException(error, { jobType: type });
+    }
+    getPrometheusMetrics() {
+        let out = "# HELP jobs_processed_total Total number of jobs successfully processed\n";
+        out += "# TYPE jobs_processed_total counter\n";
+        for (const [type, count] of this.jobSuccessCounter) {
+            out += `jobs_processed_total{type="${type}"} ${count}\n`;
+        }
+        out += "\n# HELP jobs_failed_total Total number of failed jobs\n";
+        out += "# TYPE jobs_failed_total counter\n";
+        for (const [type, count] of this.jobFailureCounter) {
+            out += `jobs_failed_total{type="${type}"} ${count}\n`;
+        }
+        out += "\n# HELP job_duration_seconds_sum Latency of job processing in seconds\n";
+        out += "# TYPE job_duration_seconds_sum counter\n";
+        for (const [type, record] of this.jobDurationHistogram) {
+            out += `job_duration_seconds_sum{type="${type}"} ${record.sum / 1000}\n`;
+            out += `job_duration_seconds_count{type="${type}"} ${record.count}\n`;
+            out += `job_duration_seconds_max{type="${type}"} ${record.max / 1000}\n`;
+        }
+        return out;
+    }
+    getStats() {
+        const stats = {};
+        for (const [type, record] of this.jobDurationHistogram) {
+            stats[type] = {
+                successCount: this.jobSuccessCounter.get(type) || 0,
+                failureCount: this.jobFailureCounter.get(type) || 0,
+                avgDuration: (record.sum / record.count).toFixed(2) + "ms",
+                maxDuration: record.max + "ms"
+            };
+        }
+        return stats;
+    }
+    updateMetricRecord(type, value) {
+        let record = this.jobDurationHistogram.get(type);
+        if (!record) {
+            record = { count: 0, sum: 0, min: value, max: value };
+            this.jobDurationHistogram.set(type, record);
+        }
+        record.count++;
+        record.sum += value;
+        record.min = Math.min(record.min, value);
+        record.max = Math.max(record.max, value);
+    }
+}

package/build/quality/analyzer.js

@@ -0,0 +1,57 @@
+import { spawn } from "child_process";
+import path from "path";
+export class QualityAnalyzer {
+    cache;
+    pythonPath = "python"; // Assumes python is in PATH
+    scriptPath;
+    constructor(cache, projectRoot = process.cwd()) {
+        this.cache = cache;
+        this.scriptPath = path.join(projectRoot, "src", "python", "quality_engine.py");
+    }
+    /**
+     * Run quality analysis on a local file (CSV/Parquet/JSON)
+     * @param datasetId Used for caching
+     */
+    async analyze(filePath, datasetId) {
+        if (this.cache && datasetId) {
+            const cached = await this.cache.getReport(datasetId);
+            if (cached) {
+                console.log(`[QualityAnalyzer] Cache hit for ${datasetId}`);
+                return cached;
+            }
+        }
+        const report = await new Promise((resolve, reject) => {
+            const process = spawn(this.pythonPath, [this.scriptPath, filePath]);
+            let stdout = "";
+            let stderr = "";
+            process.stdout.on("data", (data) => {
+                stdout += data.toString();
+            });
+            process.stderr.on("data", (data) => {
+                stderr += data.toString();
+            });
+            process.on("close", (code) => {
+                if (code !== 0) {
+                    reject(new Error(`Quality Analyzer failed (code ${code}): ${stderr}`));
+                    return;
+                }
+                try {
+                    const report = JSON.parse(stdout);
+                    if (report.error) {
+                        reject(new Error(report.error));
+                    }
+                    else {
+                        resolve(report);
+                    }
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse analyzer output: ${stdout}`));
+                }
+            });
+        });
+        if (this.cache && datasetId) {
+            await this.cache.saveReport(datasetId, report);
+        }
+        return report;
+    }
+}

package/build/quality/image-analyzer.js

@@ -0,0 +1,46 @@
+import { spawn } from "child_process";
+import path from "path";
+export class ImageAnalyzer {
+    pythonPath = "python";
+    scriptPath;
+    constructor(projectRoot = process.cwd()) {
+        this.scriptPath = path.join(projectRoot, "src", "python", "image_engine.py");
+    }
+    /**
+     * Analyze image quality for a single file or a directory
+     */
+    async analyze(inputPath) {
+        return new Promise((resolve, reject) => {
+            const process = spawn(this.pythonPath, [
+                this.scriptPath,
+                inputPath
+            ]);
+            let stdout = "";
+            let stderr = "";
+            process.stdout.on("data", (data) => {
+                stdout += data.toString();
+            });
+            process.stderr.on("data", (data) => {
+                stderr += data.toString();
+            });
+            process.on("close", (code) => {
+                if (code !== 0) {
+                    reject(new Error(`Image Analyzer failed (code ${code}): ${stderr}`));
+                    return;
+                }
+                try {
+                    const result = JSON.parse(stdout);
+                    if (result.error) {
+                        reject(new Error(result.error));
+                    }
+                    else {
+                        resolve(result);
+                    }
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse image analyzer output: ${stdout}`));
+                }
+            });
+        });
+    }
+}

package/build/quality/media-analyzer.js

@@ -0,0 +1,46 @@
+import { spawn } from "child_process";
+import path from "path";
+export class MediaAnalyzer {
+    pythonPath = "python";
+    scriptPath;
+    constructor(projectRoot = process.cwd()) {
+        this.scriptPath = path.join(projectRoot, "src", "python", "media_engine.py");
+    }
+    /**
+     * Analyze audio/video quality for a single file or a directory
+     */
+    async analyze(inputPath) {
+        return new Promise((resolve, reject) => {
+            const process = spawn(this.pythonPath, [
+                this.scriptPath,
+                inputPath
+            ]);
+            let stdout = "";
+            let stderr = "";
+            process.stdout.on("data", (data) => {
+                stdout += data.toString();
+            });
+            process.stderr.on("data", (data) => {
+                stderr += data.toString();
+            });
+            process.on("close", (code) => {
+                if (code !== 0) {
+                    reject(new Error(`Media Analyzer failed (code ${code}): ${stderr}`));
+                    return;
+                }
+                try {
+                    const result = JSON.parse(stdout);
+                    if (result.error) {
+                        reject(new Error(result.error));
+                    }
+                    else {
+                        resolve(result);
+                    }
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse media analyzer output: ${stdout}`));
+                }
+            });
+        });
+    }
+}

package/build/quality/quality-orchestrator.js

@@ -0,0 +1,162 @@
+import fs from "fs";
+import { ImageAnalyzer } from "./image-analyzer.js";
+import { MediaAnalyzer } from "./media-analyzer.js";
+export class QualityOrchestrator {
+    imageAnalyzer;
+    mediaAnalyzer;
+    constructor(projectRoot = process.cwd()) {
+        this.imageAnalyzer = new ImageAnalyzer(projectRoot);
+        this.mediaAnalyzer = new MediaAnalyzer(projectRoot);
+    }
+    /**
+     * Detect modalities present in a dataset directory
+     */
+    detectModalities(datasetPath) {
+        const modalities = [];
+        if (!fs.existsSync(datasetPath)) {
+            return modalities;
+        }
+        const files = fs.readdirSync(datasetPath);
+        // Check for text/tabular data
+        const hasText = files.some(f => /\.(csv|json|parquet|txt)$/i.test(f));
+        if (hasText)
+            modalities.push("text");
+        // Check for images
+        const hasImages = files.some(f => /\.(jpg|jpeg|png|bmp|webp)$/i.test(f));
+        if (hasImages)
+            modalities.push("image");
+        // Check for audio
+        const hasAudio = files.some(f => /\.(wav|mp3|flac|ogg|m4a)$/i.test(f));
+        if (hasAudio)
+            modalities.push("audio");
+        // Check for video
+        const hasVideo = files.some(f => /\.(mp4|avi|mkv|mov|wmv)$/i.test(f));
+        if (hasVideo)
+            modalities.push("video");
+        return modalities;
+    }
+    /**
+     * Generate a unified quality report for a dataset
+     */
+    async generateReport(datasetId, datasetPath, textQuality) {
+        const modalities = this.detectModalities(datasetPath);
+        const report = {
+            dataset_id: datasetId,
+            modalities,
+            overall_quality_score: 0,
+            recommendations: [],
+            generated_at: new Date().toISOString()
+        };
+        let totalScore = 0;
+        let scoreCount = 0;
+        // Text quality (if provided from existing analysis)
+        if (textQuality) {
+            report.text_quality = {
+                row_count: textQuality.row_count || 0,
+                column_count: textQuality.column_count || 0,
+                missing_percentage: textQuality.missing_percentage || 0,
+                duplicate_percentage: textQuality.duplicate_percentage || 0
+            };
+            // Calculate text quality score (0-100)
+            const textScore = Math.max(0, 100 - (report.text_quality.missing_percentage * 2) - (report.text_quality.duplicate_percentage));
+            totalScore += textScore;
+            scoreCount++;
+            if (report.text_quality.missing_percentage > 20) {
+                report.recommendations.push("High missing data detected. Consider imputation or removal.");
+            }
+            if (report.text_quality.duplicate_percentage > 10) {
+                report.recommendations.push("Significant duplicates found. Run deduplication.");
+            }
+        }
+        // Image quality
+        if (modalities.includes("image")) {
+            try {
+                const imageReport = await this.imageAnalyzer.analyze(datasetPath);
+                report.image_quality = {
+                    total_images: imageReport.total_images,
+                    corrupted_count: imageReport.corrupted_count,
+                    avg_resolution: `${Math.round(imageReport.average_width)}x${Math.round(imageReport.average_height)}`,
+                    blurry_percentage: (imageReport.blurry_count / imageReport.total_images) * 100
+                };
+                // Calculate image quality score
+                const corruptionPenalty = (imageReport.corrupted_count / imageReport.total_images) * 50;
+                const blurPenalty = report.image_quality.blurry_percentage * 0.3;
+                const imageScore = Math.max(0, 100 - corruptionPenalty - blurPenalty);
+                totalScore += imageScore;
+                scoreCount++;
+                if (report.image_quality.corrupted_count > 0) {
+                    report.recommendations.push(`Remove ${imageReport.corrupted_count} corrupted images.`);
+                }
+                if (report.image_quality.blurry_percentage > 15) {
+                    report.recommendations.push("High blur detected. Consider filtering blurry images.");
+                }
+            }
+            catch (e) {
+                console.error("Image analysis failed:", e);
+            }
+        }
+        // Audio quality
+        if (modalities.includes("audio")) {
+            try {
+                const audioReport = await this.mediaAnalyzer.analyze(datasetPath);
+                if ('avg_audio_duration' in audioReport) {
+                    const silentFiles = audioReport.details.filter(d => d.status === "ok" && 'is_silent' in d && d.is_silent).length;
+                    const avgSampleRate = audioReport.details
+                        .filter(d => d.status === "ok" && 'sample_rate' in d)
+                        .reduce((sum, d) => sum + (('sample_rate' in d) ? (d.sample_rate || 0) : 0), 0) / audioReport.ok_files;
+                    report.audio_quality = {
+                        total_files: audioReport.total_files,
+                        avg_duration: audioReport.avg_audio_duration || 0,
+                        avg_sample_rate: avgSampleRate,
+                        silent_percentage: (silentFiles / audioReport.total_files) * 100
+                    };
+                    // Calculate audio quality score
+                    const failurePenalty = (audioReport.failed_files / audioReport.total_files) * 50;
+                    const silentPenalty = report.audio_quality.silent_percentage * 0.5;
+                    const audioScore = Math.max(0, 100 - failurePenalty - silentPenalty);
+                    totalScore += audioScore;
+                    scoreCount++;
+                    if (report.audio_quality.silent_percentage > 10) {
+                        report.recommendations.push("High percentage of silent audio files detected.");
+                    }
+                }
+            }
+            catch (e) {
+                console.error("Audio analysis failed:", e);
+            }
+        }
+        // Video quality
+        if (modalities.includes("video")) {
+            try {
+                const videoReport = await this.mediaAnalyzer.analyze(datasetPath);
+                if ('avg_video_duration' in videoReport) {
+                    const highRiskFiles = videoReport.details.filter(d => d.status === "ok" && d.corruption_risk === "high").length;
+                    report.video_quality = {
+                        total_files: videoReport.total_files,
+                        avg_duration: videoReport.avg_video_duration || 0,
+                        avg_fps: videoReport.avg_fps || 0,
+                        corruption_risk_high: highRiskFiles
+                    };
+                    // Calculate video quality score
+                    const failurePenalty = (videoReport.failed_files / videoReport.total_files) * 50;
+                    const corruptionPenalty = (highRiskFiles / videoReport.total_files) * 30;
+                    const videoScore = Math.max(0, 100 - failurePenalty - corruptionPenalty);
+                    totalScore += videoScore;
+                    scoreCount++;
+                    if (highRiskFiles > 0) {
+                        report.recommendations.push(`${highRiskFiles} video files have high corruption risk.`);
+                    }
+                }
+            }
+            catch (e) {
+                console.error("Video analysis failed:", e);
+            }
+        }
+        // Calculate overall quality score
+        report.overall_quality_score = scoreCount > 0 ? Math.round(totalScore / scoreCount) : 0;
+        if (report.recommendations.length === 0) {
+            report.recommendations.push("Dataset quality is good. No major issues detected.");
+        }
+        return report;
+    }
+}

package/build/quality/types.js

@@ -0,0 +1 @@
+export {};

package/build/scripts/build-index.js

@@ -0,0 +1,54 @@
+import { Embedder } from "../search/embedder.js";
+import { VectorStore } from "../search/vector-store.js";
+import { MetadataStore } from "../metadata/store.js";
+import path from "path";
+async function main() {
+    const dbPath = path.join(process.cwd(), "data", "metadata.db");
+    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
+    const metadataStore = new MetadataStore(dbPath);
+    const vectorStore = new VectorStore(vectorPath);
+    const embedder = Embedder.getInstance();
+    const datasets = metadataStore.getAllDatasets();
+    const indexedIds = new Set(vectorStore.getAllIds());
+    // Filter to only new datasets
+    const toIndex = datasets.filter(ds => !indexedIds.has(ds.id));
+    console.error(`Total datasets: ${datasets.length}, Already indexed: ${indexedIds.size}, To index: ${toIndex.length}`);
+    const BATCH_SIZE = 50;
+    let processed = 0;
+    for (let i = 0; i < toIndex.length; i += BATCH_SIZE) {
+        const batch = toIndex.slice(i, i + BATCH_SIZE);
+        try {
+            // Prepare texts for batch embedding
+            const texts = batch.map(ds => [
+                ds.name,
+                ds.description,
+                `Task: ${ds.task}`,
+                `Languages: ${ds.languages?.join(", ") || ""}`,
+                `Tags: ${ds.tags?.join(" ") || ""}`
+            ].join(" ").slice(0, 1500));
+            // Embed batch (Xenova supports array input)
+            // Note: Parallelizing at the embed level is better for CPU utilization
+            await Promise.all(batch.map(async (ds, idx) => {
+                try {
+                    const vector = await embedder.embed(texts[idx]);
+                    vectorStore.add(ds.id, vector);
+                }
+                catch (err) {
+                    console.error(`Failed to index ${ds.id}:`, err);
+                }
+            }));
+            processed += batch.length;
+            if (processed % 100 === 0 || i + BATCH_SIZE >= toIndex.length) {
+                console.error(`Indexed ${processed}/${toIndex.length} new datasets...`);
+                vectorStore.save();
+            }
+        }
+        catch (err) {
+            console.error(`Batch around ${i} failed:`, err);
+        }
+    }
+    vectorStore.save();
+    console.error("Vector indexing complete.");
+    metadataStore.close();
+}
+main().catch(console.error);

package/build/scripts/check-db.js

@@ -0,0 +1,73 @@
+import Database from "better-sqlite3";
+import path from "path";
+import fs from "fs";
+async function main() {
+    const dbPath = path.join(process.cwd(), "data", "metadata.db");
+    if (!fs.existsSync(dbPath)) {
+        console.error("Database not found. Run 'npm run scrape' first.");
+        process.exit(1);
+    }
+    const db = new Database(dbPath);
+    try {
+        // Get total count
+        const count = db.prepare("SELECT COUNT(*) as count FROM datasets").get();
+        console.log(`\nTotal datasets in database: ${count.count}\n`);
+        // Check which columns exist
+        const tableInfo = db.prepare("PRAGMA table_info(datasets)").all();
+        const columns = tableInfo.map(col => col.name);
+        const hasNewColumns = columns.includes("is_safe_source");
+        // Get basic statistics (works with both old and new schema)
+        const stats = db.prepare(`
+            SELECT 
+                COUNT(*) as total,
+                SUM(downloads) as total_downloads,
+                AVG(quality_score) as avg_quality,
+                SUM(CASE WHEN license_category = 'safe' THEN 1 ELSE 0 END) as safe_licenses,
+                SUM(CASE WHEN has_train_split = 1 THEN 1 ELSE 0 END) as with_train_split
+            FROM datasets
+        `).get();
+        console.log("Statistics:");
+        console.log(`   Total downloads: ${stats.total_downloads?.toLocaleString() || 0}`);
+        console.log(`   Average quality score: ${Math.round(stats.avg_quality || 0)}`);
+        console.log(`   Safe licenses: ${stats.safe_licenses || 0}`);
+        console.log(`   With train split: ${stats.with_train_split || 0}`);
+        // Show extended stats if new schema is available
+        if (hasNewColumns) {
+            const extendedStats = db.prepare(`
+                SELECT 
+                    SUM(CASE WHEN is_safe_source = 1 THEN 1 ELSE 0 END) as safe_sources,
+                    SUM(CASE WHEN is_structured = 1 THEN 1 ELSE 0 END) as structured,
+                    SUM(total_examples) as total_examples
+                FROM datasets
+            `).get();
+            console.log(`   Safe sources: ${extendedStats.safe_sources || 0}`);
+            console.log(`   Structured datasets: ${extendedStats.structured || 0}`);
+            console.log(`   Total examples: ${extendedStats.total_examples?.toLocaleString() || 0}`);
+        }
+        else {
+            console.log(`   WARNING: Database uses old schema. Re-scrape to get extended statistics.`);
+        }
+        console.log();
+        // Top 5 by downloads
+        const top5 = db.prepare(`
+            SELECT id, name, downloads, quality_score, license_category
+            FROM datasets
+            ORDER BY downloads DESC
+            LIMIT 5
+        `).all();
+        console.log("Top 5 datasets by downloads:");
+        top5.forEach((ds, i) => {
+            console.log(`   ${i + 1}. ${ds.id}`);
+            console.log(`      Downloads: ${ds.downloads.toLocaleString()}, Quality: ${ds.quality_score}, License: ${ds.license_category}`);
+        });
+        console.log();
+    }
+    catch (error) {
+        console.error("Error reading database:", error);
+        process.exit(1);
+    }
+    finally {
+        db.close();
+    }
+}
+main();

package/build/scripts/check-jobs.js

@@ -0,0 +1,24 @@
+import Database from "better-sqlite3";
+import path from "path";
+// Checking all plausible databases for jobs
+const dbs = ["metadata.db", "vesper.db", "datasets.db"];
+for (const dbName of dbs) {
+    const dbPath = path.resolve("data", dbName);
+    try {
+        const db = new Database(dbPath);
+        const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='jobs'").all();
+        if (tables.length > 0) {
+            console.log(`\n--- Checking jobs in ${dbName} ---`);
+            const jobs = db.prepare("SELECT * FROM jobs ORDER BY created_at DESC LIMIT 20").all();
+            for (const job of jobs) {
+                if (JSON.stringify(job).toLowerCase().includes("naruto")) {
+                    console.log(JSON.stringify(job, null, 2));
+                }
+            }
+        }
+        db.close();
+    }
+    catch (e) {
+        // Silently skip if DB doesn't exist or table missing
+    }
+}

package/build/scripts/check-naruto.js

@@ -0,0 +1,17 @@
+import Database from "better-sqlite3";
+import path from "path";
+// Trying metadata.db which is larger
+const dbPath = path.resolve("data", "metadata.db");
+const db = new Database(dbPath);
+try {
+    const query = "naruto";
+    const results = db.prepare("SELECT * FROM datasets WHERE name LIKE ? OR description LIKE ?").all(`%${query}%`, `%${query}%`);
+    console.log(`Found ${results.length} results for "${query}" in metadata.db:`);
+    console.log(JSON.stringify(results, null, 2));
+}
+catch (e) {
+    console.error("Error checking database:", e.message);
+}
+finally {
+    db.close();
+}

package/build/scripts/demo-full-pipeline.js

@@ -0,0 +1,62 @@
+import { PipelineExecutor } from "../cleaning/executor.js";
+import { ScriptGenerator } from "../cleaning/exporter.js";
+import fs from "fs";
+import path from "path";
+async function main() {
+    console.log(" Vesper Dataset Ops Engine: Full Demo\n");
+    const executor = new PipelineExecutor();
+    const exporter = new ScriptGenerator();
+    const demoFile = path.join(process.cwd(), "vesper_demo_data.csv");
+    // 1. Create a Realistic Dirty Dataset
+    // - duplicate: Duplicate Customer
+    // - age: Mixed types ("25", "twenty"), Outliers (200)
+    // - email: PII
+    // - empty_col: 100% missing
+    // - score: Good data
+    const csvContent = `customer_id,age,email,score,empty_col
+C001,25,john.doe@example.com,88.5,
+C002,"30",jane.smith@work.org,92.0,
+C003,200,bob.jones@gmail.com,15.0,
+C001,25,john.doe@example.com,88.5,
+C004,"forty",alice@co.uk,80.0,
+C005,35,,75.0,`;
+    fs.writeFileSync(demoFile, csvContent);
+    console.log(`📦 Created dirty dataset: ${demoFile}`);
+    console.log(`Contains: Duplicates, PII (Emails), Mixed Types (Age), Outliers, Empty Columns.\n`);
+    try {
+        // 2. Run the Auto-Cleaning Pipeline
+        console.log(" Running Auto-Cleaning Pipeline...");
+        const result = await executor.runPipeline("demo-dataset", demoFile);
+        console.log("\n --- Quality Inspection Report ---");
+        console.log(`   Duplicates: ${result.initial_quality.duplicate_rows} rows`);
+        console.log(`   PII Warnings: ${result.initial_quality.pii_warnings?.length || 0}`);
+        if (result.initial_quality.schema_warnings.length > 0) {
+            console.log("   Schema Issues:");
+            result.initial_quality.schema_warnings.forEach(w => console.log(`     ⚠️  ${w}`));
+        }
+        console.log("\n --- Generated Cleaning Plan ---");
+        result.plan.operations.forEach((op, i) => {
+            console.log(`   ${i + 1}. [${op.type}] Reason: ${op.reason}`);
+        });
+        console.log("\n --- Execution Result ---");
+        if (result.cleaning_result.success) {
+            console.log(`    Success! Cleaned file saved to:`);
+            console.log(`      ${result.final_output_path}`);
+        }
+        else {
+            console.error(`    Failed: ${result.cleaning_result.error}`);
+        }
+        // 3. Generate Reproducibility Script
+        console.log("\n --- Reproducibility ---");
+        const pythonScript = exporter.generatePythonScript(result.plan, demoFile);
+        const scriptPath = path.join(process.cwd(), "demo_cleaning_script.py");
+        fs.writeFileSync(scriptPath, pythonScript);
+        console.log(`    Generated Python script: ${scriptPath}`);
+        console.log(`   (You can run this script independently to reproduce these steps!)`);
+        console.log("\n Demo Complete.");
+    }
+    catch (error) {
+        console.error("\n Demo failed:", error);
+    }
+}
+main().catch(console.error);