@vespermcp/mcp-server 1.0.0

package/build/scripts/test-ingestion-infra.js

@@ -0,0 +1,39 @@
+import { MetadataStore } from "../metadata/store.js";
+import { DataIngestor } from "../ingestion/ingestor.js";
+import path from "path";
+import fs from "fs";
+import { fileURLToPath } from "url";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectRoot = path.join(__dirname, "..", "..");
+const dbPath = path.join(projectRoot, "data", "test-metadata.db");
+// Clean test DB if exists
+if (fs.existsSync(dbPath))
+    fs.unlinkSync(dbPath);
+const store = new MetadataStore(dbPath);
+const ingestor = new DataIngestor(projectRoot, store);
+async function testInfra() {
+    console.log("--- Testing Ingestion Infrastructure (6.1) ---");
+    const testId = "test/dataset";
+    // 1. Get target path
+    const target = ingestor.getTargetPath(testId);
+    console.log(`Target path: ${target}`);
+    // 2. Register downloading
+    console.log("Registering download start...");
+    store.registerDownload(testId, target, 'downloading');
+    let status = store.getDownloadStatus(testId);
+    console.log("Status after start:", status?.status);
+    // 3. Register completion
+    console.log("Registering completion...");
+    ingestor.completeDownload(testId, target, 1024);
+    status = store.getDownloadStatus(testId);
+    console.log("Status after completion:", status?.status);
+    console.log("Saved path:", status?.local_path);
+    console.log("Size:", status?.size_bytes, "bytes");
+    // 4. Verify directory exists
+    const rawDir = path.join(projectRoot, "data", "raw");
+    console.log(`Raw data dir created: ${fs.existsSync(rawDir)}`);
+    console.log("\nInfrastructure test PASSED!");
+    store.close();
+}
+testInfra().catch(console.error);

package/build/scripts/test-install.js

@@ -0,0 +1,40 @@
+import { InstallService } from "../install/install-service.js";
+import { MetadataStore } from "../metadata/store.js";
+import path from "path";
+import fs from "fs";
+async function runTest() {
+    console.log("--- Testing Automatic Installation ---");
+    const projectRoot = path.resolve(".");
+    const metadataStore = new MetadataStore(path.join(projectRoot, "data", "metadata.db"));
+    const installService = new InstallService(projectRoot, metadataStore);
+    const query = "naruto";
+    const dataset = metadataStore.db.prepare("SELECT * FROM datasets WHERE name LIKE ? LIMIT 1").get(`%${query}%`);
+    if (!dataset) {
+        console.error("Naruto dataset not found in metadata.db. Please run a search first.");
+        return;
+    }
+    console.log(`Found dataset: ${dataset.name}`);
+    // Mock a prepared file
+    const mockFile = path.join(projectRoot, "data", "raw", "naruto_mock.csv");
+    if (!fs.existsSync(path.dirname(mockFile))) {
+        fs.mkdirSync(path.dirname(mockFile), { recursive: true });
+    }
+    fs.writeFileSync(mockFile, "quote,character\nBelieve it!,Naruto\nI will be Hokage,Naruto");
+    console.log(`Installing ${mockFile}...`);
+    const installPath = await installService.install(dataset.id, mockFile);
+    console.log(`✅ Success! Installed to: ${installPath}`);
+    // Verify it exists
+    if (fs.existsSync(installPath)) {
+        console.log("File exists at install location.");
+    }
+    else {
+        console.error("File MISSING from install location!");
+    }
+    // Verify metadata updated
+    const updated = metadataStore.getDataset(dataset.id);
+    const success = updated?.install_path === installPath;
+    console.log(`Updated install_path in metadata: ${updated?.install_path}`);
+    console.log(`VERIFICATION_STATUS: ${success ? "✅ PASS" : "❌ FAIL"}`);
+    console.log("\n--- Test Complete ---");
+}
+runTest().catch(console.error);

package/build/scripts/test-institutional.js

@@ -0,0 +1,26 @@
+import { WorldBankScraper, NASAScraper } from "../metadata/institutional-scrapers.js";
+async function runTest() {
+    console.log("--- Testing Institutional Data Sources ---");
+    const wb = new WorldBankScraper();
+    const nasa = new NASAScraper();
+    console.log('Searching World Bank for: "climate"...');
+    const wbResults = await wb.scrape("climate", 3);
+    console.log(`Found ${wbResults.length} World Bank results.`);
+    if (wbResults.length > 0) {
+        console.log("✅ World Bank Sample:");
+        console.log(` - ID: ${wbResults[0].id}`);
+        console.log(` - Name: ${wbResults[0].name}`);
+        console.log(` - URL: ${wbResults[0].metadata_url}`);
+    }
+    console.log('\nSearching NASA for: "astronomy"...');
+    const nasaResults = await nasa.scrape("astronomy", 3);
+    console.log(`Found ${nasaResults.length} NASA results.`);
+    if (nasaResults.length > 0) {
+        console.log("✅ NASA Sample:");
+        console.log(` - ID: ${nasaResults[0].id}`);
+        console.log(` - Name: ${nasaResults[0].name}`);
+        console.log(` - URL: ${nasaResults[0].metadata_url}`);
+    }
+    console.log("\n--- Test Complete ---");
+}
+runTest().catch(console.error);

package/build/scripts/test-integrity.js

@@ -0,0 +1,41 @@
+import { QualityAnalyzer } from "../quality/analyzer.js";
+import fs from "fs";
+import path from "path";
+async function main() {
+    console.log("=== Vesper Integrity Check Test ===\n");
+    const analyzer = new QualityAnalyzer();
+    const testFile = path.join(process.cwd(), "test_integrity.csv");
+    // Create a CSV with unique IDs but duplicate CONTENT
+    // This simulates a common data issue: exact same review scraped twice with different IDs
+    const csvContent = `id,review,label
+1,"This product involves huge risks.",negative
+2,"Great investment opportunity!",positive
+3,"This product involves huge risks.",negative
+4,"Wait and see.",neutral
+5,"Great investment opportunity!",positive`;
+    fs.writeFileSync(testFile, csvContent);
+    console.log(`Created test file: ${testFile}`);
+    console.log("Expectations:");
+    console.log("  - Duplicate Rows: 0 (because IDs differ)");
+    console.log("  - Text Duplicates: > 0 (because 'review' column has dupes)");
+    try {
+        console.log("\nRunning analyzer...");
+        const report = await analyzer.analyze(testFile);
+        console.log("\n=== Integrity Report ===");
+        console.log(`Duplicate Rows (Exact): ${report.duplicate_rows}`);
+        console.log(`Text Duplicates (Content): ${report.text_duplicates || 0}`);
+        console.log("\nWarnings:", report.warnings);
+        if (report.text_duplicates && report.text_duplicates > 0) {
+            console.log("Test passed! Detected text duplication despite unique IDs.");
+        }
+        else {
+            console.error("Test failed! Did not detect text duplicates.");
+        }
+        // Cleanup
+        fs.unlinkSync(testFile);
+    }
+    catch (error) {
+        console.error("\nTest failed:", error);
+    }
+}
+main().catch(console.error);

package/build/scripts/test-jit.js

@@ -0,0 +1,42 @@
+import { SearchEngine } from "../search/engine.js";
+import { MetadataStore } from "../metadata/store.js";
+import { VectorStore } from "../search/vector-store.js";
+import { Embedder } from "../search/embedder.js";
+import path from "path";
+/**
+ * Test JIT fallback with various queries
+ */
+async function main() {
+    const dbPath = path.join(process.cwd(), "data", "metadata.db");
+    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
+    const store = new MetadataStore(dbPath);
+    const vectorStore = new VectorStore(vectorPath);
+    const embedder = Embedder.getInstance();
+    await embedder.init();
+    const engine = new SearchEngine(store, vectorStore, embedder);
+    console.log("\n=== JIT Fallback Test Suite ===\n");
+    // Test 1: Query that should have good results (should NOT trigger JIT)
+    console.log("Test 1: Common query (should NOT trigger JIT)");
+    console.log("Query: 'image classification cats dogs'\n");
+    const results1 = await engine.search("image classification cats dogs", { limit: 5 });
+    console.log(`Results: ${results1.length}, Top score: ${results1[0]?.relevance_score || 0}`);
+    console.log("---\n");
+    // Test 2: Very niche query (should trigger JIT)
+    console.log("Test 2: Niche query (SHOULD trigger JIT)");
+    console.log("Query: 'underwater acoustic signal processing dolphins'\n");
+    const results2 = await engine.search("underwater acoustic signal processing dolphins", { limit: 5 });
+    console.log(`Results: ${results2.length}, Top score: ${results2[0]?.relevance_score || 0}`);
+    console.log("---\n");
+    // Test 3: Another niche query
+    console.log("Test 3: Another niche query (SHOULD trigger JIT)");
+    console.log("Query: 'mongolian language morphological analysis'\n");
+    const results3 = await engine.search("mongolian language morphological analysis", { limit: 5 });
+    console.log(`Results: ${results3.length}, Top score: ${results3[0]?.relevance_score || 0}`);
+    console.log("---\n");
+    // Check final dataset count
+    const finalCount = store.getAllDatasets().length;
+    console.log(`\nFinal dataset count: ${finalCount}`);
+    console.log("(Compare with initial 1238 to see if JIT added new datasets)\n");
+    store.close();
+}
+main().catch(console.error);

package/build/scripts/test-job-queue.js

@@ -0,0 +1,62 @@
+import { MetadataStore } from "../metadata/store.js";
+import { JobManager } from "../jobs/manager.js";
+async function runTest() {
+    console.log("--- Initializing Job Queue Test ---");
+    const store = new MetadataStore("data/vesper_test_jobs.db");
+    const manager = JobManager.getInstance(store);
+    manager.setConcurrency(2); // 2 parallel workers
+    const jobsFinished = [];
+    // Listener to simulate job processing
+    manager.on("processJob", async (job, run) => {
+        console.log(`[Worker] Starting Job: ${job.id} (Type: ${job.type}, Priority: ${job.priority})`);
+        await run(async () => {
+            // Simulate variable workload
+            const duration = job.priority === 10 ? 500 : 2000;
+            await new Promise(r => setTimeout(r, duration));
+            // Simulate failure for a specific job to test retries
+            if (job.metadata === "FAIL_ONCE" && job.attempts === 0) {
+                console.log(`[Worker] Simulating failure for job ${job.id}`);
+                throw new Error("Transitory error");
+            }
+            console.log(`[Worker] Finished Job: ${job.id}`);
+            jobsFinished.push(job.id);
+        });
+    });
+    console.log("\n--- Enqueuing Jobs ---");
+    // 1. A slow low-priority job
+    const j1 = manager.createJob("prepare", 0, "slow-1");
+    // 2. A fast high-priority job (Pro user)
+    const j2 = manager.createJob("clean", 10, "pro-1");
+    // 3. Another low-priority job
+    const j3 = manager.createJob("split", 0, "slow-2");
+    // 4. A job that fails once
+    const j4 = manager.createJob("fusion", 5, "FAIL_ONCE");
+    console.log(`Enqueued 4 jobs. Concurrency is 2.`);
+    // Wait for all to finish
+    return new Promise((resolve) => {
+        const check = setInterval(() => {
+            const stats = manager.queue.getStats();
+            if (stats.total === 0 && jobsFinished.includes(j4.id)) {
+                clearInterval(check);
+                console.log("\n--- Test Results ---");
+                console.log("Execution Order:", jobsFinished);
+                const proIndex = jobsFinished.indexOf(j2.id);
+                const slowIndex = jobsFinished.indexOf(j3.id);
+                if (proIndex < slowIndex) {
+                    console.log("✅ Priority verified: Pro job finished before later low-priority jobs.");
+                }
+                else {
+                    console.warn("⚠️ Priority check failed or inconclusive due to parallel timing.");
+                }
+                const j4_final = store.getJob(j4.id);
+                if (j4_final?.attempts === 1) {
+                    console.log("✅ Retry logic verified: Job retried after failure.");
+                }
+                console.log("\n✅ Success: Job queue system verified.");
+                store.close();
+                resolve(null);
+            }
+        }, 1000);
+    });
+}
+runTest().catch(console.error);

package/build/scripts/test-kaggle-download.js

@@ -0,0 +1,34 @@
+import { KaggleDownloader } from "../ingestion/kaggle-downloader.js";
+import path from "path";
+import fs from "fs";
+import { fileURLToPath } from "url";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectRoot = path.join(__dirname, "..", "..");
+async function testKaggleDownload() {
+    // Try to get credentials from env
+    const user = process.env.KAGGLE_USERNAME;
+    const key = process.env.KAGGLE_KEY;
+    if (!user || !key) {
+        console.error("KAGGLE_USERNAME and KAGGLE_KEY must be set to run this test.");
+        return;
+    }
+    const downloader = new KaggleDownloader(user, key);
+    const repoId = "shivam2503/diamonds"; // Small classic dataset
+    console.log(`Testing Kaggle Download for ${repoId}...`);
+    const targetDir = path.join(projectRoot, "data", "test", "kaggle_diamonds");
+    if (fs.existsSync(targetDir))
+        fs.rmSync(targetDir, { recursive: true });
+    try {
+        const bestFile = await downloader.download(repoId, targetDir, (p) => {
+            process.stdout.write(`\rProgress: ${p}%`);
+        });
+        console.log(`\nDownload complete!`);
+        console.log(`Best file found: ${bestFile}`);
+        console.log(`Size: ${fs.statSync(bestFile).size} bytes`);
+    }
+    catch (e) {
+        console.error("\nKaggle Download failed:", e.message);
+    }
+}
+testKaggleDownload().catch(console.error);

package/build/scripts/test-large-data.js

@@ -0,0 +1,50 @@
+import { StreamProcessor } from "../data/streaming.js";
+import { WorkerPool } from "../data/worker-pool.js";
+import { Readable } from "stream";
+async function runTest() {
+    console.log("--- Initializing Large Dataset Handling Test ---");
+    // 1. Generate Mock Data (10,000 records)
+    const records = Array.from({ length: 10000 }, (_, i) => ({
+        id: i,
+        name: `Record ${i}`,
+        value: Math.random() * 100,
+        category: i % 2 === 0 ? "A" : "B"
+    }));
+    console.log(`Generated ${records.length} mock records.`);
+    // 2. Test Sampling Strategy
+    console.log("\n--- Testing Sampling Strategy (10%) ---");
+    const isSampled = StreamProcessor.createSampler(10);
+    const sample = records.filter(isSampled);
+    console.log(`Sample size: ${sample.length} (Expected ~1000)`);
+    // 3. Test Streaming Utility
+    console.log("\n--- Testing Streaming Utility (Chunk size: 2000) ---");
+    const stream = Readable.from(records);
+    let chunkCount = 0;
+    const processChunk = async (chunk) => {
+        chunkCount++;
+        console.log(`Processing chunk ${chunkCount} of size ${chunk.length}`);
+        return chunk.map(r => ({ ...r, status: "streamed" }));
+    };
+    const streamedResults = await StreamProcessor.processInChunks(stream, { chunkSize: 2000 }, processChunk);
+    console.log(`Total streamed records: ${streamedResults.length}`);
+    // 4. Test Parallel Worker Pool
+    console.log("\n--- Testing Parallel Worker Pool (Optional) ---");
+    try {
+        const pool = new WorkerPool(4);
+        console.log("Running parallel transformation on all records...");
+        const start = Date.now();
+        const parallelResults = await pool.processParallel(records, 2500);
+        const end = Date.now();
+        console.log(`Parallel processing took ${end - start}ms`);
+    }
+    catch (err) {
+        console.warn(" Parallel Worker Pool skipped or failed (Common with tsx + worker_threads). Streaming and Sampling are still functional.");
+    }
+    if (streamedResults.length === 10000) {
+        console.log("\n✅ Success: Chunked streaming and sampling verified.");
+    }
+    else {
+        console.error("\n❌ Failure: Processing mismatch.");
+    }
+}
+runTest().catch(console.error);

package/build/scripts/test-mcp-v5.js

@@ -0,0 +1,73 @@
+import { MetadataStore } from "../metadata/store.js";
+import { JobManager } from "../jobs/manager.js";
+import { SearchEngine } from "../search/engine.js";
+import { VectorStore } from "../search/vector-store.js";
+import { Embedder } from "../search/embedder.js";
+import { PipelineExecutor } from "../cleaning/executor.js";
+import path from "path";
+import fs from "fs";
+async function testPhase5Tools() {
+    console.log(" Testing Phase 5.2: New MCP Tools Integration\n");
+    const dbPath = path.join(process.cwd(), "data", "metadata.db");
+    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
+    const store = new MetadataStore(dbPath);
+    const vectorStore = new VectorStore(vectorPath);
+    const embedder = Embedder.getInstance();
+    const searchEngine = new SearchEngine(store, vectorStore, embedder);
+    const jobManager = JobManager.getInstance(store);
+    const executor = new PipelineExecutor();
+    // 1. Test compare_datasets
+    console.log("Step 1: Testing compare_datasets...");
+    const allDs = store.getAllDatasets();
+    if (allDs.length < 2) {
+        console.warn(" ! Need at least 2 datasets in DB to test comparison. Skipping.");
+    }
+    else {
+        const ids = allDs.slice(0, 2).map(d => d.id);
+        console.log(` - Comparing: ${ids.join(", ")}`);
+        // Simulating comparison table logic from index.ts
+        const datasets = ids.map(id => store.getDataset(id)).filter(d => !!d);
+        let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
+        comparison += "| Quality Score | " + datasets.map(d => d.quality_score).join(" | ") + " |\n";
+        console.log(comparison);
+    }
+    // 2. Test analyze_quality / preview_cleaning (Local logic)
+    console.log("\nStep 2: Testing Analysis/Preview (Simulated)...");
+    const testFile = path.join(process.cwd(), "e2e_demo_output", "raw_data.csv");
+    if (fs.existsSync(testFile)) {
+        console.log(" - Testing on e2e_demo_output/raw_data.csv");
+        // Logic check
+    }
+    else {
+        console.warn(" ! No test file found at e2e_demo_output/raw_data.csv. Run 'tsx src/scripts/e2e-demo.ts' first.");
+    }
+    // 3. Test prepare_dataset (Async Job)
+    console.log("\nStep 3: Testing prepare_dataset (Autonomous Orchestrator)...");
+    const query = "financial data for stock prediction";
+    // Create job (Logic from index.ts)
+    const job = jobManager.createJob("prepare", 0, { query });
+    console.log(` - Job Created: ${job.id}`);
+    // Start background task
+    console.log(" - Starting autonomous preparation...");
+    const jobPromise = jobManager.runJob(job.id, async (update) => {
+        update({ progress: 20, status_text: "Searching..." });
+        await new Promise(r => setTimeout(r, 1000));
+        update({ progress: 50, status_text: "Analyzing quality..." });
+        await new Promise(r => setTimeout(r, 1000));
+        update({ progress: 80, status_text: "Formatting export..." });
+        await new Promise(r => setTimeout(r, 1000));
+        return "data/exports/prepared_dataset.parquet";
+    });
+    // 4. Test check_job_status (Polling)
+    console.log("\nStep 4: Polling Job Status (Simulating UI Check)...");
+    for (let i = 0; i < 5; i++) {
+        const currentJob = store.getJob(job.id);
+        console.log(` - [TS +${i}s] Status: ${currentJob?.status}, Progress: ${currentJob?.progress}%, Activity: ${currentJob?.status_text}`);
+        if (currentJob?.status === "completed")
+            break;
+        await new Promise(r => setTimeout(r, 800));
+    }
+    await jobPromise;
+    console.log("\n Phase 5 tools logic verified.");
+}
+testPhase5Tools().catch(console.error);

package/build/scripts/test-media-analysis.js

@@ -0,0 +1,61 @@
+import { MediaAnalyzer } from "../quality/media-analyzer.js";
+import path from "path";
+import fs from "fs";
+import { execSync } from "child_process";
+async function runTest() {
+    console.log("--- Testing Media Quality Analysis ---");
+    const projectRoot = path.resolve(".");
+    const analyzer = new MediaAnalyzer(projectRoot);
+    // 1. Create a sample audio file using Python (sine wave)
+    const testMediaDir = path.join(projectRoot, "data", "test-media");
+    if (!fs.existsSync(testMediaDir))
+        fs.mkdirSync(testMediaDir, { recursive: true });
+    const audioPath = path.join(testMediaDir, "test_audio.wav");
+    console.log("Generating test audio file (sine wave)...");
+    const pythonScript = `
+import numpy as np
+import soundfile as sf
+sample_rate = 44100
+duration = 2.0
+frequency = 440.0
+t = np.linspace(0, duration, int(sample_rate * duration))
+audio = 0.5 * np.sin(2 * np.pi * frequency * t)
+sf.write('${audioPath.replace(/\\/g, "\\\\")}', audio, sample_rate)
+    `;
+    fs.writeFileSync(path.join(testMediaDir, "gen_audio.py"), pythonScript);
+    try {
+        execSync(`python "${path.join(testMediaDir, "gen_audio.py")}"`);
+    }
+    catch (e) {
+        console.log("⚠️  Could not generate audio (soundfile may not be installed). Skipping audio test.");
+        console.log("VERIFICATION_STATUS: ⚠️  PARTIAL (Audio generation failed)");
+        return;
+    }
+    // 2. Run Analysis
+    console.log(`Analyzing ${audioPath}...`);
+    try {
+        const report = await analyzer.analyze(audioPath);
+        console.log("Analysis Result:");
+        console.log(`- Total Files: ${report.total_files}`);
+        console.log(`- OK Files: ${report.ok_files}`);
+        if ('avg_audio_duration' in report) {
+            console.log(`- Average Audio Duration: ${report.avg_audio_duration}s`);
+            const firstAudio = report.details[0];
+            if (firstAudio && 'sample_rate' in firstAudio) {
+                console.log(`- Sample Rate: ${firstAudio.sample_rate}Hz`);
+                console.log(`- Duration: ${firstAudio.duration}s`);
+            }
+        }
+        if (report.ok_files === 1 && report.total_files === 1) {
+            console.log("\nVERIFICATION_STATUS: ✅ PASS");
+        }
+        else {
+            console.log("\nVERIFICATION_STATUS: ❌ FAIL - Incorrect stats");
+        }
+    }
+    catch (e) {
+        console.error(`Analysis failed: ${e.message}`);
+        console.log("\nVERIFICATION_STATUS: ❌ FAIL");
+    }
+}
+runTest().catch(console.error);

package/build/scripts/test-monitoring.js

@@ -0,0 +1,91 @@
+import { MetadataStore } from "../metadata/store.js";
+import { MonitoringStore } from "../metadata/monitoring-store.js";
+import { MonitoringService } from "../metadata/monitoring-service.js";
+import path from "path";
+async function runTest() {
+    const dbPath = path.resolve("data", "test-monitoring.db");
+    const metadataStore = new MetadataStore(dbPath);
+    const monitoringStore = new MonitoringStore(metadataStore.db);
+    const service = new MonitoringService(monitoringStore, metadataStore);
+    // 1. Setup mock data
+    const dsId = "test/dataset-1";
+    const initialMetadata = {
+        id: dsId,
+        source: "huggingface",
+        name: "Test Dataset",
+        description: "Initial description",
+        downloads: 100,
+        likes: 10,
+        last_updated: "2023-01-01T00:00:00Z",
+        download_url: "https://huggingface.co/datasets/test/dataset-1",
+        quality_score: 85,
+        splits: [{ name: "train", num_examples: 1000 }],
+        license: { id: "mit", category: "safe", usage_restrictions: [], warnings: [] },
+        tags: [],
+        task: "text-classification",
+        languages: ["en"],
+        is_structured: true,
+        has_target_column: true,
+        total_examples: 1000,
+        is_safe_source: true,
+        has_personal_data: false,
+        is_paywalled: false,
+        is_scraped_web_data: false,
+        uses_https: true,
+        has_train_split: true,
+        has_test_split: false,
+        has_validation_split: false,
+        description_length: 100,
+        has_readme: true,
+        quality_warnings: []
+    };
+    metadataStore.saveDataset(initialMetadata);
+    // 2. Setup Monitor and Webhook
+    monitoringStore.saveWebhook({
+        id: "slack-1",
+        name: "General Slack",
+        channel: "slack",
+        url: "https://hooks.slack.com/services/...",
+        enabled: true
+    });
+    monitoringStore.saveMonitor({
+        dataset_id: dsId,
+        enabled: true,
+        auto_reprocess: true,
+        last_checked_version: initialMetadata.last_updated,
+        webhook_ids: ["slack-1"],
+        created_at: new Date().toISOString(),
+        updated_at: new Date().toISOString()
+    });
+    console.log("--- Monitoring Setup Complete ---");
+    // 3. Simulate an update
+    const updatedMetadata = {
+        ...initialMetadata,
+        last_updated: "2023-02-01T00:00:00Z", // New version
+        downloads: 150,
+        quality_score: 90,
+        total_examples: 1200 // Significant change
+    };
+    console.log("Checking for updates...");
+    // Mock fetch function
+    const mockFetch = async (id, source) => {
+        if (id === dsId)
+            return updatedMetadata;
+        return null;
+    };
+    const diffs = await service.checkUpdates(mockFetch);
+    console.log(`Found ${diffs.length} updates.`);
+    if (diffs.length > 0) {
+        const diff = diffs[0];
+        console.log(`Version Change: ${diff.old_version} -> ${diff.new_version}`);
+        console.log(`Impact Score: ${diff.impact_score}`);
+        console.log("Changes:", JSON.stringify(diff.changes, null, 2));
+    }
+    if (diffs.length > 0 && diffs[0].impact_score > 0) {
+        console.log("\n✅ Success: Dataset update detected and diffed correctly.");
+    }
+    else {
+        console.error("\n❌ Failure: Update not detected.");
+    }
+}
+runTest().catch(console.error);