@vespermcp/mcp-server 1.0.0

package/build/metadata/institutional-scrapers.js

@@ -0,0 +1,49 @@
+import { spawn } from "child_process";
+import path from "path";
+const WB_SCRIPT_PATH = path.resolve("src", "python", "worldbank_adapter.py");
+const NASA_SCRIPT_PATH = path.resolve("src", "python", "nasa_adapter.py");
+export class WorldBankScraper {
+    async scrape(query, limit = 10) {
+        return runAdapter(WB_SCRIPT_PATH, query, limit);
+    }
+}
+export class NASAScraper {
+    async scrape(query, limit = 10) {
+        return runAdapter(NASA_SCRIPT_PATH, query, limit);
+    }
+}
+async function runAdapter(scriptPath, query, limit) {
+    return new Promise((resolve) => {
+        const pythonProcess = spawn("python", [
+            scriptPath,
+            "--action", "search",
+            "--query", query,
+            "--limit", String(limit)
+        ]);
+        let output = "";
+        let errorOutput = "";
+        pythonProcess.stdout.on("data", (data) => { output += data.toString(); });
+        pythonProcess.stderr.on("data", (data) => { errorOutput += data.toString(); });
+        pythonProcess.on("close", (code) => {
+            if (code !== 0) {
+                console.error(`[Adapter] ${path.basename(scriptPath)} exited with code ${code}: ${errorOutput}`);
+                resolve([]);
+                return;
+            }
+            try {
+                const results = JSON.parse(output);
+                if (results.error) {
+                    console.error(`[Adapter] ${path.basename(scriptPath)} error: ${results.error}`);
+                    resolve([]);
+                }
+                else {
+                    resolve(results);
+                }
+            }
+            catch (e) {
+                console.error(`[Adapter] ${path.basename(scriptPath)} JSON error: ${e.message}`);
+                resolve([]);
+            }
+        });
+    });
+}

package/build/metadata/kaggle-scraper.js

@@ -0,0 +1,182 @@
+import { categorizeLicense } from "./license.js";
+import { calculateQualityScore } from "./quality.js";
+import { classifyDomain } from "./domain.js";
+import { rateLimitedFetch, delayBetweenRequests } from "./rate-limiter.js";
+export class KaggleMetadataScraper {
+    username;
+    key;
+    constructor(username, key) {
+        this.username = username;
+        this.key = key;
+    }
+    async scrape(query, limit = 20, usePagination = true) {
+        console.error(`[Kaggle] Searching for "${query}" (limit: ${limit}, pagination: ${usePagination})...`);
+        const auth = Buffer.from(`${this.username}:${this.key}`).toString('base64');
+        const results = [];
+        const MAX_PAGE_SIZE = 100; // Kaggle API max page size
+        const pageSize = Math.min(limit, MAX_PAGE_SIZE);
+        let page = 1;
+        let totalFetched = 0;
+        let hasMore = true;
+        try {
+            while (hasMore && totalFetched < limit) {
+                const url = `https://www.kaggle.com/api/v1/datasets/list?search=${encodeURIComponent(query)}&page_size=${pageSize}&page=${page}`;
+                console.error(`[Kaggle] Fetching page ${page} (${totalFetched}/${limit} datasets so far)...`);
+                // Use rate-limited fetch with retry logic
+                const response = await rateLimitedFetch(url, {
+                    headers: {
+                        'Authorization': `Basic ${auth}`,
+                        'Content-Type': 'application/json'
+                    }
+                }, {
+                    maxRetries: 3,
+                    initialDelay: 2000, // Start with 2 seconds
+                    maxDelay: 30000 // Max 30 seconds
+                });
+                const datasets = await response.json();
+                if (!datasets || datasets.length === 0) {
+                    hasMore = false;
+                    break;
+                }
+                // Add delay between processing datasets to avoid rate limits
+                for (let i = 0; i < datasets.length; i++) {
+                    const ds = datasets[i];
+                    try {
+                        const metadata = this.transform(ds);
+                        results.push(metadata);
+                        totalFetched++;
+                        console.error(`[Kaggle] Added: ${ds.ref} (${ds.downloadCount} downloads)`);
+                        // Add small delay every 5 datasets
+                        if ((i + 1) % 5 === 0 && i < datasets.length - 1) {
+                            await delayBetweenRequests(500);
+                        }
+                    }
+                    catch (e) {
+                        console.error(`[Kaggle] ERROR: Failed to transform ${ds.ref}:`, e);
+                    }
+                }
+                // Check if we should continue pagination
+                if (usePagination && datasets.length === pageSize && totalFetched < limit) {
+                    page++;
+                    // Add delay between pages to avoid rate limits
+                    await delayBetweenRequests(1000);
+                }
+                else {
+                    hasMore = false;
+                }
+            }
+            console.error(`[Kaggle] Completed: ${results.length} datasets found for "${query}"`);
+            return results;
+        }
+        catch (e) {
+            // Handle rate limit errors specifically
+            if (e?.status === 429 || e?.message?.includes('rate limit')) {
+                console.error("[Kaggle] Rate limit error:", e.message);
+                console.error("Consider adding delays between requests or reducing batch size");
+            }
+            else {
+                console.error("[Kaggle] Scrape error:", e.message || e);
+            }
+            // Return partial results if we got some before the error
+            if (results.length > 0) {
+                console.error(`[Kaggle] Returning ${results.length} partial results before error`);
+            }
+            return results;
+        }
+    }
+    transform(ds) {
+        const repoId = ds.ref;
+        const tags = ds.tags?.map(t => t.name) || [];
+        const description = ds.description || "";
+        const license = categorizeLicense(ds.licenseName);
+        const warnings = [];
+        // Kaggle doesn't give us splits in the list API easily
+        const sizeBytes = this.parseSize(ds.size);
+        const splits = [
+            {
+                name: "data",
+                num_examples: 0,
+                size_bytes: sizeBytes
+            }
+        ];
+        const totalSizeMB = sizeBytes ? Math.round(sizeBytes / (1024 * 1024) * 100) / 100 : 0;
+        // Populate warnings
+        if (description.length < 100)
+            warnings.push("Short description; results may be less relevant");
+        const lastUpdatedDate = new Date(ds.lastUpdated);
+        const fourYearsAgo = new Date();
+        fourYearsAgo.setFullYear(fourYearsAgo.getFullYear() - 4);
+        if (lastUpdatedDate < fourYearsAgo) {
+            warnings.push(`Stale data: Last updated ${lastUpdatedDate.getFullYear()}`);
+        }
+        warnings.push("No specific data splits identified (Kaggle API limitation)");
+        // Classify domain
+        const task = this.extractTask(tags);
+        const domain = classifyDomain(description, tags, repoId, task);
+        return {
+            id: repoId,
+            source: "kaggle",
+            name: ds.title,
+            description: description,
+            quality_warnings: warnings,
+            downloads: ds.downloadCount,
+            likes: ds.voteCount,
+            stars: 0,
+            tags: tags,
+            last_updated: ds.lastUpdated,
+            task: task,
+            domain: domain,
+            languages: [],
+            splits,
+            license,
+            quality_score: calculateQualityScore({
+                downloads: ds.downloadCount,
+                likes: ds.voteCount,
+                hasDescription: description.length > 50,
+                descriptionLength: description.length,
+                hasTrainSplit: false,
+                hasTestSplit: false,
+                lastUpdated: ds.lastUpdated,
+                licenseCategory: license.category
+            }),
+            download_url: `https://www.kaggle.com/datasets/${ds.ref}`,
+            format: undefined,
+            total_examples: 0,
+            total_size_bytes: sizeBytes,
+            total_size_mb: totalSizeMB,
+            columns: [],
+            is_structured: false,
+            has_target_column: false,
+            is_safe_source: true,
+            has_personal_data: false,
+            is_paywalled: false,
+            is_scraped_web_data: false,
+            uses_https: true,
+            has_train_split: false,
+            has_test_split: false,
+            has_validation_split: false,
+            description_length: description.length,
+            has_readme: true
+        };
+    }
+    parseSize(sizeStr) {
+        if (!sizeStr)
+            return 0;
+        const match = sizeStr.match(/^(\d+(?:\.\d+)?)\s*([KMGT]B)$/i);
+        if (!match)
+            return 0;
+        const value = parseFloat(match[1]);
+        const unit = match[2].toUpperCase();
+        switch (unit) {
+            case 'KB': return value * 1024;
+            case 'MB': return value * 1024 * 1024;
+            case 'GB': return value * 1024 * 1024 * 1024;
+            case 'TB': return value * 1024 * 1024 * 1024 * 1024;
+            default: return value;
+        }
+    }
+    extractTask(tags) {
+        // Similar to HF but Kaggle tags might be different
+        return "unknown";
+    }
+}

package/build/metadata/license.js

@@ -0,0 +1,68 @@
+const SAFE_KEYWORDS = ["mit", "apache", "bsd", "cc0", "cc-by-4.0", "cc-by-sa-4.0", "odc-by", "pddl", "openrail", "creative commons attribution 4.0", "public domain"];
+const RESTRICTED_KEYWORDS = ["nc", "non-commercial", "research-only", "academic", "gpl", "agpl", "proprietary", "custom"];
+// Permissive licenses for MVP filter
+const PERMISSIVE_LICENSES = ["mit", "apache", "apache-2.0", "bsd", "cc0", "cc-by-4.0", "odc-by", "pddl", "openrail"];
+export function categorizeLicense(licenseId, licenseUrl) {
+    const id = (licenseId || "unknown").toLowerCase();
+    const usageRestrictions = [];
+    let requiresConsent = false;
+    // Check for usage restrictions
+    if (id.includes("nc") || id.includes("non-commercial")) {
+        usageRestrictions.push("non-commercial");
+    }
+    if (id.includes("research-only") || id.includes("academic")) {
+        usageRestrictions.push("academic-only");
+    }
+    if (id.includes("nd") || id.includes("no-derivatives")) {
+        usageRestrictions.push("no-derivatives");
+    }
+    if (id.includes("gpl") || id.includes("agpl")) {
+        usageRestrictions.push("no-derivatives"); // GPL requires derivative works to be GPL
+    }
+    // Check if consent is required (GDPR, Kaggle, etc.)
+    if (id.includes("gdpr") || id.includes("consent") || id.includes("kaggle")) {
+        requiresConsent = true;
+    }
+    // If ID contains restricted keywords
+    if (RESTRICTED_KEYWORDS.some(k => id.includes(k))) {
+        return {
+            id,
+            category: "restricted",
+            commercial_use: false,
+            usage_restrictions: usageRestrictions.length > 0 ? usageRestrictions : ["non-commercial"],
+            url: licenseUrl,
+            warnings: [
+                "Restricted usage terms apply",
+                "Verify license terms before commercial application",
+            ],
+            requires_consent: requiresConsent,
+        };
+    }
+    // If ID is a common safe license
+    if (SAFE_KEYWORDS.some(k => id.includes(k))) {
+        return {
+            id,
+            category: "safe",
+            commercial_use: true,
+            usage_restrictions: [],
+            url: licenseUrl,
+            warnings: [],
+            requires_consent: requiresConsent,
+        };
+    }
+    return {
+        id: id || "unknown",
+        category: "unknown",
+        usage_restrictions: usageRestrictions,
+        url: licenseUrl,
+        warnings: [
+            "License information unclear or unknown",
+            "Use at your own risk",
+        ],
+        requires_consent: requiresConsent,
+    };
+}
+export function isPermissiveLicense(licenseId) {
+    const id = (licenseId || "unknown").toLowerCase();
+    return PERMISSIVE_LICENSES.some(perm => id.includes(perm));
+}

package/build/metadata/monitoring-service.js

@@ -0,0 +1,107 @@
+export class MonitoringService {
+    monitorStore;
+    metadataStore;
+    constructor(monitorStore, metadataStore) {
+        this.monitorStore = monitorStore;
+        this.metadataStore = metadataStore;
+    }
+    /**
+     * Checks all active monitors for updates.
+     * @param fetchLatest A function that fetches the latest metadata from the source (HF/Kaggle)
+     */
+    async checkUpdates(fetchLatest) {
+        const monitors = this.monitorStore.getActiveMonitors();
+        const results = [];
+        for (const monitor of monitors) {
+            const current = this.metadataStore.getDataset(monitor.dataset_id);
+            if (!current)
+                continue;
+            const latest = await fetchLatest(monitor.dataset_id, current.source);
+            if (!latest)
+                continue;
+            if (latest.last_updated !== monitor.last_checked_version) {
+                const diff = this.compareVersions(current, latest);
+                if (diff.changes.length > 0) {
+                    results.push(diff);
+                    await this.notify(monitor, diff);
+                    // Update monitor
+                    monitor.last_checked_version = latest.last_updated;
+                    monitor.updated_at = new Date().toISOString();
+                    this.monitorStore.saveMonitor(monitor);
+                    // Update store
+                    this.metadataStore.saveDataset(latest);
+                    if (monitor.auto_reprocess) {
+                        await this.triggerReprocess(monitor.dataset_id);
+                    }
+                }
+            }
+        }
+        return results;
+    }
+    compareVersions(oldVer, newVer) {
+        const changes = [];
+        // Check for significant field changes
+        const fieldsToTrack = ["downloads", "likes", "total_examples", "total_size_mb", "quality_score"];
+        for (const field of fieldsToTrack) {
+            if (oldVer[field] !== newVer[field]) {
+                changes.push({
+                    field: String(field),
+                    old_value: oldVer[field],
+                    new_value: newVer[field]
+                });
+            }
+        }
+        // Check for split changes
+        if (JSON.stringify(oldVer.splits) !== JSON.stringify(newVer.splits)) {
+            changes.push({
+                field: "splits",
+                old_value: oldVer.splits,
+                new_value: newVer.splits
+            });
+        }
+        return {
+            dataset_id: oldVer.id,
+            old_version: oldVer.last_updated,
+            new_version: newVer.last_updated,
+            changes,
+            impact_score: this.calculateImpact(changes)
+        };
+    }
+    calculateImpact(changes) {
+        let score = 0;
+        for (const change of changes) {
+            if (change.field === "total_examples")
+                score += 40;
+            if (change.field === "splits")
+                score += 30;
+            if (change.field === "quality_score")
+                score += 20;
+            if (change.field === "total_size_mb")
+                score += 10;
+        }
+        return Math.min(score, 100);
+    }
+    async notify(monitor, diff) {
+        for (const webhookId of monitor.webhook_ids) {
+            const webhook = this.monitorStore.getWebhook(webhookId);
+            if (webhook && webhook.enabled) {
+                await this.sendToWebhook(webhook, diff);
+            }
+        }
+    }
+    async sendToWebhook(webhook, diff) {
+        console.log(`[MonitoringService] Sending notification to ${webhook.name} (${webhook.channel}) for dataset ${diff.dataset_id}`);
+        // In a real implementation, this would be an HTTP POST
+        // For now, we simulate the payload
+        const payload = {
+            text: `Dataset ${diff.dataset_id} updated!`,
+            changes: diff.changes,
+            impact: diff.impact_score
+        };
+        // await axios.post(webhook.url, payload);
+    }
+    async triggerReprocess(datasetId) {
+        console.log(`[MonitoringService] Auto-reprocessing dataset ${datasetId}...`);
+        // This would call IngestionService or similar
+    }
+}

package/build/metadata/monitoring-store.js

@@ -0,0 +1,78 @@
+export class MonitoringStore {
+    db;
+    constructor(db) {
+        this.db = db;
+        this.init();
+    }
+    init() {
+        this.db.exec(`
+            CREATE TABLE IF NOT EXISTS dataset_monitors (
+                dataset_id TEXT PRIMARY KEY,
+                enabled BOOLEAN DEFAULT 1,
+                auto_reprocess BOOLEAN DEFAULT 0,
+                last_checked_version TEXT,
+                webhook_ids TEXT, -- JSON array
+                created_at TEXT,
+                updated_at TEXT
+            );
+
+            CREATE TABLE IF NOT EXISTS webhook_configs (
+                id TEXT PRIMARY KEY,
+                name TEXT,
+                channel TEXT,
+                url TEXT,
+                enabled BOOLEAN DEFAULT 1
+            );
+        `);
+    }
+    saveMonitor(monitor) {
+        const upsert = this.db.prepare(`
+            INSERT INTO dataset_monitors (dataset_id, enabled, auto_reprocess, last_checked_version, webhook_ids, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(dataset_id) DO UPDATE SET
+                enabled=excluded.enabled,
+                auto_reprocess=excluded.auto_reprocess,
+                last_checked_version=excluded.last_checked_version,
+                webhook_ids=excluded.webhook_ids,
+                updated_at=excluded.updated_at
+        `);
+        upsert.run(monitor.dataset_id, monitor.enabled ? 1 : 0, monitor.auto_reprocess ? 1 : 0, monitor.last_checked_version || null, JSON.stringify(monitor.webhook_ids), monitor.created_at, monitor.updated_at);
+    }
+    getMonitor(datasetId) {
+        const row = this.db.prepare("SELECT * FROM dataset_monitors WHERE dataset_id = ?").get(datasetId);
+        if (!row)
+            return null;
+        return {
+            ...row,
+            enabled: Boolean(row.enabled),
+            auto_reprocess: Boolean(row.auto_reprocess),
+            webhook_ids: JSON.parse(row.webhook_ids)
+        };
+    }
+    getActiveMonitors() {
+        const rows = this.db.prepare("SELECT * FROM dataset_monitors WHERE enabled = 1").all();
+        return rows.map(row => ({
+            ...row,
+            enabled: Boolean(row.enabled),
+            auto_reprocess: Boolean(row.auto_reprocess),
+            webhook_ids: JSON.parse(row.webhook_ids)
+        }));
+    }
+    saveWebhook(config) {
+        const upsert = this.db.prepare(`
+            INSERT INTO webhook_configs (id, name, channel, url, enabled)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(id) DO UPDATE SET
+                name=excluded.name,
+                url=excluded.url,
+                enabled=excluded.enabled
+        `);
+        upsert.run(config.id, config.name, config.channel, config.url, config.enabled ? 1 : 0);
+    }
+    getWebhook(id) {
+        const row = this.db.prepare("SELECT * FROM webhook_configs WHERE id = ?").get(id);
+        if (!row)
+            return null;
+        return { ...row, enabled: Boolean(row.enabled) };
+    }
+}

package/build/metadata/monitoring-types.js

@@ -0,0 +1 @@
+export {};

package/build/metadata/quality.js

@@ -0,0 +1,48 @@
+/**
+ * Calculates a quality score from 0-100 based on metadata.
+ */
+export function calculateQualityScore(data) {
+    let score = 0;
+    // 1. Popularity (max 30)
+    if (data.downloads > 10000)
+        score += 30;
+    else if (data.downloads > 1000)
+        score += 20;
+    else if (data.downloads > 100)
+        score += 10;
+    // 2. Structuredness (max 20)
+    if (data.hasTrainSplit)
+        score += 10;
+    if (data.hasTestSplit)
+        score += 10;
+    // 3. Documentation (max 20)
+    if (data.hasDescription) {
+        if (data.descriptionLength > 1000)
+            score += 20;
+        else if (data.descriptionLength > 200)
+            score += 10;
+        else
+            score += 5;
+    }
+    // 4. Recency (max 15)
+    const lastUpdate = new Date(data.lastUpdated);
+    const now = new Date();
+    const diffDays = Math.floor((now.getTime() - lastUpdate.getTime()) / (1000 * 3600 * 24));
+    if (diffDays < 180)
+        score += 15; // 6 months
+    else if (diffDays < 365)
+        score += 10; // 1 year
+    else if (diffDays < 730)
+        score += 5; // 2 years
+    // 5. License Clarity (max 10)
+    if (data.licenseCategory === "safe")
+        score += 10;
+    else if (data.licenseCategory === "restricted")
+        score += 5;
+    // 6. Community (max 5)
+    if (data.likes > 50)
+        score += 5;
+    else if (data.likes > 10)
+        score += 2;
+    return Math.min(100, score);
+}

package/build/metadata/rate-limiter.js

@@ -0,0 +1,128 @@
+/**
+ * Rate limiting and retry utilities for API requests
+ */
+const DEFAULT_OPTIONS = {
+    maxRetries: 5,
+    initialDelay: 1000, // 1 second
+    maxDelay: 60000, // 60 seconds
+    exponentialBase: 2,
+    jitter: true
+};
+/**
+ * Calculate delay with exponential backoff and optional jitter
+ */
+function calculateDelay(attempt, options) {
+    const exponentialDelay = options.initialDelay * Math.pow(options.exponentialBase, attempt);
+    const delay = Math.min(exponentialDelay, options.maxDelay);
+    if (options.jitter) {
+        // Add random jitter (0-20% of delay) to avoid thundering herd
+        const jitterAmount = delay * 0.2 * Math.random();
+        return Math.floor(delay + jitterAmount);
+    }
+    return Math.floor(delay);
+}
+/**
+ * Extract Retry-After header value from response or error
+ */
+function getRetryAfter(response) {
+    if (!response)
+        return null;
+    const retryAfter = response.headers.get('Retry-After');
+    if (!retryAfter)
+        return null;
+    // Retry-After can be a number of seconds or an HTTP date
+    const seconds = parseInt(retryAfter, 10);
+    if (!isNaN(seconds)) {
+        return seconds * 1000; // Convert to milliseconds
+    }
+    // Try parsing as HTTP date
+    const date = Date.parse(retryAfter);
+    if (!isNaN(date)) {
+        return Math.max(0, date - Date.now());
+    }
+    return null;
+}
+/**
+ * Check if error is a rate limit error (429)
+ */
+function isRateLimitError(error) {
+    if (error?.status === 429)
+        return true;
+    if (error?.response?.status === 429)
+        return true;
+    if (error?.message?.includes('rate limit'))
+        return true;
+    if (error?.message?.includes('429'))
+        return true;
+    return false;
+}
+/**
+ * Sleep for specified milliseconds
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Retry a function with exponential backoff on rate limit errors
+ */
+export async function retryWithBackoff(fn, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    let lastError;
+    let response = null;
+    for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
+        try {
+            const result = await fn();
+            return result;
+        }
+        catch (error) {
+            lastError = error;
+            // Extract response if available
+            if (error?.response) {
+                response = error.response;
+            }
+            // Only retry on rate limit errors
+            if (!isRateLimitError(error)) {
+                throw error;
+            }
+            // Don't retry on last attempt
+            if (attempt >= opts.maxRetries) {
+                break;
+            }
+            // Calculate delay
+            let delay = getRetryAfter(response);
+            if (!delay) {
+                delay = calculateDelay(attempt, opts);
+            }
+            console.error(`[Rate Limiter] Rate limited (attempt ${attempt + 1}/${opts.maxRetries + 1}). Waiting ${delay}ms...`);
+            await sleep(delay);
+        }
+    }
+    throw lastError;
+}
+/**
+ * Add a delay between requests to avoid hitting rate limits
+ */
+export async function delayBetweenRequests(ms = 500) {
+    await sleep(ms);
+}
+/**
+ * Rate-limited fetch wrapper with automatic retry
+ */
+export async function rateLimitedFetch(url, options = {}, retryOptions = {}) {
+    return retryWithBackoff(async () => {
+        const response = await fetch(url, options);
+        if (response.status === 429) {
+            const error = new Error(`Rate limit exceeded: ${response.status}`);
+            error.status = 429;
+            error.response = response;
+            throw error;
+        }
+        if (!response.ok) {
+            const error = new Error(`HTTP error: ${response.status}`);
+            error.status = response.status;
+            error.response = response;
+            throw error;
+        }
+        return response;
+    }, retryOptions);
+}