@vespermcp/mcp-server 1.0.0

package/build/ingestion/hf-downloader.js

@@ -0,0 +1,64 @@
+import { listFiles } from "@huggingface/hub";
+import { RobustDownloader } from "../utils/downloader.js";
+export class HFDownloader {
+    hfToken;
+    downloader;
+    constructor(token) {
+        this.hfToken = token || process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
+        this.downloader = new RobustDownloader();
+    }
+    /**
+     * Finds the most suitable data file in a repository
+     * Returns the relative path within the repo
+     */
+    async findBestFile(repoId) {
+        try {
+            const files = [];
+            for await (const file of listFiles({
+                repo: { type: "dataset", name: repoId },
+                recursive: true,
+                ...(this.hfToken ? { accessToken: this.hfToken } : {})
+            })) {
+                if (file.type === "file") {
+                    files.push(file.path);
+                }
+            }
+            // Priority logic for data scientists
+            const priorities = [
+                /train.*\.parquet$/i,
+                /data.*\.parquet$/i,
+                /.*\.parquet$/i,
+                /train.*\.csv$/i,
+                /data.*\.csv$/i,
+                /.*\.csv$/i,
+                /.*\.jsonl$/i,
+                /.*\.json$/i
+            ];
+            for (const pattern of priorities) {
+                const match = files.find(f => pattern.test(f));
+                if (match)
+                    return match;
+            }
+            return files.length > 0 ? files[0] : null;
+        }
+        catch (error) {
+            console.error(`[HF] Failed to list files for ${repoId}:`, error.message);
+            return null;
+        }
+    }
+    /**
+     * Downloads a file from HF to local path
+     */
+    async download(repoId, filePath, targetPath, onProgress) {
+        const url = `https://huggingface.co/datasets/${repoId}/resolve/main/${filePath}`;
+        await this.downloader.download(url, targetPath, {
+            headers: this.hfToken ? { 'Authorization': `Bearer ${this.hfToken}` } : {},
+            resume: true,
+            onProgress: (bytes, total) => {
+                if (total > 0 && onProgress) {
+                    onProgress(Math.round((bytes / total) * 100));
+                }
+            }
+        });
+    }
+}

package/build/ingestion/ingestor.js

@@ -0,0 +1,96 @@
+import path from "path";
+import fs from "fs";
+import { HFDownloader } from "./hf-downloader.js";
+import { KaggleDownloader } from "./kaggle-downloader.js";
+export class DataIngestor {
+    projectRoot;
+    store;
+    rawDataDir;
+    hfDownloader;
+    kaggleDownloader;
+    constructor(projectRoot, store) {
+        this.projectRoot = projectRoot;
+        this.store = store;
+        this.rawDataDir = path.join(this.projectRoot, "data", "raw");
+        if (!fs.existsSync(this.rawDataDir)) {
+            fs.mkdirSync(this.rawDataDir, { recursive: true });
+        }
+        this.hfDownloader = new HFDownloader();
+        this.kaggleDownloader = new KaggleDownloader();
+    }
+    /**
+     * Ensures a dataset is available locally
+     */
+    async ensureData(datasetId, source, onProgress) {
+        // 1. Check database for existing download
+        const status = this.store.getDownloadStatus(datasetId);
+        if (status && status.status === 'completed' && fs.existsSync(status.local_path)) {
+            return status.local_path;
+        }
+        if (status && status.status === 'downloading') {
+            console.log(`[Ingestor] Dataset ${datasetId} status is 'downloading'. Attempting to resume/concurrently monitor.`);
+            // In a better system we'd use a lock, but for now we let it resume 
+            // the RobustDownloader handles the actual file locking/range logic.
+        }
+        // 2. Trigger source-specific download
+        if (source === "huggingface") {
+            onProgress?.("Discovering data files on HuggingFace Hub...");
+            const remotePath = await this.hfDownloader.findBestFile(datasetId);
+            if (!remotePath)
+                throw new Error(`No suitable data files found in HuggingFace repo: ${datasetId}`);
+            const ext = path.extname(remotePath).substring(1).toLowerCase() || "csv";
+            const targetPath = this.getTargetPath(datasetId, ext);
+            this.store.registerDownload(datasetId, targetPath, "downloading");
+            try {
+                await this.hfDownloader.download(datasetId, remotePath, targetPath, (progress) => {
+                    onProgress?.(`Downloading ${path.basename(remotePath)}...`, progress);
+                });
+                const stats = fs.statSync(targetPath);
+                this.completeDownload(datasetId, targetPath, stats.size);
+                return targetPath;
+            }
+            catch (e) {
+                this.failDownload(datasetId, e.message);
+                throw e;
+            }
+        }
+        else if (source === "kaggle") {
+            const safeId = datasetId.replace("kaggle:", "").replace(/\//g, "_");
+            const targetDir = path.join(this.rawDataDir, `kaggle_${safeId}`);
+            this.store.registerDownload(datasetId, targetDir, "downloading");
+            try {
+                const primaryFile = await this.kaggleDownloader.download(datasetId.replace("kaggle:", ""), targetDir, (progress) => {
+                    onProgress?.("Downloading Kaggle archive...", progress);
+                });
+                const stats = fs.statSync(primaryFile);
+                this.completeDownload(datasetId, primaryFile, stats.size);
+                return primaryFile;
+            }
+            catch (e) {
+                this.failDownload(datasetId, e.message);
+                throw e;
+            }
+        }
+        throw new Error(`Download logic for ${source} not yet implemented`);
+    }
+    /**
+     * Register a successful download
+     */
+    completeDownload(datasetId, actualPath, sizeBytes) {
+        this.store.registerDownload(datasetId, actualPath, 'completed', sizeBytes);
+    }
+    /**
+     * Register a failed download
+     */
+    failDownload(datasetId, error) {
+        const existing = this.store.getDownloadStatus(datasetId);
+        this.store.registerDownload(datasetId, existing?.local_path || "", 'failed', 0, error);
+    }
+    /**
+     * Generates a safe local filename for a dataset ID
+     */
+    getTargetPath(datasetId, extension = "csv") {
+        const safeId = datasetId.replace(/\//g, "_").replace(/:/g, "_");
+        return path.join(this.rawDataDir, `${safeId}.${extension}`);
+    }
+}

package/build/ingestion/kaggle-downloader.js

@@ -0,0 +1,79 @@
+import fs from "fs";
+import path from "path";
+import AdmZip from "adm-zip";
+import { RobustDownloader } from "../utils/downloader.js";
+export class KaggleDownloader {
+    username;
+    key;
+    downloader;
+    constructor(username, key) {
+        this.username = username || process.env.KAGGLE_USERNAME || "";
+        this.key = key || process.env.KAGGLE_KEY || "";
+        this.downloader = new RobustDownloader();
+    }
+    /**
+     * Downloads and extracts a Kaggle dataset
+     * returns the path to the primary data file
+     */
+    async download(repoId, targetDir, onProgress) {
+        if (!this.username || !this.key) {
+            throw new Error("Kaggle credentials missing (KAGGLE_USERNAME, KAGGLE_KEY)");
+        }
+        const auth = Buffer.from(`${this.username}:${this.key}`).toString('base64');
+        const url = `https://www.kaggle.com/api/v1/datasets/download/${repoId}`;
+        // Ensure target directory exists
+        if (!fs.existsSync(targetDir)) {
+            fs.mkdirSync(targetDir, { recursive: true });
+        }
+        const zipPath = path.join(targetDir, "data.zip");
+        await this.downloader.download(url, zipPath, {
+            headers: {
+                'Authorization': `Basic ${auth}`
+            },
+            resume: true,
+            onProgress: (bytes, total) => {
+                if (total > 0 && onProgress) {
+                    onProgress(Math.round((bytes / total) * 100));
+                }
+            }
+        });
+        // Unzip
+        const zip = new AdmZip(zipPath);
+        zip.extractAllTo(targetDir, true);
+        // Find best file
+        const extractedFiles = this.getAllFiles(targetDir);
+        const priorities = [
+            /.*\.parquet$/i,
+            /.*\.csv$/i,
+            /.*\.jsonl$/i,
+            /.*\.json$/i
+        ];
+        let bestFile = null;
+        for (const pattern of priorities) {
+            const match = extractedFiles.find(f => pattern.test(f) && !f.endsWith(".zip"));
+            if (match) {
+                bestFile = match;
+                break;
+            }
+        }
+        // Cleanup ZIP
+        fs.unlinkSync(zipPath);
+        if (!bestFile) {
+            throw new Error("No suitable data files found in Kaggle archive");
+        }
+        return bestFile;
+    }
+    getAllFiles(dir, allFiles = []) {
+        const files = fs.readdirSync(dir);
+        files.forEach(file => {
+            const name = path.join(dir, file);
+            if (fs.statSync(name).isDirectory()) {
+                this.getAllFiles(name, allFiles);
+            }
+            else {
+                allFiles.push(name);
+            }
+        });
+        return allFiles;
+    }
+}

package/build/install/install-service.js

@@ -0,0 +1,41 @@
+import fs from "fs";
+import path from "path";
+export class InstallService {
+    projectRoot;
+    metadataStore;
+    constructor(projectRoot, metadataStore) {
+        this.projectRoot = projectRoot;
+        this.metadataStore = metadataStore;
+    }
+    /**
+     * Installs a prepared dataset file into the ./datasets directory
+     * @param datasetId The ID of the dataset
+     * @param sourcePath The current location of the processed file
+     * @returns The absolute path to the installed file
+     */
+    async install(datasetId, sourcePath, targetDir) {
+        if (!fs.existsSync(sourcePath)) {
+            throw new Error(`Source file not found for installation: ${sourcePath}`);
+        }
+        const dataset = this.metadataStore.getDataset(datasetId);
+        if (!dataset) {
+            throw new Error(`Dataset metadata not found for ${datasetId}`);
+        }
+        // Create target directory
+        const sanitizedName = dataset.name.replace(/[^a-z0-9]/gi, "_").toLowerCase();
+        const installDir = targetDir || path.join(this.projectRoot, "datasets", sanitizedName);
+        if (!fs.existsSync(installDir)) {
+            fs.mkdirSync(installDir, { recursive: true });
+        }
+        const extension = path.extname(sourcePath);
+        const targetFilename = `${sanitizedName}${extension}`;
+        const targetPath = path.join(installDir, targetFilename);
+        // Copy file
+        fs.copyFileSync(sourcePath, targetPath);
+        // Update metadata
+        const absolutePath = path.resolve(targetPath);
+        this.metadataStore.updateInstallPath(datasetId, absolutePath);
+        console.log(`[InstallService] Dataset ${datasetId} installed to ${absolutePath}`);
+        return absolutePath;
+    }
+}

package/build/jobs/manager.js

@@ -0,0 +1,129 @@
+import { v4 as uuidv4 } from "uuid";
+import { EventEmitter } from "events";
+import { QueueService } from "./queue.js";
+import { ObservabilityService } from "../monitoring/observability.js";
+export class JobManager extends EventEmitter {
+    store;
+    static instance;
+    queue;
+    stats;
+    maxConcurrency = 2;
+    activeWorkers = 0;
+    constructor(store, stats) {
+        super();
+        this.store = store;
+        this.queue = new QueueService();
+        this.stats = stats || new ObservabilityService();
+    }
+    static getInstance(store, stats) {
+        if (!JobManager.instance) {
+            JobManager.instance = new JobManager(store, stats);
+        }
+        return JobManager.instance;
+    }
+    /**
+     * Set max parallel workers
+     */
+    setConcurrency(count) {
+        this.maxConcurrency = count;
+    }
+    /**
+     * Create a new job and add it to the priority queue
+     */
+    createJob(type, priority = 0, metadata, maxAttempts = 3) {
+        const now = new Date().toISOString();
+        const job = {
+            id: uuidv4(),
+            type,
+            status: "pending",
+            priority,
+            progress: 0,
+            status_text: "Job created",
+            attempts: 0,
+            max_attempts: maxAttempts,
+            created_at: now,
+            updated_at: now,
+            metadata: metadata ? JSON.stringify(metadata) : undefined
+        };
+        this.store.saveJob(job);
+        this.queue.enqueue(job);
+        this.emit("jobCreated", job);
+        // Start processing background queue
+        this.processQueue();
+        return job;
+    }
+    /**
+     * Background loop to process queued jobs
+     */
+    async processQueue() {
+        if (this.activeWorkers >= this.maxConcurrency)
+            return;
+        const job = this.queue.dequeue();
+        if (!job)
+            return;
+        this.activeWorkers++;
+        this.updateJob(job.id, { status: "running", status_text: "Picked up by worker" });
+        const startTime = Date.now();
+        // In a real system, we'd have a registry of handlers for each JobType.
+        // For now, we emit an event so the orchestrator can run it.
+        this.emit("processJob", job, async (task) => {
+            try {
+                const resultUrl = await task();
+                const duration = Date.now() - startTime;
+                this.updateJob(job.id, {
+                    status: "completed",
+                    progress: 100,
+                    status_text: "Completed successfully",
+                    result_url: resultUrl || undefined
+                });
+                this.stats.recordJobSuccess(job.type, duration);
+            }
+            catch (error) {
+                if (this.queue.shouldRetry(job)) {
+                    this.updateJob(job.id, {
+                        status: "retrying",
+                        status_text: `Failed: ${error.message}. Retrying...`
+                    });
+                }
+                else {
+                    this.updateJob(job.id, {
+                        status: "failed",
+                        status_text: "Max retries exceeded",
+                        error: error.message || String(error)
+                    });
+                    this.stats.recordJobFailure(job.type, error);
+                }
+            }
+            finally {
+                this.queue.finalize(job.id);
+                this.activeWorkers--;
+                this.processQueue(); // Look for next job
+            }
+        });
+        // Try to start another worker if capacity allows
+        this.processQueue();
+    }
+    /**
+     * Helper to run a task as a job with automatic status updates
+     */
+    runJob(id, task) {
+        this.emit("processJob", { id }, async () => {
+            return await task((updates) => this.updateJob(id, updates));
+        });
+    }
+    /**
+     * Update job status and progress
+     */
+    updateJob(id, updates) {
+        const job = this.store.getJob(id);
+        if (!job)
+            return;
+        const updatedJob = {
+            ...job,
+            ...updates,
+            updated_at: new Date().toISOString()
+        };
+        this.store.saveJob(updatedJob);
+        this.emit("jobUpdated", updatedJob);
+    }
+}

package/build/jobs/queue.js

@@ -0,0 +1,59 @@
+export class QueueService {
+    queue = [];
+    running = new Map();
+    /**
+     * Adds a job to the queue, sorted by priority.
+     */
+    enqueue(job) {
+        job.status = "queued";
+        this.queue.push(job);
+        this.sortQueue();
+    }
+    /**
+     * Gets the next job from the queue.
+     */
+    dequeue() {
+        const job = this.queue.shift();
+        if (job) {
+            job.status = "running";
+            this.running.set(job.id, job);
+        }
+        return job;
+    }
+    /**
+     * Mark a job as finished (completed or failed permanently).
+     */
+    finalize(id) {
+        this.running.delete(id);
+    }
+    /**
+     * Handles retry logic with exponential backoff (simulated).
+     */
+    shouldRetry(job) {
+        if (job.attempts < job.max_attempts) {
+            job.attempts++;
+            job.status = "retrying";
+            // In a real system, we'd schedule this for later.
+            // For now, we'll re-enqueue it immediately.
+            this.enqueue(job);
+            return true;
+        }
+        return false;
+    }
+    getStats() {
+        return {
+            pending: this.queue.length,
+            running: this.running.size,
+            total: this.queue.length + this.running.size
+        };
+    }
+    sortQueue() {
+        // Priority high to low, then created_at old to new
+        this.queue.sort((a, b) => {
+            if (b.priority !== a.priority) {
+                return b.priority - a.priority;
+            }
+            return new Date(a.created_at).getTime() - new Date(b.created_at).getTime();
+        });
+    }
+}

package/build/jobs/types.js

@@ -0,0 +1 @@
+export {};

package/build/metadata/domain.js

@@ -0,0 +1,147 @@
+const DOMAIN_KEYWORDS = [
+    {
+        domain: "medicine",
+        keywords: ["medical", "medicine", "clinical", "diagnosis", "patient", "disease", "symptom", "treatment", "drug", "pharmaceutical", "healthcare", "hospital", "doctor", "physician", "medical imaging", "radiology", "pathology", "oncology", "cardiology"],
+        weight: 10
+    },
+    {
+        domain: "healthcare",
+        keywords: ["health", "healthcare", "wellness", "public health", "epidemiology", "biomedical", "health data", "medical records", "ehr", "electronic health"],
+        weight: 8
+    },
+    {
+        domain: "security",
+        keywords: ["security", "cybersecurity", "malware", "vulnerability", "threat", "attack", "defense", "encryption", "privacy", "authentication", "authorization", "penetration testing", "intrusion"],
+        weight: 10
+    },
+    {
+        domain: "finance",
+        keywords: ["finance", "financial", "banking", "trading", "stock", "market", "investment", "credit", "loan", "mortgage", "cryptocurrency", "bitcoin", "blockchain", "accounting"],
+        weight: 9
+    },
+    {
+        domain: "education",
+        keywords: ["education", "learning", "student", "academic", "curriculum", "pedagogy", "teaching", "school", "university", "course", "tutorial"],
+        weight: 8
+    },
+    {
+        domain: "science",
+        keywords: ["scientific", "research", "experiment", "laboratory", "physics", "chemistry", "biology", "astronomy", "geology", "environmental"],
+        weight: 7
+    },
+    {
+        domain: "technology",
+        keywords: ["software", "programming", "code", "algorithm", "system", "application", "development", "engineering", "technical"],
+        weight: 5
+    },
+    {
+        domain: "social",
+        keywords: ["social", "society", "community", "demographic", "census", "population", "sociology", "anthropology", "culture"],
+        weight: 6
+    },
+    {
+        domain: "legal",
+        keywords: ["legal", "law", "court", "judicial", "litigation", "contract", "regulation", "compliance", "legislation"],
+        weight: 9
+    },
+    {
+        domain: "business",
+        keywords: ["business", "corporate", "enterprise", "company", "organization", "management", "marketing", "sales", "customer"],
+        weight: 6
+    },
+    {
+        domain: "multimedia",
+        keywords: ["image", "video", "audio", "multimedia", "visual", "photography", "music", "sound"],
+        weight: 7
+    },
+    {
+        domain: "nlp",
+        keywords: ["natural language", "nlp", "text", "language model", "translation", "sentiment", "text classification", "named entity", "ner", "question answering", "qa"],
+        weight: 8
+    },
+    {
+        domain: "computer-vision",
+        keywords: ["computer vision", "cv", "image classification", "object detection", "segmentation", "visual", "optical", "camera", "facial recognition"],
+        weight: 8
+    }
+];
+export function classifyDomain(description, tags, name, task) {
+    const text = `${description} ${name} ${tags.join(" ")} ${task || ""}`.toLowerCase();
+    const scores = new Map();
+    // Initialize scores
+    DOMAIN_KEYWORDS.forEach(dk => {
+        scores.set(dk.domain, 0);
+    });
+    // Score each domain
+    DOMAIN_KEYWORDS.forEach(dk => {
+        let matches = 0;
+        dk.keywords.forEach(keyword => {
+            if (text.includes(keyword.toLowerCase())) {
+                matches++;
+            }
+        });
+        if (matches > 0) {
+            const currentScore = scores.get(dk.domain) || 0;
+            scores.set(dk.domain, currentScore + (matches * dk.weight));
+        }
+    });
+    // Find domain with highest score
+    let maxScore = 0;
+    let bestDomain = "unknown";
+    scores.forEach((score, domain) => {
+        if (score > maxScore) {
+            maxScore = score;
+            bestDomain = domain;
+        }
+    });
+    // If no strong match, check task type for hints
+    if (maxScore === 0 && task) {
+        const taskLower = task.toLowerCase();
+        if (taskLower.includes("image") || taskLower.includes("vision") || taskLower.includes("detection")) {
+            return "computer-vision";
+        }
+        if (taskLower.includes("text") || taskLower.includes("language") || taskLower.includes("nlp")) {
+            return "nlp";
+        }
+    }
+    // Require minimum score threshold to avoid false positives
+    if (maxScore < 5) {
+        return "general";
+    }
+    // Special handling: medicine and healthcare are related
+    if (bestDomain === "medicine" || bestDomain === "healthcare") {
+        const medicineScore = scores.get("medicine") || 0;
+        const healthcareScore = scores.get("healthcare") || 0;
+        if (medicineScore > healthcareScore * 1.5) {
+            return "medicine";
+        }
+        else if (healthcareScore > medicineScore * 1.5) {
+            return "healthcare";
+        }
+        else {
+            // If both are high, prefer medicine for medical-specific terms
+            return (medicineScore >= healthcareScore ? "medicine" : "healthcare");
+        }
+    }
+    return bestDomain;
+}
+export function getDomainDisplayName(domain) {
+    const names = {
+        "medicine": "Medicine",
+        "healthcare": "Healthcare",
+        "security": "Security",
+        "finance": "Finance",
+        "education": "Education",
+        "science": "Science",
+        "technology": "Technology",
+        "social": "Social",
+        "legal": "Legal",
+        "business": "Business",
+        "multimedia": "Multimedia",
+        "nlp": "Natural Language Processing",
+        "computer-vision": "Computer Vision",
+        "general": "General",
+        "unknown": "Unknown"
+    };
+    return names[domain] || "Unknown";
+}

package/build/metadata/github-scraper.js

@@ -0,0 +1,47 @@
+import { spawn } from "child_process";
+import path from "path";
+const PYTHON_SCRIPT_PATH = path.resolve("src", "python", "github_adapter.py");
+export class GitHubScraper {
+    /**
+     * Search GitHub repositories for datasets using the Python adapter
+     */
+    async scrape(query, limit = 10) {
+        return new Promise((resolve, reject) => {
+            const pythonProcess = spawn("python", [
+                PYTHON_SCRIPT_PATH,
+                "--action", "search",
+                "--query", query,
+                "--limit", String(limit)
+            ]);
+            let output = "";
+            let errorOutput = "";
+            pythonProcess.stdout.on("data", (data) => {
+                output += data.toString();
+            });
+            pythonProcess.stderr.on("data", (data) => {
+                errorOutput += data.toString();
+            });
+            pythonProcess.on("close", (code) => {
+                if (code !== 0) {
+                    console.error(`[GitHubScraper] Process exited with code ${code}: ${errorOutput}`);
+                    resolve([]);
+                    return;
+                }
+                try {
+                    const results = JSON.parse(output);
+                    if (results.error) {
+                        console.error(`[GitHubScraper] Internal error: ${results.error}`);
+                        resolve([]);
+                    }
+                    else {
+                        resolve(results);
+                    }
+                }
+                catch (e) {
+                    console.error(`[GitHubScraper] JSON parse error: ${e.message}. Output: ${output.substring(0, 100)}...`);
+                    resolve([]);
+                }
+            });
+        });
+    }
+}