@vespermcp/mcp-server 1.0.0

package/build/cleaning/planner.js

@@ -0,0 +1,111 @@
+export class CleaningPlanner {
+    cache;
+    constructor(cache) {
+        this.cache = cache;
+    }
+    /**
+     * Generate a cleaning plan based on the quality report and optional custom rules
+     */
+    async generatePlan(datasetId, report, ruleSet) {
+        if (this.cache) {
+            const cached = await this.cache.getPlan(datasetId, { report, ruleSet });
+            if (cached) {
+                console.log(`[CleaningPlanner] Cache hit for ${datasetId}`);
+                return cached;
+            }
+        }
+        const ops = [];
+        let estimatedRowsSaved = 0;
+        let estimatedColsSaved = 0;
+        // 1. Remove Duplicates (Global)
+        if (report.duplicate_rows > 0) {
+            ops.push({
+                type: "RemoveDuplicates",
+                params: {},
+                reason: `Found ${report.duplicate_rows} exact duplicate rows`
+            });
+            estimatedRowsSaved += report.duplicate_rows;
+        }
+        // 2. Column-level operations
+        for (const col of report.columns) {
+            // A. Drop Empty / Useless Columns
+            if (col.missing_percentage > 90 || col.is_constant) {
+                ops.push({
+                    type: "DropColumns",
+                    params: { columns: [col.name] },
+                    reason: col.is_constant ? "Column is constant (zero variance)" : `High missing values (${col.missing_percentage.toFixed(1)}%)`
+                });
+                estimatedColsSaved++;
+                continue;
+            }
+            // B. Fix Types
+            if (this.shouldFixType(col)) {
+                const targetType = col.inferred_type.toLowerCase().includes("numeric") ? "float" : "string";
+                ops.push({
+                    type: "FixTypes",
+                    params: { column: col.name, type: targetType },
+                    reason: `Inferred type is ${col.inferred_type} but stored as ${col.type}`
+                });
+            }
+            // C. Impute Missing Values
+            if (col.missing_count > 0) {
+                let method = "constant";
+                let value = "unknown";
+                if (col.inferred_type.includes("Numeric") || col.type.includes("Int") || col.type.includes("Float")) {
+                    method = "median";
+                    value = 0;
+                }
+                else {
+                    method = "constant";
+                    value = "missing";
+                }
+                ops.push({
+                    type: "FillMissing",
+                    params: { column: col.name, method, value },
+                    reason: `${col.missing_count} missing values`
+                });
+            }
+        }
+        // 3. Apply Custom Rules
+        if (ruleSet) {
+            for (const rule of ruleSet.rules) {
+                const targets = rule.condition.column === "*"
+                    ? report.columns.map(c => c.name)
+                    : [rule.condition.column];
+                for (const targetCol of targets) {
+                    const colStats = report.columns.find(c => c.name === targetCol);
+                    if (!colStats)
+                        continue;
+                    ops.push({
+                        type: rule.action.type,
+                        params: { ...rule.action.params, column: targetCol },
+                        reason: `Custom Rule: ${rule.name} - ${rule.description}`
+                    });
+                }
+            }
+        }
+        const plan = {
+            dataset_id: datasetId,
+            operations: ops,
+            estimated_impact: {
+                rows_saved: estimatedRowsSaved,
+                columns_saved: estimatedColsSaved,
+                quality_score_improvement: 10 + (ops.length * 5)
+            }
+        };
+        if (this.cache) {
+            await this.cache.savePlan(datasetId, { report, ruleSet }, plan);
+        }
+        return plan;
+    }
+    shouldFixType(col) {
+        if (col.inferred_type && col.inferred_type.includes("Numeric") && (col.type.includes("String") || col.type.includes("Utf8"))) {
+            return true;
+        }
+        return false;
+    }
+    isNumeric(col) {
+        const t = col.type.toLowerCase();
+        return t.includes("int") || t.includes("float") || t.includes("numeric");
+    }
+}

package/build/cleaning/rules.js

@@ -0,0 +1,57 @@
+// --- Domain Presets ---
+export const NLP_PRESET = {
+    id: "preset-nlp",
+    name: "NLP Data Prep",
+    domain: "nlp",
+    rules: [
+        {
+            id: "nlp-1",
+            name: "Normalize Case",
+            description: "Convert all text to lowercase",
+            condition: { column: "*", operator: "is_null", value: false }, // Apply to all non-null if type is string (logic in evaluator)
+            action: { type: "NormalizeText", params: { case: "lower" } }
+        },
+        {
+            id: "nlp-2",
+            name: "Remove URLs",
+            description: "Strip http/https links",
+            condition: { column: "*", operator: "contains", value: "http" },
+            action: { type: "Replace", params: { pattern: "https?://\\S+", replacement: "" } }
+        }
+    ]
+};
+export const HEALTHCARE_PRESET = {
+    id: "preset-healthcare",
+    name: "Healthcare (HIPAA) Prep",
+    domain: "healthcare",
+    rules: [
+        {
+            id: "hc-1",
+            name: "Mask Emails",
+            description: "Identify and mask email addresses",
+            condition: { column: "*", operator: "contains", value: "@" },
+            action: { type: "CustomMask", params: { method: "hash", salt: "vesper-pii" } }
+        },
+        {
+            id: "hc-2",
+            name: "Normalize Dates",
+            description: "Ensure ISO-8601 for DOB/Admit dates",
+            condition: { column: "date", operator: "is_null", value: false },
+            action: { type: "FixTypes", params: { type: "date" } }
+        }
+    ]
+};
+export const FINANCE_PRESET = {
+    id: "preset-finance",
+    name: "Financial Data Prep",
+    domain: "finance",
+    rules: [
+        {
+            id: "fin-1",
+            name: "Currency Cleanup",
+            description: "Remove currency symbols and parse as float",
+            condition: { column: "amount", operator: "matches_regex", value: "[\\$\\€\\£]" },
+            action: { type: "FixTypes", params: { type: "float", strip: "[^0-9\\.]" } }
+        }
+    ]
+};

package/build/cleaning/types.js

@@ -0,0 +1 @@
+export {};

package/build/cloud/adapters/local.js

@@ -0,0 +1,37 @@
+import fs from "fs";
+import path from "path";
+/**
+ * LocalAdapter mimics cloud storage by copying files to a local vault directory.
+ */
+export class LocalAdapter {
+    vaultPath;
+    constructor(vaultPath) {
+        this.vaultPath = path.resolve(vaultPath);
+        if (!fs.existsSync(this.vaultPath)) {
+            fs.mkdirSync(this.vaultPath, { recursive: true });
+        }
+    }
+    async upload(localPath, remotePath) {
+        if (!fs.existsSync(localPath)) {
+            throw new Error(`Local file not found: ${localPath}`);
+        }
+        const destPath = path.join(this.vaultPath, remotePath);
+        const destDir = path.dirname(destPath);
+        if (!fs.existsSync(destDir)) {
+            fs.mkdirSync(destDir, { recursive: true });
+        }
+        fs.copyFileSync(localPath, destPath);
+        // Returns a file URI as the "url"
+        return `file://${destPath}`;
+    }
+    async delete(remotePath) {
+        const destPath = path.join(this.vaultPath, remotePath);
+        if (fs.existsSync(destPath)) {
+            fs.unlinkSync(destPath);
+        }
+    }
+    async getSignedUrl(remotePath, expiresValue) {
+        // For local, just return the file URI
+        return `file://${path.join(this.vaultPath, remotePath)}`;
+    }
+}

package/build/cloud/adapters/s3.js

@@ -0,0 +1,24 @@
+/**
+ * S3Adapter Stub.
+ * Note: Requires @aws-sdk/client-s3 to be installed for full functionality.
+ */
+export class S3Adapter {
+    bucket;
+    region;
+    credentials;
+    constructor(bucket, region, credentials) {
+        this.bucket = bucket;
+        this.region = region;
+        this.credentials = credentials;
+    }
+    async upload(localPath, remotePath) {
+        console.warn("S3Adapter: Full implementation requires @aws-sdk/client-s3. This is a stub.");
+        return `https://${this.bucket}.s3.${this.region}.amazonaws.com/${remotePath}`;
+    }
+    async delete(remotePath) {
+        console.warn("S3Adapter: Delete stub called.");
+    }
+    async getSignedUrl(remotePath) {
+        return `https://${this.bucket}.s3.${this.region}.amazonaws.com/${remotePath}?stub=true`;
+    }
+}

package/build/cloud/storage-manager.js

@@ -0,0 +1,20 @@
+import { LocalAdapter } from "./adapters/local.js";
+import { S3Adapter } from "./adapters/s3.js";
+export class StorageManager {
+    /**
+     * Creates an adapter based on configuration
+     */
+    static createAdapter(config) {
+        switch (config.type) {
+            case "local":
+                return new LocalAdapter(config.options.basePath || "./storage_vault");
+            case "s3":
+                if (!config.options.bucket || !config.options.region) {
+                    throw new Error("S3 requires bucket and region");
+                }
+                return new S3Adapter(config.options.bucket, config.options.region, config.options.credentials);
+            default:
+                throw new Error(`Unsupported storage type: ${config.type}`);
+        }
+    }
+}

package/build/cloud/types.js

@@ -0,0 +1 @@
+export {};

package/build/compliance/service.js

@@ -0,0 +1,73 @@
+import { v4 as uuidv4 } from "uuid";
+export class ComplianceService {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    async verifyGDPR(dataset) {
+        const issues = [];
+        // 1. Check for PII metadata flag
+        if (dataset.has_personal_data) {
+            issues.push("Dataset explicitly flagged as containing personal data.");
+        }
+        // 2. Check quality warnings for PII
+        if (dataset.quality_warnings) {
+            const piiWarnings = dataset.quality_warnings.filter(w => w.toLowerCase().includes("pii") || w.toLowerCase().includes("personal"));
+            if (piiWarnings.length > 0) {
+                issues.push(...piiWarnings);
+            }
+        }
+        // 3. Check for consent record
+        const consent = this.store.getConsent(dataset.id);
+        if (!consent || !consent.consent_obtained) {
+            issues.push("No valid consent record found for this dataset.");
+        }
+        const result = {
+            dataset_id: dataset.id,
+            standard: "GDPR",
+            passed: issues.length === 0,
+            issues,
+            timestamp: new Date().toISOString()
+        };
+        this.store.saveCheck(result);
+        return result;
+    }
+    async verifyHIPAA(dataset) {
+        const issues = [];
+        // 1. De-identification check (simulated)
+        // If domain is medical but no de-identification flag exists
+        if (dataset.domain === "healthcare" || dataset.domain === "medical") {
+            if (dataset.has_personal_data) {
+                issues.push("Medical dataset contains personal data (not de-identified).");
+            }
+        }
+        const result = {
+            dataset_id: dataset.id,
+            standard: "HIPAA",
+            passed: issues.length === 0,
+            issues,
+            timestamp: new Date().toISOString()
+        };
+        this.store.saveCheck(result);
+        return result;
+    }
+    logOperation(userId, datasetId, operation, metadata = {}) {
+        const event = {
+            id: uuidv4(),
+            user_id: userId,
+            dataset_id: datasetId,
+            operation,
+            timestamp: new Date().toISOString(),
+            details: JSON.stringify(metadata)
+        };
+        this.store.saveAudit(event);
+    }
+    exportAuditLog(datasetId) {
+        const logs = this.store.getAuditLogs(datasetId);
+        if (logs.length === 0)
+            return "No audit logs found.";
+        const header = "ID,Timestamp,User,Operation,Dataset,Details\n";
+        const rows = logs.map(l => `${l.id},${l.timestamp},${l.user_id},${l.operation},${l.dataset_id},"${l.details.replace(/"/g, '""')}"`).join("\n");
+        return header + rows;
+    }
+}

package/build/compliance/store.js

@@ -0,0 +1,80 @@
+export class ComplianceStore {
+    db;
+    constructor(db) {
+        this.db = db;
+        this.init();
+    }
+    init() {
+        this.db.exec(`
+            CREATE TABLE IF NOT EXISTS audit_logs (
+                id TEXT PRIMARY KEY,
+                user_id TEXT,
+                dataset_id TEXT,
+                operation TEXT,
+                timestamp TEXT,
+                details TEXT
+            );
+
+            CREATE TABLE IF NOT EXISTS compliance_checks (
+                dataset_id TEXT,
+                standard TEXT,
+                passed BOOLEAN,
+                issues TEXT, -- JSON array
+                timestamp TEXT,
+                PRIMARY KEY (dataset_id, standard)
+            );
+
+            CREATE TABLE IF NOT EXISTS consent_records (
+                dataset_id TEXT PRIMARY KEY,
+                consent_obtained BOOLEAN,
+                source TEXT,
+                last_verified TEXT
+            );
+        `);
+    }
+    saveAudit(event) {
+        const stmt = this.db.prepare(`
+            INSERT INTO audit_logs (id, user_id, dataset_id, operation, timestamp, details)
+            VALUES (?, ?, ?, ?, ?, ?)
+        `);
+        stmt.run(event.id, event.user_id, event.dataset_id, event.operation, event.timestamp, event.details);
+    }
+    getAuditLogs(datasetId) {
+        let query = "SELECT * FROM audit_logs";
+        const params = [];
+        if (datasetId) {
+            query += " WHERE dataset_id = ?";
+            params.push(datasetId);
+        }
+        query += " ORDER BY timestamp DESC";
+        return this.db.prepare(query).all(...params);
+    }
+    saveCheck(result) {
+        const stmt = this.db.prepare(`
+            INSERT INTO compliance_checks (dataset_id, standard, passed, issues, timestamp)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(dataset_id, standard) DO UPDATE SET
+                passed=excluded.passed,
+                issues=excluded.issues,
+                timestamp=excluded.timestamp
+        `);
+        stmt.run(result.dataset_id, result.standard, result.passed ? 1 : 0, JSON.stringify(result.issues), result.timestamp);
+    }
+    saveConsent(record) {
+        const stmt = this.db.prepare(`
+            INSERT INTO consent_records (dataset_id, consent_obtained, source, last_verified)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(dataset_id) DO UPDATE SET
+                consent_obtained=excluded.consent_obtained,
+                source=excluded.source,
+                last_verified=excluded.last_verified
+        `);
+        stmt.run(record.dataset_id, record.consent_obtained ? 1 : 0, record.source, record.last_verified);
+    }
+    getConsent(datasetId) {
+        const row = this.db.prepare("SELECT * FROM consent_records WHERE dataset_id = ?").get(datasetId);
+        if (!row)
+            return null;
+        return { ...row, consent_obtained: Boolean(row.consent_obtained) };
+    }
+}

package/build/compliance/types.js

@@ -0,0 +1 @@
+export {};

package/build/data/processing-worker.js

@@ -0,0 +1,23 @@
+import { parentPort } from "worker_threads";
+if (parentPort) {
+    parentPort.on("message", (data) => {
+        const { records, task, params } = data;
+        let results = [];
+        switch (task) {
+            case "transform":
+                results = records.map(r => {
+                    // Simulated transformation (e.g. alignment/harmonization logic)
+                    const updated = { ...r, processed_at: new Date().toISOString() };
+                    // Complex CPU bound work simulation
+                    let sum = 0;
+                    for (let i = 0; i < 1000; i++)
+                        sum += i;
+                    return updated;
+                });
+                break;
+            default:
+                results = records;
+        }
+        parentPort?.postMessage(results);
+    });
+}

package/build/data/streaming.js

@@ -0,0 +1,38 @@
+export class StreamProcessor {
+    /**
+     * Processes a readable stream in chunks.
+     */
+    static async processInChunks(stream, options, processor) {
+        let chunk = [];
+        const results = [];
+        let totalProcessed = 0;
+        for await (const record of stream) {
+            // Sampling logic
+            if (options.samplePercentage !== undefined) {
+                if (Math.random() * 100 > options.samplePercentage) {
+                    continue;
+                }
+            }
+            chunk.push(record);
+            if (chunk.length >= options.chunkSize) {
+                const processedChunk = await processor(chunk);
+                results.push(...processedChunk);
+                totalProcessed += chunk.length;
+                chunk = [];
+            }
+        }
+        // Process remaining
+        if (chunk.length > 0) {
+            const processedChunk = await processor(chunk);
+            results.push(...processedChunk);
+            totalProcessed += chunk.length;
+        }
+        return results;
+    }
+    /**
+     * Creates a sampler stream that only emits a percentage of records.
+     */
+    static createSampler(percentage) {
+        return () => Math.random() * 100 <= percentage;
+    }
+}

package/build/data/worker-pool.js

@@ -0,0 +1,39 @@
+import { Worker } from "worker_threads";
+import path from "path";
+import { fileURLToPath } from "url";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+export class WorkerPool {
+    poolSize;
+    workers = [];
+    queue = [];
+    activeWorkers = 0;
+    constructor(poolSize = 4) {
+        this.poolSize = poolSize;
+        const workerPath = path.resolve(__dirname, "processing-worker.js");
+        // Note: In development with tsx, we might need to handle .ts vs .js
+    }
+    async process(records, task = "transform") {
+        return new Promise((resolve, reject) => {
+            const worker = new Worker(new URL("./processing-worker.ts", import.meta.url), {
+                execArgv: ["--import", "tsx"]
+            });
+            worker.postMessage({ records, task });
+            worker.on("message", (results) => {
+                worker.terminate();
+                resolve(results);
+            });
+            worker.on("error", reject);
+        });
+    }
+    /**
+     * Splits records into chunks and processes them in parallel across workers.
+     */
+    async processParallel(records, chunkSize) {
+        const chunks = [];
+        for (let i = 0; i < records.length; i += chunkSize) {
+            chunks.push(records.slice(i, i + chunkSize));
+        }
+        const results = await Promise.all(chunks.map(chunk => this.process(chunk)));
+        return results.flat();
+    }
+}

package/build/export/exporter.js

@@ -0,0 +1,45 @@
+import { spawn } from "child_process";
+import path from "path";
+import fs from "fs";
+export class DataExporter {
+    pythonPath = "python";
+    scriptPath;
+    constructor(projectRoot = process.cwd()) {
+        this.scriptPath = path.join(projectRoot, "src", "python", "export_engine.py");
+    }
+    /**
+     * Exports a dataset file to a specified format
+     */
+    async export(inputFile, outputFile, format, options = {}) {
+        return new Promise((resolve, reject) => {
+            if (!fs.existsSync(inputFile)) {
+                reject(new Error(`Input file not found: ${inputFile}`));
+                return;
+            }
+            const args = [this.scriptPath, inputFile, outputFile, format, JSON.stringify(options)];
+            const process = spawn(this.pythonPath, args);
+            let stdout = "";
+            let stderr = "";
+            process.stdout.on("data", (data) => stdout += data.toString());
+            process.stderr.on("data", (data) => stderr += data.toString());
+            process.on("close", (code) => {
+                if (code !== 0) {
+                    reject(new Error(`Export failed: ${stderr || stdout}`));
+                    return;
+                }
+                try {
+                    const result = JSON.parse(stdout);
+                    if (result.error) {
+                        reject(new Error(result.error));
+                    }
+                    else {
+                        resolve(result);
+                    }
+                }
+                catch (e) {
+                    reject(new Error(`Failed to parse export output: ${stdout}`));
+                }
+            });
+        });
+    }
+}

package/build/export/packager.js

@@ -0,0 +1,100 @@
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+/**
+ * MetadataPackager bundles data files with metadata and quality reports.
+ * Follows a simplified Frictionless Data Package standard.
+ */
+export class MetadataPackager {
+    /**
+     * Creates a data package in the target directory
+     */
+    async createPackage(targetDir, dataFiles, metadata, extraArtifacts) {
+        try {
+            if (!fs.existsSync(targetDir)) {
+                fs.mkdirSync(targetDir, { recursive: true });
+            }
+            const resources = [];
+            // 1. Process Data Files
+            for (const file of dataFiles) {
+                if (!fs.existsSync(file.path)) {
+                    throw new Error(`Data file not found: ${file.path}`);
+                }
+                const fileName = path.basename(file.path);
+                const destPath = path.join(targetDir, fileName);
+                // Copy file to package dir
+                fs.copyFileSync(file.path, destPath);
+                // Compute hash and size
+                const fileBuffer = fs.readFileSync(destPath);
+                const hash = crypto.createHash("sha256").update(fileBuffer).digest("hex");
+                const stats = fs.statSync(destPath);
+                resources.push({
+                    name: file.name,
+                    path: fileName,
+                    format: file.format,
+                    mediatype: this.getMediaType(file.format),
+                    bytes: stats.size,
+                    hash: `sha256:${hash}`
+                });
+            }
+            // 2. Add Extra Artifacts
+            if (extraArtifacts?.qualityReport) {
+                const qrPath = path.join(targetDir, "quality_report.json");
+                fs.writeFileSync(qrPath, JSON.stringify(extraArtifacts.qualityReport, null, 2));
+                resources.push({
+                    name: "quality-report",
+                    path: "quality_report.json",
+                    format: "json"
+                });
+            }
+            if (extraArtifacts?.cleaningLog) {
+                const clPath = path.join(targetDir, "cleaning_log.json");
+                fs.writeFileSync(clPath, JSON.stringify(extraArtifacts.cleaningLog, null, 2));
+                resources.push({
+                    name: "cleaning-log",
+                    path: "cleaning_log.json",
+                    format: "json"
+                });
+            }
+            // 3. Generate Manifest (datapackage.json)
+            const manifest = {
+                profile: "tabular-data-package",
+                name: metadata.name.toLowerCase().replace(/\s+/g, "-"),
+                title: metadata.name,
+                description: metadata.description,
+                version: metadata.version,
+                homepage: "https://github.com/vesper-data",
+                license: metadata.license,
+                author: metadata.author,
+                keywords: metadata.tags,
+                created: new Date().toISOString(),
+                resources: resources
+            };
+            const manifestPath = path.join(targetDir, "datapackage.json");
+            fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+            return {
+                success: true,
+                packagePath: targetDir,
+                manifest: manifest
+            };
+        }
+        catch (e) {
+            return {
+                success: false,
+                packagePath: targetDir,
+                manifest: null,
+                error: e.message
+            };
+        }
+    }
+    getMediaType(format) {
+        switch (format.toLowerCase()) {
+            case "csv": return "text/csv";
+            case "parquet": return "application/x-parquet";
+            case "jsonl": return "application/x-jsonlines";
+            case "arrow": return "application/x-apache-arrow-file";
+            case "json": return "application/json";
+            default: return "application/octet-stream";
+        }
+    }
+}

package/build/export/types.js

@@ -0,0 +1 @@
+export {};