@vespermcp/mcp-server 1.0.0

package/build/fusion/aligner.js

@@ -0,0 +1,56 @@
+export class SchemaAligner {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Finds common columns across datasets and creates a mapping to canonical names.
+     */
+    align(datasets) {
+        const alignment = {};
+        for (const ds of datasets) {
+            alignment[ds.id] = {};
+            if (!ds.columns)
+                continue;
+            for (const col of ds.columns) {
+                const canonicalName = this.getCanonicalName(col.name);
+                if (canonicalName) {
+                    alignment[ds.id][col.name] = canonicalName;
+                }
+            }
+        }
+        return alignment;
+    }
+    /**
+     * Resolves the most specific common type for a canonical column.
+     */
+    resolveType(canonicalName, types) {
+        if (this.config.type_overrides[canonicalName]) {
+            return this.config.type_overrides[canonicalName];
+        }
+        const uniqueTypes = new Set(types.map(t => t.toLowerCase()));
+        if (uniqueTypes.size === 1) {
+            return Array.from(uniqueTypes)[0];
+        }
+        if (uniqueTypes.has("float") || uniqueTypes.has("number")) {
+            return "number";
+        }
+        return "string"; // Default to string for mixed types
+    }
+    getCanonicalName(colName) {
+        // Direct match with any canonical name
+        for (const canonical in this.config.column_aliases) {
+            if (canonical.toLowerCase() === colName.toLowerCase()) {
+                return canonical;
+            }
+            // Match with aliases
+            const aliases = this.config.column_aliases[canonical];
+            if (aliases.some((a) => a.toLowerCase() === colName.toLowerCase())) {
+                return canonical;
+            }
+        }
+        // If no alias, use the name itself if it's already one of the canonical names we expect
+        // This is a bit recursive, let's simplify.
+        return null;
+    }
+}

package/build/fusion/deduplicator.js

@@ -0,0 +1,69 @@
+import * as crypto from "crypto";
+export class Deduplicator {
+    config;
+    seenHashes = new Set();
+    seenTexts = new Map(); // column -> tokens[]
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Checks if a record is an exact duplicate based on all columns.
+     */
+    isExactDuplicate(record) {
+        if (!this.config.dedupe_config.exact)
+            return false;
+        const hash = crypto
+            .createHash("md5")
+            .update(JSON.stringify(record))
+            .digest("hex");
+        if (this.seenHashes.has(hash)) {
+            return true;
+        }
+        this.seenHashes.add(hash);
+        return false;
+    }
+    /**
+     * Checks if a record is a fuzzy duplicate based on configured columns.
+     */
+    isFuzzyDuplicate(record) {
+        if (!this.config.dedupe_config.fuzzy)
+            return false;
+        if (!this.config.dedupe_config.fuzzy_columns.length)
+            return false;
+        for (const col of this.config.dedupe_config.fuzzy_columns) {
+            const text = String(record[col] || "");
+            if (!text || text.length < 10)
+                continue; // Skip short/empty tags
+            const tokens = this.tokenize(text);
+            if (this.isSimilar(col, tokens)) {
+                return true;
+            }
+            // In a real implementation, we'd use a more efficient data structure (like LSH)
+            // For now, we store tokens and compare (O(N^2) in worst case, use with caution)
+            // But we'll keep it simple for this phase.
+            // Actually, let's just store the tokens.
+        }
+        return false;
+    }
+    tokenize(text) {
+        return new Set(text.toLowerCase().split(/\s+/).filter(t => t.length > 2));
+    }
+    isSimilar(column, tokens) {
+        if (!this.seenTexts.has(column)) {
+            this.seenTexts.set(column, []);
+        }
+        const stored = this.seenTexts.get(column);
+        for (const existingSerialized of stored) {
+            const existing = new Set(JSON.parse(existingSerialized));
+            const intersection = new Set([...tokens].filter(t => existing.has(t)));
+            const union = new Set([...tokens, ...existing]);
+            const jaccard = intersection.size / union.size;
+            if (jaccard >= this.config.dedupe_config.fuzzy_threshold) {
+                return true;
+            }
+        }
+        // Add to seen
+        stored.push(JSON.stringify([...tokens]));
+        return false;
+    }
+}

package/build/fusion/harmonizer.js

@@ -0,0 +1,39 @@
+export class LabelHarmonizer {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Maps a raw label value to a unified canonical value.
+     */
+    harmonize(label) {
+        let harmonized = label;
+        // 1. apply explicit mapping
+        if (this.config.label_map && label in this.config.label_map) {
+            harmonized = this.config.label_map[label];
+        }
+        // 2. apply multi-class to binary conversion
+        if (this.config.multi_to_binary) {
+            const { positive_classes, positive_label, negative_label } = this.config.multi_to_binary;
+            const isPositive = positive_classes.some((pc) => String(pc).toLowerCase() === String(harmonized).toLowerCase());
+            return isPositive ? positive_label : negative_label;
+        }
+        return harmonized;
+    }
+    /**
+     * Checks if a label distribution is balanced enough (placeholder for quality scoring).
+     */
+    checkBalance(distribution) {
+        const warnings = [];
+        const total = Object.values(distribution).reduce((a, b) => a + b, 0);
+        if (total === 0)
+            return warnings;
+        for (const [label, count] of Object.entries(distribution)) {
+            const pct = count / total;
+            if (pct < 0.05) {
+                warnings.push(`Extreme minority class detected: "${label}" (${(pct * 100).toFixed(1)}%)`);
+            }
+        }
+        return warnings;
+    }
+}

package/build/fusion/orchestrator.js

@@ -0,0 +1,86 @@
+import { SchemaAligner } from "./aligner.js";
+import { LabelHarmonizer } from "./harmonizer.js";
+import { Deduplicator } from "./deduplicator.js";
+export class FusionOrchestrator {
+    config;
+    aligner;
+    harmonizer;
+    deduplicator;
+    constructor(config) {
+        this.config = config;
+        this.aligner = new SchemaAligner(config);
+        this.harmonizer = new LabelHarmonizer(config);
+        this.deduplicator = new Deduplicator(config);
+    }
+    /**
+     * Fuses multiple datasets into one.
+     * @param datasets Metadata of datasets to fuse
+     * @param dataRecords Actual records from all datasets (flattened)
+     */
+    async fuse(datasets, dataRecords) {
+        const stats = {
+            total_input_rows: dataRecords.length,
+            total_output_rows: 0,
+            duplicates_removed: 0,
+            fuzzy_duplicates_removed: 0,
+            schema_overlaps: [],
+            label_distribution: {}
+        };
+        const warnings = [];
+        const alignmentMap = this.aligner.align(datasets);
+        // Find canonical columns common to at least one dataset (in this simple version)
+        const allCanonicalCols = new Set();
+        for (const dsId in alignmentMap) {
+            for (const canonical of Object.values(alignmentMap[dsId])) {
+                allCanonicalCols.add(canonical);
+            }
+        }
+        stats.schema_overlaps = Array.from(allCanonicalCols);
+        const fusedData = [];
+        for (const { datasetId, record } of dataRecords) {
+            const alignedRecord = {};
+            const colMap = alignmentMap[datasetId];
+            if (!colMap) {
+                warnings.push(`No alignment found for dataset ${datasetId}`);
+                continue;
+            }
+            // 1. Align columns
+            for (const [sourceCol, val] of Object.entries(record)) {
+                const canonical = colMap[sourceCol];
+                if (canonical) {
+                    alignedRecord[canonical] = val;
+                }
+            }
+            // 2. Harmonize label
+            if (this.config.target_column && alignedRecord[this.config.target_column] !== undefined) {
+                alignedRecord[this.config.target_column] = this.harmonizer.harmonize(alignedRecord[this.config.target_column]);
+            }
+            // 3. Deduplicate
+            if (this.deduplicator.isExactDuplicate(alignedRecord)) {
+                stats.duplicates_removed++;
+                continue;
+            }
+            if (this.deduplicator.isFuzzyDuplicate(alignedRecord)) {
+                stats.fuzzy_duplicates_removed++;
+                continue;
+            }
+            // Update distribution ONLY for kept records
+            if (this.config.target_column && alignedRecord[this.config.target_column] !== undefined) {
+                const labelStr = String(alignedRecord[this.config.target_column]);
+                stats.label_distribution[labelStr] = (stats.label_distribution[labelStr] || 0) + 1;
+            }
+            fusedData.push(alignedRecord);
+        }
+        stats.total_output_rows = fusedData.push(); // Wait, push returns new length
+        stats.total_output_rows = fusedData.length;
+        // Add balance warnings
+        const balanceWarnings = this.harmonizer.checkBalance(stats.label_distribution);
+        warnings.push(...balanceWarnings);
+        return {
+            success: true,
+            output_path: "fused_dataset.json", // Placeholder
+            stats,
+            warnings
+        };
+    }
+}

package/build/fusion/types.js

@@ -0,0 +1 @@
+export {};