@vespermcp/mcp-server 1.2.21 → 1.2.22

package/README.md

@@ -146,6 +146,33 @@ generate_quality_report(
 
 ### Dataset Discovery
 
+#### `unified_dataset_api`
+Single facade over multiple dataset backends. Use one tool for provider capability inspection, dataset discovery, dataset download, and dataset info lookup. The gateway prefers public/keyless providers and can also use server-managed credentials for connectors like Kaggle or data.world when configured by the operator.
+
+**Parameters:**
+- `operation` (string): `providers`, `discover`, `download`, or `info`
+- `source` (string, optional): `auto`, `huggingface`, `openml`, `kaggle`, `dataworld`, `s3`, `bigquery`
+- `query` (string, required for `discover`)
+- `dataset_id` (string, required for `download`/`info`)
+- `limit` (number, optional)
+- `target_dir` (string, optional)
+- `public_only` (boolean, optional)
+
+**Examples:**
+```
+unified_dataset_api(operation="providers")
+```
+
+```
+unified_dataset_api(operation="discover", query="credit risk", source="auto")
+```
+
+```
+unified_dataset_api(operation="download", dataset_id="huggingface:imdb")
+```
+
+---
+
 #### `search_datasets`
 Search for datasets across multiple sources.
 
@@ -195,6 +222,28 @@ export_dataset(
 
 ---
 
+#### `vesper_download_assets`
+Download image/media assets to a user-controlled local directory.
+
+**Parameters:**
+- `dataset_id` (string): Dataset identifier
+- `source` (string): `huggingface`, `kaggle`, or `url`
+- `target_dir` (string, optional): Exact local directory where assets should be written
+- `output_dir` (string, optional): Alias for `target_dir`
+- `output_format` (string, optional): `webdataset`, `imagefolder`, or `parquet`
+
+**Example:**
+```
+vesper_download_assets(
+  dataset_id="cats_vs_dogs",
+  source="kaggle",
+  target_dir="./datasets/cats_dogs_100",
+  output_format="imagefolder"
+)
+```
+
+---
+
 ### Quality Analysis
 
 #### `analyze_image_quality`

package/build/cloud/adapters/supabase.js

@@ -0,0 +1,49 @@
+import fs from "fs";
+import { readFile } from "fs/promises";
+import { createClient } from "@supabase/supabase-js";
+export class SupabaseAdapter {
+    bucket;
+    client;
+    constructor(bucket, supabaseUrl, supabaseServiceRoleKey) {
+        this.bucket = bucket;
+        const resolvedUrl = supabaseUrl || process.env.SUPABASE_URL;
+        const resolvedServiceRoleKey = supabaseServiceRoleKey || process.env.SUPABASE_SERVICE_ROLE_KEY;
+        if (!resolvedUrl || !resolvedServiceRoleKey) {
+            throw new Error("Supabase requires SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY");
+        }
+        this.client = createClient(resolvedUrl, resolvedServiceRoleKey, {
+            auth: {
+                persistSession: false,
+            },
+        });
+    }
+    async upload(localPath, remotePath) {
+        if (!fs.existsSync(localPath)) {
+            throw new Error(`Local file not found: ${localPath}`);
+        }
+        const fileBuffer = await readFile(localPath);
+        const { error } = await this.client.storage
+            .from(this.bucket)
+            .upload(remotePath, fileBuffer, { upsert: true });
+        if (error) {
+            throw new Error(`Supabase upload failed: ${error.message}`);
+        }
+        const { data } = this.client.storage.from(this.bucket).getPublicUrl(remotePath);
+        return data.publicUrl;
+    }
+    async delete(remotePath) {
+        const { error } = await this.client.storage.from(this.bucket).remove([remotePath]);
+        if (error) {
+            throw new Error(`Supabase delete failed: ${error.message}`);
+        }
+    }
+    async getSignedUrl(remotePath, expiresValue = 3600) {
+        const { data, error } = await this.client.storage
+            .from(this.bucket)
+            .createSignedUrl(remotePath, expiresValue);
+        if (error || !data?.signedUrl) {
+            throw new Error(`Supabase signed URL failed: ${error?.message || "No signed URL returned"}`);
+        }
+        return data.signedUrl;
+    }
+}

package/build/cloud/storage-manager.js

@@ -1,5 +1,6 @@
 import { LocalAdapter } from "./adapters/local.js";
 import { S3Adapter } from "./adapters/s3.js";
+import { SupabaseAdapter } from "./adapters/supabase.js";
 export class StorageManager {
     /**
      * Creates an adapter based on configuration
@@ -13,6 +14,11 @@ export class StorageManager {
                     throw new Error("S3 requires bucket and region");
                 }
                 return new S3Adapter(config.options.bucket, config.options.region, config.options.credentials);
+            case "supabase":
+                if (!config.options.bucket) {
+                    throw new Error("Supabase requires bucket");
+                }
+                return new SupabaseAdapter(config.options.bucket, config.options.supabaseUrl, config.options.supabaseServiceRoleKey);
             default:
                 throw new Error(`Unsupported storage type: ${config.type}`);
         }

package/build/export/exporter.js

@@ -1,10 +1,12 @@
 import { spawn } from "child_process";
 import path from "path";
 import fs from "fs";
+import { ensurePythonPackages, resolvePythonCommand } from "../utils/python-runtime.js";
 export class DataExporter {
-    pythonPath = "python";
+    buildDir;
     scriptPath;
     constructor(buildDir = process.cwd()) {
+        this.buildDir = buildDir;
         const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
         const dataRoot = path.join(homeDir, ".vesper");
         const scriptPath0 = path.resolve(dataRoot, "python", "export_engine.py");
@@ -26,27 +28,38 @@ export class DataExporter {
         else {
             this.scriptPath = scriptPath0;
         }
-        // Detect Python command
-        if (process.platform === "win32") {
-            this.pythonPath = "py";
-        }
     }
     /**
      * Exports a dataset file to a specified format
      */
     async export(inputFile, outputFile, format, options = {}) {
+        const pythonRequirements = [
+            { module: "polars", packageName: "polars" },
+        ];
+        if (format === "feather") {
+            pythonRequirements.push({ module: "pyarrow", packageName: "pyarrow" });
+        }
+        if (format === "tfrecord") {
+            pythonRequirements.push({ module: "tensorflow", packageName: "tensorflow" });
+        }
+        const pythonPath = await ensurePythonPackages(this.buildDir, pythonRequirements).catch(() => resolvePythonCommand(this.buildDir));
         return new Promise((resolve, reject) => {
             if (!fs.existsSync(inputFile)) {
                 reject(new Error(`Input file not found: ${inputFile}`));
                 return;
             }
             const args = [this.scriptPath, inputFile, outputFile, format, JSON.stringify(options)];
-            const process = spawn(this.pythonPath, args);
+            const childProcess = spawn(pythonPath, args, {
+                env: {
+                    ...process.env,
+                    PYTHONIOENCODING: "utf-8",
+                },
+            });
             let stdout = "";
             let stderr = "";
-            process.stdout.on("data", (data) => stdout += data.toString());
-            process.stderr.on("data", (data) => stderr += data.toString());
-            process.on("close", (code) => {
+            childProcess.stdout.on("data", (data) => stdout += data.toString());
+            childProcess.stderr.on("data", (data) => stderr += data.toString());
+            childProcess.on("close", (code) => {
                 if (code !== 0) {
                     reject(new Error(`Export failed: ${stderr || stdout}`));
                     return;

package/build/gateway/unified-dataset-gateway.js

@@ -0,0 +1,410 @@
+import fs from "fs";
+import path from "path";
+import http from "http";
+import https from "https";
+import { HuggingFaceScraper } from "../metadata/scraper.js";
+import { analyzeDatasetQuery } from "../search/query-intent.js";
+export class UnifiedDatasetGateway {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    getProviderStatuses(includeUnavailable = true) {
+        const hasHfToken = !!(process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN);
+        const hasKaggle = this.deps.dataIngestor.hasKaggleCredentials();
+        const hasDataWorld = this.deps.hasDataWorldToken();
+        const hasBigQuery = !!(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GOOGLE_CLOUD_PROJECT);
+        const providers = [
+            {
+                source: "huggingface",
+                display_name: "Hugging Face",
+                available: true,
+                auth_mode: hasHfToken ? "public-or-server-managed" : "public",
+                supported_operations: ["discover", "download", "info"],
+                requires_end_user_key: false,
+                notes: hasHfToken
+                    ? ["Public datasets are open by default. Gated/private repos can be accessed via the server-managed HF token when configured."]
+                    : ["Public datasets work without any user key. Gated/private repos need an operator or user token."],
+            },
+            {
+                source: "openml",
+                display_name: "OpenML",
+                available: true,
+                auth_mode: "public",
+                supported_operations: ["discover", "download", "info"],
+                requires_end_user_key: false,
+                notes: ["OpenML is exposed as a keyless public provider through the gateway."],
+            },
+            {
+                source: "kaggle",
+                display_name: "Kaggle",
+                available: hasKaggle,
+                auth_mode: hasKaggle ? "server-managed" : "not-configured",
+                supported_operations: ["discover", "download", "info"],
+                requires_end_user_key: false,
+                notes: hasKaggle
+                    ? ["Kaggle is available through server-managed credentials. End users do not need to pass their own key."]
+                    : ["Kaggle support exists, but no server-managed credentials are configured yet."],
+            },
+            {
+                source: "dataworld",
+                display_name: "data.world",
+                available: hasDataWorld,
+                auth_mode: hasDataWorld ? "server-managed" : "not-configured",
+                supported_operations: ["discover", "download", "info"],
+                requires_end_user_key: false,
+                notes: hasDataWorld
+                    ? ["data.world is available through server-managed credentials."]
+                    : ["data.world support exists, but no server-managed token is configured yet."],
+            },
+            {
+                source: "s3",
+                display_name: "Amazon S3",
+                available: true,
+                auth_mode: "public-or-server-managed",
+                supported_operations: ["download", "info"],
+                requires_end_user_key: false,
+                notes: ["Supports keyless download of public S3 objects via s3://bucket/key or HTTPS S3 URLs.", "Bucket listing and search are intentionally not exposed."],
+            },
+            {
+                source: "bigquery",
+                display_name: "BigQuery",
+                available: hasBigQuery,
+                auth_mode: hasBigQuery ? "server-managed" : "not-configured",
+                supported_operations: ["info"],
+                requires_end_user_key: false,
+                notes: hasBigQuery
+                    ? ["BigQuery is reserved for operator-managed connectors. Query execution is not implemented in this patch."]
+                    : ["BigQuery is scaffolded in the gateway contract, but no server-managed GCP configuration is present."],
+            },
+        ];
+        return includeUnavailable ? providers : providers.filter(provider => provider.available);
+    }
+    async discover(options) {
+        const query = String(options.query || "").trim();
+        const requestedSource = options.source || "auto";
+        const limit = Math.max(1, Number(options.limit || 10));
+        const publicOnly = options.publicOnly !== false;
+        if (!query) {
+            throw new Error("query is required");
+        }
+        const notes = [];
+        const providers = this.resolveDiscoverSources(requestedSource, publicOnly, notes);
+        const perSourceLimit = Math.max(5, Math.ceil(limit / Math.max(providers.length, 1)) * 2);
+        const allResults = [];
+        for (const provider of providers) {
+            try {
+                const partial = await this.discoverFromSource(provider, query, perSourceLimit);
+                for (const dataset of partial) {
+                    try {
+                        this.deps.metadataStore.saveDataset(dataset);
+                    }
+                    catch {
+                        // best-effort metadata persistence
+                    }
+                    allResults.push(dataset);
+                }
+            }
+            catch (error) {
+                notes.push(`${provider}: ${(error?.message || error || "Unknown provider error").toString()}`);
+            }
+        }
+        const deduped = new Map();
+        for (const dataset of allResults) {
+            deduped.set(`${dataset.source}:${dataset.id}`, dataset);
+        }
+        const results = Array.from(deduped.values())
+            .sort((a, b) => this.rankDataset(b) - this.rankDataset(a))
+            .slice(0, limit);
+        return {
+            query,
+            requested_source: requestedSource,
+            providers_tried: providers,
+            notes,
+            results,
+        };
+    }
+    async download(options) {
+        const requested = String(options.datasetId || "").trim();
+        if (!requested) {
+            throw new Error("dataset_id is required");
+        }
+        const notes = [];
+        const resolved = this.resolveDatasetReference(requested, options.source || "auto");
+        if (resolved.source === "bigquery") {
+            throw new Error("BigQuery gateway support is scaffolded for operator-managed connectors, but query/download execution is not implemented yet.");
+        }
+        if (resolved.source === "s3") {
+            const localPath = await this.downloadPublicS3Object(resolved.datasetId, options.targetDir);
+            return {
+                dataset_id: requested,
+                resolved_source: "s3",
+                local_path: localPath,
+                notes: ["Downloaded via the keyless S3 gateway path."],
+            };
+        }
+        let source = this.toIngestSource(resolved.source);
+        let datasetId = resolved.datasetId;
+        if (!source) {
+            const metadataMatch = this.lookupKnownDataset(requested);
+            const metadataSource = this.toIngestSource(metadataMatch?.source);
+            if (metadataMatch && metadataSource) {
+                source = metadataSource;
+                datasetId = metadataMatch.id;
+            }
+        }
+        if (!source) {
+            const discovery = await this.discover({ query: requested, source: "auto", limit: 1, publicOnly: false });
+            if (discovery.results.length === 0) {
+                throw new Error(`Unable to resolve provider for '${requested}'. Run unified_dataset_api with operation='discover' first or pass an explicit source.`);
+            }
+            const discoveredSource = this.toIngestSource(discovery.results[0].source);
+            if (!discoveredSource) {
+                throw new Error(`Resolved provider '${discovery.results[0].source}' cannot be downloaded through the dataset ingestor.`);
+            }
+            source = discoveredSource;
+            datasetId = discovery.results[0].id;
+            notes.push(`Auto-resolved provider to ${source}.`);
+        }
+        if (source === "kaggle" && !this.deps.dataIngestor.hasKaggleCredentials()) {
+            throw new Error("Kaggle is configured as a gateway source, but no server-managed credentials are available.");
+        }
+        if (source === "dataworld" && !this.deps.hasDataWorldToken()) {
+            throw new Error("data.world is configured as a gateway source, but no server-managed token is available.");
+        }
+        const localPath = await this.deps.dataIngestor.ensureData(datasetId, source, () => undefined);
+        let copiedTo;
+        if (options.targetDir) {
+            copiedTo = this.copyDownloadOutput(localPath, options.targetDir);
+            notes.push(`Copied dataset output to ${copiedTo}.`);
+        }
+        return {
+            dataset_id: datasetId,
+            resolved_source: source,
+            local_path: localPath,
+            copied_to: copiedTo,
+            notes,
+        };
+    }
+    async info(options) {
+        const requested = String(options.datasetId || "").trim();
+        if (!requested) {
+            throw new Error("dataset_id is required");
+        }
+        const resolved = this.resolveDatasetReference(requested, options.source || "auto");
+        const metadataMatch = this.lookupKnownDataset(requested) || (resolved.datasetId !== requested ? this.lookupKnownDataset(resolved.datasetId) : undefined);
+        if (metadataMatch) {
+            return {
+                dataset_id: requested,
+                resolved_source: metadataMatch.source,
+                notes: [],
+                dataset: metadataMatch,
+            };
+        }
+        if (resolved.source === "s3") {
+            return {
+                dataset_id: requested,
+                resolved_source: "s3",
+                notes: ["S3 info is derived from the object URI. Discovery/listing is intentionally not supported."],
+                dataset: {
+                    id: requested,
+                    source: "s3",
+                    uri: this.toS3HttpsUrl(resolved.datasetId),
+                },
+            };
+        }
+        if (resolved.source === "bigquery") {
+            return {
+                dataset_id: requested,
+                resolved_source: "bigquery",
+                notes: ["BigQuery is reserved for operator-managed connectors. Detailed inspection is not implemented in this patch."],
+            };
+        }
+        const discovery = await this.discover({
+            query: resolved.datasetId,
+            source: resolved.source || "auto",
+            limit: 5,
+            publicOnly: options.publicOnly !== false,
+        });
+        const exact = discovery.results.find(dataset => this.matchesDatasetReference(dataset, requested));
+        return {
+            dataset_id: requested,
+            resolved_source: exact?.source,
+            notes: discovery.notes,
+            dataset: exact || discovery.results[0],
+        };
+    }
+    async discoverFromSource(source, query, limit) {
+        switch (source) {
+            case "huggingface":
+                return await new HuggingFaceScraper().scrape(limit, true, await analyzeDatasetQuery(query));
+            case "openml":
+                return await this.deps.openmlSource.discover(query, limit);
+            case "kaggle":
+                return await this.deps.kaggleSource.discover(query, limit);
+            case "dataworld":
+                return await this.deps.dataworldSource.discover(query, limit);
+            case "s3":
+                throw new Error("S3 does not support search/discovery in the unified gateway. Use a direct s3://bucket/key object reference.");
+            case "bigquery":
+                throw new Error("BigQuery discovery is not implemented in the unified gateway.");
+            default:
+                throw new Error(`Unsupported provider: ${source}`);
+        }
+    }
+    resolveDiscoverSources(source, publicOnly, notes) {
+        if (source !== "auto") {
+            if (source === "kaggle" && !this.deps.dataIngestor.hasKaggleCredentials()) {
+                throw new Error("Kaggle requires server-managed credentials and none are configured.");
+            }
+            if (source === "dataworld" && !this.deps.hasDataWorldToken()) {
+                throw new Error("data.world requires a server-managed token and none is configured.");
+            }
+            if (source === "s3" || source === "bigquery") {
+                throw new Error(`${source} does not currently support discover operation through the gateway.`);
+            }
+            return [source];
+        }
+        const providers = ["huggingface", "openml"];
+        if (!publicOnly && this.deps.dataIngestor.hasKaggleCredentials()) {
+            providers.push("kaggle");
+        }
+        else if (!publicOnly) {
+            notes.push("Kaggle skipped because no server-managed credentials are configured.");
+        }
+        if (!publicOnly && this.deps.hasDataWorldToken()) {
+            providers.push("dataworld");
+        }
+        else if (!publicOnly) {
+            notes.push("data.world skipped because no server-managed token is configured.");
+        }
+        return providers;
+    }
+    resolveDatasetReference(datasetId, source) {
+        const trimmed = datasetId.trim();
+        if (source !== "auto") {
+            if (source === "s3") {
+                return { source, datasetId: trimmed };
+            }
+            return { source, datasetId: this.stripSourcePrefix(trimmed, source) };
+        }
+        if (/^s3:\/\//i.test(trimmed) || /^https?:\/\/[^\s]+\.s3[.-][^\s]+/i.test(trimmed) || /^https?:\/\/s3\.[^\s]+amazonaws\.com\//i.test(trimmed)) {
+            return { source: "s3", datasetId: trimmed };
+        }
+        if (/^kaggle:/i.test(trimmed))
+            return { source: "kaggle", datasetId: trimmed.replace(/^kaggle:/i, "") };
+        if (/^(huggingface|hf):/i.test(trimmed))
+            return { source: "huggingface", datasetId: trimmed.replace(/^(huggingface|hf):/i, "") };
+        if (/^openml:/i.test(trimmed))
+            return { source: "openml", datasetId: trimmed.replace(/^openml:/i, "") };
+        if (/^dataworld:/i.test(trimmed))
+            return { source: "dataworld", datasetId: trimmed.replace(/^dataworld:/i, "") };
+        if (/^bigquery:/i.test(trimmed))
+            return { source: "bigquery", datasetId: trimmed.replace(/^bigquery:/i, "") };
+        if (/^\d+$/.test(trimmed))
+            return { source: "openml", datasetId: trimmed };
+        if (trimmed.includes("/") && !trimmed.includes(":"))
+            return { source: "huggingface", datasetId: trimmed };
+        return { datasetId: trimmed };
+    }
+    stripSourcePrefix(datasetId, source) {
+        if (source === "huggingface") {
+            return datasetId.replace(/^(huggingface|hf):/i, "");
+        }
+        return datasetId.replace(new RegExp(`^${source}:`, "i"), "");
+    }
+    lookupKnownDataset(datasetId) {
+        const candidates = new Set([
+            datasetId,
+            datasetId.replace(/^(huggingface|hf|kaggle|openml|dataworld|bigquery):/i, ""),
+        ]);
+        for (const candidate of candidates) {
+            const dataset = this.deps.metadataStore.getDataset(candidate);
+            if (dataset)
+                return dataset;
+        }
+        return undefined;
+    }
+    matchesDatasetReference(dataset, requested) {
+        const normalizedRequested = requested.replace(/^(huggingface|hf|kaggle|openml|dataworld|bigquery):/i, "").toLowerCase();
+        const fullId = `${dataset.source}:${dataset.id}`.toLowerCase();
+        return dataset.id.toLowerCase() === normalizedRequested || fullId === requested.toLowerCase();
+    }
+    toIngestSource(source) {
+        if (source === "huggingface" || source === "openml" || source === "kaggle" || source === "dataworld") {
+            return source;
+        }
+        return undefined;
+    }
+    rankDataset(dataset) {
+        const relevance = Number(dataset.relevance_score || 0) * 1000;
+        const quality = Number(dataset.quality_score || 0) * 100;
+        const downloads = Number(dataset.downloads || 0);
+        return relevance + quality + downloads;
+    }
+    copyDownloadOutput(localPath, targetDir) {
+        const resolvedTargetDir = path.resolve(targetDir);
+        fs.mkdirSync(resolvedTargetDir, { recursive: true });
+        const destination = path.join(resolvedTargetDir, path.basename(localPath));
+        fs.cpSync(localPath, destination, { recursive: true, force: true });
+        return destination;
+    }
+    async downloadPublicS3Object(datasetId, targetDir) {
+        const httpsUrl = this.toS3HttpsUrl(datasetId);
+        const parsed = new URL(httpsUrl);
+        const fileName = path.basename(parsed.pathname) || "s3-object.bin";
+        const outputDir = path.resolve(targetDir || path.join(this.deps.dataRoot, "data", "raw"));
+        fs.mkdirSync(outputDir, { recursive: true });
+        const outputPath = path.join(outputDir, fileName);
+        await this.downloadToFile(httpsUrl, outputPath);
+        this.deps.metadataStore.registerDownload(datasetId, outputPath, "completed", fs.statSync(outputPath).size);
+        return outputPath;
+    }
+    toS3HttpsUrl(datasetId) {
+        if (/^https?:\/\//i.test(datasetId)) {
+            return datasetId;
+        }
+        const match = datasetId.match(/^s3:\/\/([^/]+)\/(.+)$/i);
+        if (!match) {
+            throw new Error("S3 source expects an s3://bucket/key object reference or a direct HTTPS S3 URL.");
+        }
+        const bucket = match[1];
+        const objectKey = match[2].split("/").map(encodeURIComponent).join("/");
+        return `https://${bucket}.s3.amazonaws.com/${objectKey}`;
+    }
+    async downloadToFile(url, destination) {
+        await new Promise((resolve, reject) => {
+            const transport = url.startsWith("https:") ? https : http;
+            const request = transport.get(url, response => {
+                const statusCode = response.statusCode || 0;
+                const location = response.headers.location;
+                if (statusCode >= 300 && statusCode < 400 && location) {
+                    response.resume();
+                    this.downloadToFile(location, destination).then(resolve).catch(reject);
+                    return;
+                }
+                if (statusCode < 200 || statusCode >= 300) {
+                    response.resume();
+                    reject(new Error(`Download failed with status ${statusCode}`));
+                    return;
+                }
+                const file = fs.createWriteStream(destination);
+                response.pipe(file);
+                file.on("finish", () => {
+                    file.close();
+                    resolve();
+                });
+                file.on("error", error => {
+                    try {
+                        file.close();
+                    }
+                    catch {
+                        // no-op
+                    }
+                    reject(error);
+                });
+            });
+            request.on("error", reject);
+        });
+    }
+}