@vespermcp/mcp-server 1.2.21 → 1.2.24

package/build/web/web-core.js

@@ -0,0 +1,242 @@
+function nowIso() {
+    return new Date().toISOString();
+}
+function clamp01(x) {
+    if (!Number.isFinite(x))
+        return 0;
+    return Math.max(0, Math.min(1, x));
+}
+function scoreTo01(score0to100) {
+    return clamp01(Number(score0to100 || 0) / 100);
+}
+export class WebCoreEngine {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async find(input) {
+        const query = String(input.query || "").trim();
+        if (!query)
+            throw new Error("query is required");
+        const limit = Math.max(1, Math.min(50, Number(input.limit || 10)));
+        const requested = (input.sources && input.sources.length > 0 ? input.sources : ["arxiv", "github"])
+            .filter((s) => s === "arxiv" || s === "github" || s === "semantic_scholar" || s === "hackernews");
+        const arxivFullText = input.arxiv_full_text === true;
+        const githubIncludeReadme = input.github_include_readme === true;
+        const collectedAt = nowIso();
+        const results = [];
+        const telemetry = { per_source: [] };
+        const perSource = Math.max(1, Math.ceil(limit / Math.max(1, requested.length)));
+        for (const source of requested) {
+            if (source === "arxiv") {
+                const t0 = Date.now();
+                try {
+                    const out = await this.deps.arxivSource.discoverWithTelemetry(query, perSource, { full_text: arxivFullText });
+                    const docs = out.results.map((m) => this.fromArxiv(m, collectedAt)).filter((d) => d !== null);
+                    results.push(...docs);
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: out.cacheHit,
+                        latency_ms: out.latencyMs || (Date.now() - t0),
+                        result_count: docs.length,
+                        pdf_extract_ms_total: out.pdf_extract_ms_total,
+                    });
+                }
+                catch (e) {
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: false,
+                        latency_ms: Date.now() - t0,
+                        result_count: 0,
+                        error: e?.message || String(e),
+                    });
+                }
+            }
+            if (source === "github") {
+                const t0 = Date.now();
+                try {
+                    const out = await this.deps.githubSource.discoverWithTelemetry(query, perSource, { include_readme: githubIncludeReadme });
+                    const docs = out.results.map((m) => this.fromGithub(m, collectedAt)).filter((d) => d !== null);
+                    results.push(...docs);
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: out.cacheHit,
+                        latency_ms: out.latencyMs || (Date.now() - t0),
+                        result_count: docs.length,
+                        readme_fetch_ms_total: out.readme_fetch_ms_total,
+                    });
+                }
+                catch (e) {
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: false,
+                        latency_ms: Date.now() - t0,
+                        result_count: 0,
+                        error: e?.message || String(e),
+                    });
+                }
+            }
+            if (source === "semantic_scholar") {
+                const t0 = Date.now();
+                try {
+                    const out = await this.deps.semanticScholarSource.discoverWithTelemetry(query, perSource);
+                    const docs = out.results.map((m) => this.fromSemanticScholar(m, collectedAt)).filter((d) => d !== null);
+                    results.push(...docs);
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: out.cacheHit,
+                        latency_ms: out.latencyMs || (Date.now() - t0),
+                        result_count: docs.length,
+                    });
+                }
+                catch (e) {
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: false,
+                        latency_ms: Date.now() - t0,
+                        result_count: 0,
+                        error: e?.message || String(e),
+                    });
+                }
+            }
+            if (source === "hackernews") {
+                const t0 = Date.now();
+                try {
+                    const out = await this.deps.hackerNewsSource.discoverWithTelemetry(query, perSource);
+                    const docs = out.results.map((m) => this.fromHackerNews(m, collectedAt)).filter((d) => d !== null);
+                    results.push(...docs);
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: out.cacheHit,
+                        latency_ms: out.latencyMs || (Date.now() - t0),
+                        result_count: docs.length,
+                    });
+                }
+                catch (e) {
+                    telemetry.per_source.push({
+                        source,
+                        cache_hit: false,
+                        latency_ms: Date.now() - t0,
+                        result_count: 0,
+                        error: e?.message || String(e),
+                    });
+                }
+            }
+        }
+        return {
+            query,
+            sources: requested,
+            results: results.slice(0, limit),
+            collected_at: collectedAt,
+            telemetry,
+        };
+    }
+    fromArxiv(meta, collectedAt) {
+        const arxivId = meta.id;
+        const absUrl = meta.metadata_url || `https://arxiv.org/abs/${arxivId}`;
+        const pdfUrl = meta.download_url || `https://arxiv.org/pdf/${arxivId}`;
+        const title = meta.name;
+        const abstract = meta.description;
+        const authors = Array.isArray(meta.authors) ? meta.authors : undefined;
+        const content = meta.webcore_content || abstract;
+        if (!absUrl || !content)
+            return null;
+        return {
+            source_type: "arxiv",
+            source_url: absUrl,
+            content,
+            metadata_json: {
+                arxiv_id: arxivId,
+                title,
+                authors,
+                categories: meta.tags,
+                updated_at: meta.last_updated,
+                pdf_url: pdfUrl,
+                abstract,
+                pdf_text_present: !!meta.webcore_content,
+                pdf_text_preview_length: meta.webcore_content ? meta.webcore_content.length : 0,
+            },
+            quality_score: scoreTo01(meta.quality_score),
+            collected_at: collectedAt,
+            content_type: meta.webcore_content ? "paper_fulltext" : "paper_abstract",
+        };
+    }
+    fromGithub(meta, collectedAt) {
+        const fullName = meta.id;
+        const url = meta.metadata_url || meta.download_url || `https://github.com/${fullName}`;
+        const content = meta.webcore_content || meta.description || meta.name;
+        if (!url || !content)
+            return null;
+        return {
+            source_type: "github",
+            source_url: url,
+            content,
+            metadata_json: {
+                full_name: fullName,
+                name: meta.name,
+                description: meta.description,
+                stars: meta.stars,
+                forks_proxy_downloads: meta.downloads,
+                topics: meta.tags,
+                license_id: meta.license?.id,
+                license_name: meta.license?.name,
+                updated_at: meta.last_updated,
+                language: (meta.languages || [])[0],
+                readme_present: !!meta.webcore_content,
+                readme_preview_length: meta.webcore_content ? meta.webcore_content.length : 0,
+            },
+            quality_score: scoreTo01(meta.quality_score),
+            collected_at: collectedAt,
+            content_type: meta.webcore_content ? "text" : "repo",
+        };
+    }
+    fromSemanticScholar(meta, collectedAt) {
+        const paperId = meta.id;
+        const url = meta.metadata_url || meta.download_url || (paperId ? `https://www.semanticscholar.org/paper/${paperId}` : "");
+        const content = meta.description;
+        if (!url || !content)
+            return null;
+        return {
+            source_type: "semantic_scholar",
+            source_url: url,
+            content,
+            metadata_json: {
+                paper_id: paperId,
+                title: meta.name,
+                authors: meta.authors,
+                abstract: meta.description,
+                tags: meta.tags,
+                citation_count: meta.stars,
+                updated_at: meta.last_updated,
+            },
+            quality_score: scoreTo01(meta.quality_score),
+            collected_at: collectedAt,
+            content_type: "paper_abstract",
+        };
+    }
+    fromHackerNews(meta, collectedAt) {
+        const itemId = meta.id;
+        const url = meta.metadata_url || meta.download_url || (itemId ? `https://news.ycombinator.com/item?id=${itemId}` : "");
+        const content = meta.description;
+        const authors = meta.authors;
+        if (!url || !content)
+            return null;
+        return {
+            source_type: "hackernews",
+            source_url: url,
+            content,
+            metadata_json: {
+                item_id: itemId,
+                title: meta.name,
+                authors,
+                points: meta.stars,
+                comments_proxy: meta.downloads,
+                tags: meta.tags,
+                updated_at: meta.last_updated,
+            },
+            quality_score: scoreTo01(meta.quality_score),
+            collected_at: collectedAt,
+            content_type: "thread",
+        };
+    }
+}

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@vespermcp/mcp-server",
-  "version": "1.2.21",
+  "version": "1.2.24",
   "description": "AI-powered dataset discovery, quality analysis, and preparation MCP server with multimodal support (text, image, audio, video)",
   "type": "module",
   "main": "build/index.js",
   "bin": {
-    "mcp-server": "./build/index.js",
-    "vespermcp": "./build/index.js",
-    "vesper-wizard": "scripts/wizard.js"
+    "vespermcp": "build/index.js",
+    "vesper-wizard": "wizard.cjs"
   },
   "files": [
     "build/**/*",
     "src/python/**/*",
+    "wizard.cjs",
     "scripts/**/*",
     "README.md",
     "LICENSE",
@@ -31,6 +31,7 @@
     "fuse": "node build/index.js fuse",
     "discover": "node build/index.js discover",
     "download": "node build/index.js download",
+    "export": "node build/index.js export",
     "config": "node build/index.js config",
     "test-fusion-engine": "py src/python/test_fusion_engine.py",
     "setup": "node build/index.js --setup",
@@ -59,15 +60,20 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/vesper/mcp-server"
+    "url": "git+https://github.com/vesper/mcp-server.git"
   },
   "engines": {
     "node": ">=18.0.0",
     "npm": ">=8.0.0"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1017.0",
+    "@aws-sdk/credential-providers": "^3.1017.0",
     "@huggingface/hub": "^2.7.1",
     "@modelcontextprotocol/sdk": "^1.25.2",
+    "@polar-sh/nextjs": "^0.9.4",
+    "@supabase/supabase-js": "^2.98.0",
+    "@vercel/analytics": "^2.0.0",
     "@xenova/transformers": "^2.17.2",
     "adm-zip": "^0.5.16",
     "ajv": "^8.17.1",
@@ -75,6 +81,7 @@
     "better-sqlite3": "^12.6.0",
     "inquirer": "^13.3.0",
     "lodash": "^4.17.21",
+    "pdf-parse": "^2.4.5",
     "uuid": "^13.0.0",
     "zod": "^4.3.5",
     "zod-to-json-schema": "^3.25.1"

package/scripts/postinstall.cjs

@@ -2,13 +2,34 @@
 
 const { execSync } = require('child_process');
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
 
 console.log('\n🚀 Setting up Vesper MCP Server...\n');
 
+function getPythonBootstrapCommand() {
+    const attempts = process.platform === 'win32'
+        ? ['py -3', 'python']
+        : ['python3', 'python'];
+
+    for (const command of attempts) {
+        try {
+            execSync(`${command} --version`, { stdio: 'pipe' });
+            return command;
+        } catch {
+            // try next command
+        }
+    }
+
+    return null;
+}
+
 // 1. Check for Python
+const pythonBootstrap = getPythonBootstrapCommand();
 try {
-    execSync('python --version', { stdio: 'pipe' });
+    if (!pythonBootstrap) {
+        throw new Error('Python not found');
+    }
     console.log('✅ Python found');
 } catch (e) {
     console.warn('⚠️  Python not found. Please install Python 3.8+ for full functionality.');
@@ -16,36 +37,15 @@ try {
     process.exit(0); // Don't fail installation
 }
 
-// 2. Install Python dependencies
-console.log('\n📦 Installing Python dependencies...');
-const pythonPackages = [
-    'opencv-python',
-    'pillow',
-    'numpy',
-    'librosa',
-    'soundfile',
-    'aiohttp',
-    'aiofiles',
-    'datasets',
-    'webdataset',
-    'kaggle'
-];
-
-try {
-    execSync(`python -m pip install ${pythonPackages.join(' ')}`, {
-        stdio: 'inherit',
-        timeout: 120000 // 2 minutes timeout
-    });
-    console.log('✅ Python dependencies installed');
-} catch (e) {
-    console.warn('⚠️  Failed to install some Python dependencies.');
-    console.warn('   You may need to install them manually:');
-    console.warn(`   pip install ${pythonPackages.join(' ')}\n`);
-}
-
-// 3. Create data directories
-const homeDir = process.env.HOME || process.env.USERPROFILE;
+const homeDir = os.homedir() || process.env.HOME || process.env.USERPROFILE;
 const vesperDataDir = path.join(homeDir, '.vesper');
+const managedVenvDir = path.join(vesperDataDir, '.venv');
+const managedPython = process.platform === 'win32'
+    ? path.join(managedVenvDir, 'Scripts', 'python.exe')
+    : path.join(managedVenvDir, 'bin', 'python');
+const requirementsPath = path.resolve(__dirname, '..', 'src', 'python', 'requirements.txt');
+
+// 2. Create data directories
 const dirs = [
     vesperDataDir,
     path.join(vesperDataDir, 'data'),
@@ -62,7 +62,63 @@ dirs.forEach(dir => {
 
 console.log(`✅ Data directories created at ${vesperDataDir}`);
 
-// 4. Auto-configure Claude Desktop (Best Effort)
+// 3. Create a managed Vesper Python environment
+console.log('\n🐍 Preparing managed Python environment...');
+try {
+    if (!fs.existsSync(managedPython)) {
+        execSync(`${pythonBootstrap} -m venv "${managedVenvDir}"`, {
+            stdio: 'inherit',
+            timeout: 180000,
+        });
+    }
+    console.log(`✅ Managed Python ready at ${managedVenvDir}`);
+} catch (e) {
+    console.warn('⚠️  Failed to create the managed Vesper Python environment.');
+    console.warn(`   Vesper will fall back to PATH Python and may need to self-heal at runtime. ${(e && e.message) || ''}`.trim());
+}
+
+// 4. Install Python dependencies into the managed environment
+console.log('\n📦 Installing Python dependencies...');
+const pythonPackages = [
+    'opencv-python',
+    'pillow',
+    'librosa',
+    'soundfile',
+    'pyarrow'
+];
+
+try {
+    const targetPython = fs.existsSync(managedPython) ? `"${managedPython}"` : pythonBootstrap;
+    execSync(`${targetPython} -m pip install --disable-pip-version-check --upgrade pip`, {
+        stdio: 'inherit',
+        timeout: 180000,
+    });
+    execSync(`${targetPython} -m pip install --disable-pip-version-check -r "${requirementsPath}" ${pythonPackages.join(' ')}`, {
+        stdio: 'inherit',
+        timeout: 600000,
+    });
+    console.log('✅ Python dependencies installed');
+} catch (e) {
+    console.warn('⚠️  Failed to install some Python dependencies.');
+    console.warn('   You may need to install them manually into the Vesper runtime:');
+    console.warn(`   ${fs.existsSync(managedPython) ? managedPython : pythonBootstrap} -m pip install -r "${requirementsPath}" ${pythonPackages.join(' ')}\n`);
+}
+
+// 5. Rebuild better-sqlite3 for current Node.js version
+console.log('\n🔧 Rebuilding native modules for current Node.js...');
+try {
+    execSync('npm rebuild better-sqlite3', {
+        stdio: 'pipe',
+        timeout: 60000,
+        cwd: path.resolve(__dirname, '..')
+    });
+    console.log('✅ Native modules rebuilt successfully');
+} catch (e) {
+    console.warn('⚠️  Could not rebuild better-sqlite3: ' + (e.message || e));
+    console.warn('   If you see ERR_DLOPEN_FAILED, run: npm rebuild better-sqlite3');
+}
+
+// 6. Auto-configure Claude Desktop (Best Effort)
 console.log('\n⚙️  Attempting to auto-configure Claude Desktop...');
 
 function getClaudeConfigPath() {