@vespermcp/mcp-server 1.2.21 → 1.2.24

package/build/python/kaggle_engine.py

@@ -12,6 +12,19 @@ except Exception:
     HAS_KAGGLE = False
 
 
+IMAGE_EXTENSIONS = {
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".webp",
+    ".bmp",
+    ".gif",
+    ".tiff",
+    ".tif",
+    ".svg",
+}
+
+
 def _ensure_auth() -> Dict[str, Any]:
     if not HAS_KAGGLE:
         return {
@@ -135,7 +148,56 @@ def discover(query: str, limit: int = 20) -> Dict[str, Any]:
         return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}
 
 
-def _pick_best_file(root: str) -> str:
+def _find_image_files(root: str) -> List[str]:
+    image_files: List[str] = []
+    for base, _, files in os.walk(root):
+        for name in files:
+            full = os.path.join(base, name)
+            if os.path.splitext(name)[1].lower() in IMAGE_EXTENSIONS:
+                image_files.append(full)
+    image_files.sort()
+    return image_files
+
+
+def _infer_image_record(root: str, full_path: str, index: int) -> Dict[str, Any]:
+    relative_path = os.path.relpath(full_path, root).replace("\\", "/")
+    parent_dir = os.path.dirname(relative_path)
+    parts = [part for part in parent_dir.split("/") if part and part != "."]
+
+    split = None
+    label = None
+    if parts:
+        first = parts[0].lower()
+        if first in {"train", "test", "val", "valid", "validation"}:
+            split = parts[0]
+            if len(parts) > 1:
+                label = parts[-1]
+        else:
+            label = parts[-1]
+
+    record: Dict[str, Any] = {
+        "id": index,
+        "image_path": os.path.abspath(full_path),
+        "relative_path": relative_path,
+        "file_name": os.path.basename(full_path),
+        "extension": os.path.splitext(full_path)[1].lower().lstrip("."),
+    }
+    if split:
+        record["split"] = split
+    if label:
+        record["label"] = label
+    return record
+
+
+def _write_image_manifest(root: str, image_files: List[str]) -> str:
+    manifest_path = os.path.join(root, "_vesper_image_manifest.jsonl")
+    with open(manifest_path, "w", encoding="utf-8") as handle:
+        for index, full_path in enumerate(image_files):
+            handle.write(json.dumps(_infer_image_record(root, full_path, index), ensure_ascii=False) + "\n")
+    return manifest_path
+
+
+def _pick_best_file(root: str) -> Dict[str, Any]:
     candidates: List[str] = []
     for base, _, files in os.walk(root):
         for name in files:
@@ -145,6 +207,14 @@ def _pick_best_file(root: str) -> str:
                 candidates.append(full)
 
     if not candidates:
+        image_files = _find_image_files(root)
+        if image_files:
+            manifest_path = _write_image_manifest(root, image_files)
+            return {
+                "local_path": manifest_path,
+                "dataset_kind": "image-manifest",
+                "image_count": len(image_files),
+            }
         raise RuntimeError("No suitable data file found after download")
 
     # prioritize common tabular formats
@@ -152,8 +222,8 @@ def _pick_best_file(root: str) -> str:
     for ext in priorities:
         for c in candidates:
             if c.lower().endswith(ext):
-                return c
-    return candidates[0]
+                return {"local_path": c, "dataset_kind": "tabular", "image_count": 0}
+    return {"local_path": candidates[0], "dataset_kind": "tabular", "image_count": 0}
 
 
 def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
@@ -174,12 +244,14 @@ def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
 
         # unzip in place, remove zip for convenience
         api.dataset_download_files(dataset_ref, path=target_dir, unzip=True, quiet=True)
-        best_file = _pick_best_file(target_dir)
+        artifact = _pick_best_file(target_dir)
         return {
             "ok": True,
             "dataset_id": dataset_ref,
             "target_dir": target_dir,
-            "local_path": best_file,
+            "local_path": artifact["local_path"],
+            "dataset_kind": artifact["dataset_kind"],
+            "image_count": artifact.get("image_count", 0),
         }
     except Exception as e:
         msg = str(e)

package/build/python/normalize_engine.py

@@ -0,0 +1,83 @@
+"""
+Normalize any supported dataset file to parquet format.
+Usage: normalize_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        # Try NDJSON
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        # Try wrapper object
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            # Last resort - take first list value
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    if ext == ".txt":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def normalize(input_path: str, output_path: str):
+    df = _load(input_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    df.write_parquet(output_path)
+    return df.height
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        rows = normalize(input_path, output_path)
+        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/build/python/vesper/core/asset_downloader.py

@@ -191,6 +191,7 @@ class AssetDownloader:
         kaggle_ref: Optional[str] = None,
         urls: Optional[List[str]] = None,
         output_format: str = "webdataset",
+        output_dir: Optional[str] = None,
         max_items: Optional[int] = None,
         image_column: Optional[str] = None,
     ) -> Dict[str, Any]:
@@ -231,7 +232,10 @@ class AssetDownloader:
                 raise ValueError("urls are required for source=url")
 
         # --- Now safe to create directories ---
-        dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
+        if output_dir:
+            dataset_dir = Path(output_dir).expanduser().resolve()
+        else:
+            dataset_dir = self.output_root / dataset_id.replace("/", "_").replace(":", "_")
         images_dir = dataset_dir / "images"
         dataset_dir.mkdir(parents=True, exist_ok=True)
         images_dir.mkdir(parents=True, exist_ok=True)

package/build/scripts/test-phase1-webcore-quality.js

@@ -0,0 +1,104 @@
+import { CacheService, MockRedisProvider } from "../cache/service.js";
+import { ArxivSource } from "../metadata/arxiv-source.js";
+import { GithubSource } from "../metadata/github-source.js";
+import { SemanticScholarSource } from "../metadata/semantic-scholar-source.js";
+import { HackerNewsSource } from "../metadata/hackernews-source.js";
+import { WebCoreEngine } from "../web/web-core.js";
+function percentile(values, p) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const idx = Math.max(0, Math.ceil(p * sorted.length) - 1);
+    return sorted[idx] ?? 0;
+}
+function makeQueries() {
+    const topics = [
+        "agentic RAG evaluation",
+        "tool-augmented retrieval",
+        "LLM dataset quality scoring",
+        "semantic deduplication embeddings",
+        "cross-source dataset fusion",
+        "retrieval augmented generation metrics",
+        "dataset export parquet arrow jsonl",
+        "data safety dataset provenance",
+        "synthetic data generation alignment",
+        "multi-source corpus building",
+    ];
+    const suffixes = [
+        "paper",
+        "benchmark",
+        "latency",
+        "quality",
+        "dedup",
+        "provenance",
+        "evaluation",
+        "pipeline",
+        "MCP",
+        "agents",
+    ];
+    const out = [];
+    for (let i = 0; out.length < 100; i++) {
+        const t = topics[i % topics.length];
+        const s = suffixes[Math.floor(i / topics.length) % suffixes.length];
+        out.push(`${t} ${s}`.trim());
+    }
+    return out.slice(0, 100);
+}
+async function main() {
+    const cache = new CacheService(new MockRedisProvider());
+    const engine = new WebCoreEngine({
+        arxivSource: new ArxivSource(cache),
+        githubSource: new GithubSource(cache),
+        semanticScholarSource: new SemanticScholarSource(cache),
+        hackerNewsSource: new HackerNewsSource(cache),
+    });
+    const baseQuery = "agentic RAG evaluation";
+    const flags = { arxiv_full_text: true, github_include_readme: true };
+    const sources = ["arxiv", "github"];
+    console.log("Phase 1 Web Core validation:");
+    console.log("- Checking cached latency regression on a single query...");
+    const run1 = await engine.find({ query: baseQuery, sources: [...sources], limit: 2, ...flags });
+    const run2 = await engine.find({ query: baseQuery, sources: [...sources], limit: 2, ...flags });
+    const run2Arxiv = run2.telemetry?.per_source.find((t) => t.source === "arxiv");
+    const run2Github = run2.telemetry?.per_source.find((t) => t.source === "github");
+    console.log("Cached telemetry (run2):", {
+        arxiv: run2Arxiv ? { cache_hit: run2Arxiv.cache_hit, latency_ms: run2Arxiv.latency_ms } : null,
+        github: run2Github ? { cache_hit: run2Github.cache_hit, latency_ms: run2Github.latency_ms } : null,
+    });
+    // 100 query quality distribution test
+    console.log("- Running 100 test queries (quality distribution + extraction latency)...");
+    const queries = makeQueries();
+    const qualityScores = [];
+    const pdfExtractMs = [];
+    for (let i = 0; i < queries.length; i++) {
+        const q = queries[i];
+        const res = await engine.find({ query: q, sources: [...sources], limit: 2, ...flags });
+        for (const doc of res.results) {
+            qualityScores.push(Number(doc.quality_score));
+        }
+        const arxivTelemetry = res.telemetry?.per_source.find((t) => t.source === "arxiv");
+        if (arxivTelemetry) {
+            pdfExtractMs.push(Number(arxivTelemetry.pdf_extract_ms_total || 0));
+        }
+        if ((i + 1) % 10 === 0) {
+            console.log(`  progress: ${i + 1}/100`);
+        }
+    }
+    const q95 = percentile(qualityScores, 0.95);
+    const pdfP95 = percentile(pdfExtractMs, 0.95);
+    console.log("\nResults:");
+    console.log(`- Quality score p95: ${q95}`);
+    console.log(`- PDF extract ms p95 (arxiv): ${pdfP95}`);
+    const okQuality = q95 > 0.9;
+    const okPdfLatency = pdfP95 < 5000;
+    console.log(`\nSuccess criteria:`);
+    console.log(`- Quality p95 > 0.9: ${okQuality ? "PASS" : "FAIL"}`);
+    console.log(`- PDF extract p95 < 5000ms additional: ${okPdfLatency ? "PASS" : "FAIL"}`);
+    if (!okQuality) {
+        console.log("Tip: adjust estimateQualityScore() weights/thresholds in src/metadata/quality.ts then rerun.");
+    }
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});

package/build/search/engine.js

@@ -1,4 +1,5 @@
 import { JITOrchestrator } from "./jit-orchestrator.js";
+import { analyzeDatasetQuery, scoreDatasetAgainstIntent, shouldExcludeByLanguage } from "./query-intent.js";
 import fs from "fs";
 function log(msg) {
     fs.appendFileSync("debug.log", new Date().toISOString() + " " + msg + "\n");
@@ -17,9 +18,10 @@ export class SearchEngine {
     async search(query, options = {}) {
         const limit = options.limit || 5;
         const enableJIT = options.enableJIT !== false; // Default: true
+        const intent = await analyzeDatasetQuery(query);
         log(`Search request: "${query}" Limit=${limit} Safe=${options.safeOnly} JIT=${enableJIT}`);
         // 1. Perform local search
-        const localResults = await this.localSearch(query, options);
+        const localResults = await this.localSearch(query, options, intent);
         // 2. Check if JIT should be triggered
         const shouldTrigger = enableJIT && this.shouldTriggerJIT(localResults, query);
         if (!shouldTrigger) {
@@ -28,10 +30,10 @@ export class SearchEngine {
         }
         // 3. Trigger JIT fallback
         console.error(`\nWARNING: Low confidence results (${localResults.length} found, top score: ${localResults[0]?.relevance_score || 0})`);
-        await this.jitOrchestrator.fetchAndIngest(query, 10);
+        await this.jitOrchestrator.fetchAndIngest(query, 10, intent);
         // 4. Re-run local search with updated index
         console.error(`Re-searching with updated library...`);
-        const enhancedResults = await this.localSearch(query, options);
+        const enhancedResults = await this.localSearch(query, options, intent);
         const newCount = enhancedResults.length - localResults.length;
         if (newCount > 0) {
             console.error(`Found ${newCount} additional results\n`);
@@ -41,7 +43,7 @@ export class SearchEngine {
     /**
      * Perform hybrid search (Vector + Lexical + Penalties)
      */
-    async localSearch(query, options) {
+    async localSearch(query, options, intent) {
         const limit = options.limit || 5;
         // 1. Parse Query
         const words = query.toLowerCase().split(/\s+/);
@@ -66,6 +68,12 @@ export class SearchEngine {
             // Filter: Safe only
             if (options.safeOnly && metadata.license.category === "restricted")
                 continue;
+            // Filter: Hard language exclusion — completely drop bilingual/multilingual datasets
+            // when user explicitly requested a single language
+            if (shouldExcludeByLanguage(metadata, intent)) {
+                log(`Language exclusion: Dropped ${match.id} (bilingual/multilingual for single-language query)`);
+                continue;
+            }
             const text = `${metadata.name} ${metadata.description} ${metadata.tags.join(" ")}`.toLowerCase();
             // Filter: Explicit Negative Keywords
             if (negativeKeywords.some(neg => text.includes(neg))) {
@@ -96,6 +104,34 @@ export class SearchEngine {
             if (lexicalScore === 0 && positiveKeywords.length > 1) {
                 penalty += 0.2;
             }
+            // Penalty: Modality Mismatch
+            // Infer the expected modality from the query and penalize mismatches.
+            // e.g., "anime quotes" is text, so image-classification datasets get penalized.
+            const textIndicators = ["quotes", "text", "nlp", "sentiment", "review", "comment", "caption", "dialogue", "chat", "translation", "summarization", "classification"];
+            const imageIndicators = ["image", "photo", "picture", "vision", "detection", "segmentation", "face", "background"];
+            const queryLower = query.toLowerCase();
+            const queryLooksText = textIndicators.some(t => queryLower.includes(t));
+            const queryLooksImage = imageIndicators.some(t => queryLower.includes(t));
+            if (queryLooksText && !queryLooksImage) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isImageResult = resultTask.includes("image") || resultTask.includes("object-detection") ||
+                    text.includes("image classification") || text.includes("image-classification") ||
+                    text.includes("object detection") || text.includes("image segmentation");
+                if (isImageResult) {
+                    log(`Modality penalty: text query but image dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
+            if (queryLooksImage && !queryLooksText) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isTextResult = resultTask.includes("text-classification") || resultTask.includes("text-generation") ||
+                    resultTask.includes("translation") || resultTask.includes("summarization") ||
+                    resultTask.includes("question-answering");
+                if (isTextResult) {
+                    log(`Modality penalty: image query but text dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
             // D. Accessibility Bonuses (Prioritize low-friction sources)
             let bonus = 0;
             const sourceBonuses = {
@@ -103,16 +139,19 @@ export class SearchEngine {
                 "uci": 0.1,
                 "github": 0.1,
                 "worldbank": 0.1,
-                "nasa": 0.1
+                "nasa": 0.1,
+                "arxiv": 0.1
             };
             bonus = sourceBonuses[metadata.source] || 0;
             // Final Combined Score
             // 70% Vector, 30% Lexical, minus Penalties, plus Bonuses
-            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus;
+            const intentScore = scoreDatasetAgainstIntent(metadata, intent);
+            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus + intentScore;
             metadata.relevance_score = Math.round(finalScore * 100) / 100;
             metadata.vector_score = Math.round(vectorScore * 100) / 100;
             metadata.lexical_score = Math.round(lexicalScore * 100) / 100;
             metadata.accessibility_bonus = bonus;
+            metadata.intent_score = intentScore;
             results.push(metadata);
         }
         // Sort by final score and limit

package/build/search/jit-orchestrator.js

@@ -2,6 +2,7 @@ import { HuggingFaceScraper } from "../metadata/scraper.js";
 import { UCIScraper } from "../metadata/uci-scraper.js";
 import { GitHubScraper } from "../metadata/github-scraper.js";
 import { WorldBankScraper, NASAScraper } from "../metadata/institutional-scrapers.js";
+import { analyzeDatasetQuery, buildIntentSearchQuery } from "./query-intent.js";
 // Common stop words to filter out for better search
 const STOP_WORDS = new Set([
     "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
@@ -61,7 +62,7 @@ export class JITOrchestrator {
     /**
      * Main JIT workflow: fetch, save, index, return new datasets
      */
-    async fetchAndIngest(query, limit = 10) {
+    async fetchAndIngest(query, limit = 10, providedIntent) {
         // Rate limiting check
         if (!this.canTrigger(query)) {
             console.error(`[JIT] Query "${query}" was searched recently. Waiting...`);
@@ -69,9 +70,12 @@ export class JITOrchestrator {
         }
         console.error(`\n[JIT] Searching live sources for: "${query}"`);
         this.lastTriggerTime.set(query, Date.now());
-        // Simplify query for better API results
-        const keywords = this.simplifyQuery(query);
-        if (keywords.length > 0) {
+        const intent = providedIntent || await analyzeDatasetQuery(query);
+        const keywords = this.simplifyQuery(buildIntentSearchQuery(intent));
+        if (intent.llmBacked || intent.language || intent.task || intent.domain || intent.minRows) {
+            console.error(`[JIT] Intent: ${JSON.stringify({ language: intent.language, task: intent.task, domain: intent.domain, minRows: intent.minRows, searchQuery: intent.searchQuery })}`);
+        }
+        else if (keywords.length > 0) {
             console.error(`[JIT] Keywords extracted: ${keywords.join(", ")}`);
         }
         const newDatasets = [];
@@ -81,15 +85,16 @@ export class JITOrchestrator {
             // Get existing dataset IDs to avoid duplicates
             const existing = this.metadataStore.getAllDatasets();
             existing.forEach(ds => existingIds.add(ds.id));
-            // 1. Scrape HuggingFace - try each keyword separately for better results
-            let hfResults = [];
-            for (const keyword of keywords) {
-                if (hfResults.length >= limit)
-                    break;
-                const results = await this.scrapeHuggingFace(keyword, Math.ceil(limit / keywords.length));
-                for (const ds of results) {
-                    if (!hfResults.some(existing => existing.id === ds.id)) {
-                        hfResults.push(ds);
+            let hfResults = await this.scrapeHuggingFace(intent, limit);
+            if (hfResults.length < Math.max(3, Math.floor(limit / 2))) {
+                for (const keyword of keywords) {
+                    if (hfResults.length >= limit)
+                        break;
+                    const results = await this.scrapeHuggingFace(keyword, Math.ceil(limit / Math.max(keywords.length, 1)));
+                    for (const ds of results) {
+                        if (!hfResults.some(existing => existing.id === ds.id)) {
+                            hfResults.push(ds);
+                        }
                     }
                 }
             }
@@ -170,7 +175,6 @@ export class JITOrchestrator {
     async scrapeHuggingFace(query, limit) {
         const scraper = new HuggingFaceScraper();
         try {
-            // Pass the query as a general search term
             return await scraper.scrape(limit, true, query);
         }
         catch (error) {