@vespermcp/mcp-server 1.2.21 → 1.2.22

package/build/search/engine.js

@@ -1,4 +1,5 @@
 import { JITOrchestrator } from "./jit-orchestrator.js";
+import { analyzeDatasetQuery, scoreDatasetAgainstIntent, shouldExcludeByLanguage } from "./query-intent.js";
 import fs from "fs";
 function log(msg) {
     fs.appendFileSync("debug.log", new Date().toISOString() + " " + msg + "\n");
@@ -17,9 +18,10 @@ export class SearchEngine {
     async search(query, options = {}) {
         const limit = options.limit || 5;
         const enableJIT = options.enableJIT !== false; // Default: true
+        const intent = await analyzeDatasetQuery(query);
         log(`Search request: "${query}" Limit=${limit} Safe=${options.safeOnly} JIT=${enableJIT}`);
         // 1. Perform local search
-        const localResults = await this.localSearch(query, options);
+        const localResults = await this.localSearch(query, options, intent);
         // 2. Check if JIT should be triggered
         const shouldTrigger = enableJIT && this.shouldTriggerJIT(localResults, query);
         if (!shouldTrigger) {
@@ -28,10 +30,10 @@ export class SearchEngine {
         }
         // 3. Trigger JIT fallback
         console.error(`\nWARNING: Low confidence results (${localResults.length} found, top score: ${localResults[0]?.relevance_score || 0})`);
-        await this.jitOrchestrator.fetchAndIngest(query, 10);
+        await this.jitOrchestrator.fetchAndIngest(query, 10, intent);
         // 4. Re-run local search with updated index
         console.error(`Re-searching with updated library...`);
-        const enhancedResults = await this.localSearch(query, options);
+        const enhancedResults = await this.localSearch(query, options, intent);
         const newCount = enhancedResults.length - localResults.length;
         if (newCount > 0) {
             console.error(`Found ${newCount} additional results\n`);
@@ -41,7 +43,7 @@ export class SearchEngine {
     /**
      * Perform hybrid search (Vector + Lexical + Penalties)
      */
-    async localSearch(query, options) {
+    async localSearch(query, options, intent) {
         const limit = options.limit || 5;
         // 1. Parse Query
         const words = query.toLowerCase().split(/\s+/);
@@ -66,6 +68,12 @@ export class SearchEngine {
             // Filter: Safe only
             if (options.safeOnly && metadata.license.category === "restricted")
                 continue;
+            // Filter: Hard language exclusion — completely drop bilingual/multilingual datasets
+            // when user explicitly requested a single language
+            if (shouldExcludeByLanguage(metadata, intent)) {
+                log(`Language exclusion: Dropped ${match.id} (bilingual/multilingual for single-language query)`);
+                continue;
+            }
             const text = `${metadata.name} ${metadata.description} ${metadata.tags.join(" ")}`.toLowerCase();
             // Filter: Explicit Negative Keywords
             if (negativeKeywords.some(neg => text.includes(neg))) {
@@ -96,6 +104,34 @@ export class SearchEngine {
             if (lexicalScore === 0 && positiveKeywords.length > 1) {
                 penalty += 0.2;
             }
+            // Penalty: Modality Mismatch
+            // Infer the expected modality from the query and penalize mismatches.
+            // e.g., "anime quotes" is text, so image-classification datasets get penalized.
+            const textIndicators = ["quotes", "text", "nlp", "sentiment", "review", "comment", "caption", "dialogue", "chat", "translation", "summarization", "classification"];
+            const imageIndicators = ["image", "photo", "picture", "vision", "detection", "segmentation", "face", "background"];
+            const queryLower = query.toLowerCase();
+            const queryLooksText = textIndicators.some(t => queryLower.includes(t));
+            const queryLooksImage = imageIndicators.some(t => queryLower.includes(t));
+            if (queryLooksText && !queryLooksImage) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isImageResult = resultTask.includes("image") || resultTask.includes("object-detection") ||
+                    text.includes("image classification") || text.includes("image-classification") ||
+                    text.includes("object detection") || text.includes("image segmentation");
+                if (isImageResult) {
+                    log(`Modality penalty: text query but image dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
+            if (queryLooksImage && !queryLooksText) {
+                const resultTask = (metadata.task || "").toLowerCase();
+                const isTextResult = resultTask.includes("text-classification") || resultTask.includes("text-generation") ||
+                    resultTask.includes("translation") || resultTask.includes("summarization") ||
+                    resultTask.includes("question-answering");
+                if (isTextResult) {
+                    log(`Modality penalty: image query but text dataset ${match.id}`);
+                    penalty += 0.35;
+                }
+            }
             // D. Accessibility Bonuses (Prioritize low-friction sources)
             let bonus = 0;
             const sourceBonuses = {
@@ -108,11 +144,13 @@ export class SearchEngine {
             bonus = sourceBonuses[metadata.source] || 0;
             // Final Combined Score
             // 70% Vector, 30% Lexical, minus Penalties, plus Bonuses
-            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus;
+            const intentScore = scoreDatasetAgainstIntent(metadata, intent);
+            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus + intentScore;
             metadata.relevance_score = Math.round(finalScore * 100) / 100;
             metadata.vector_score = Math.round(vectorScore * 100) / 100;
             metadata.lexical_score = Math.round(lexicalScore * 100) / 100;
             metadata.accessibility_bonus = bonus;
+            metadata.intent_score = intentScore;
             results.push(metadata);
         }
         // Sort by final score and limit

package/build/search/jit-orchestrator.js

@@ -2,6 +2,7 @@ import { HuggingFaceScraper } from "../metadata/scraper.js";
 import { UCIScraper } from "../metadata/uci-scraper.js";
 import { GitHubScraper } from "../metadata/github-scraper.js";
 import { WorldBankScraper, NASAScraper } from "../metadata/institutional-scrapers.js";
+import { analyzeDatasetQuery, buildIntentSearchQuery } from "./query-intent.js";
 // Common stop words to filter out for better search
 const STOP_WORDS = new Set([
     "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
@@ -61,7 +62,7 @@ export class JITOrchestrator {
     /**
      * Main JIT workflow: fetch, save, index, return new datasets
      */
-    async fetchAndIngest(query, limit = 10) {
+    async fetchAndIngest(query, limit = 10, providedIntent) {
         // Rate limiting check
         if (!this.canTrigger(query)) {
             console.error(`[JIT] Query "${query}" was searched recently. Waiting...`);
@@ -69,9 +70,12 @@ export class JITOrchestrator {
         }
         console.error(`\n[JIT] Searching live sources for: "${query}"`);
         this.lastTriggerTime.set(query, Date.now());
-        // Simplify query for better API results
-        const keywords = this.simplifyQuery(query);
-        if (keywords.length > 0) {
+        const intent = providedIntent || await analyzeDatasetQuery(query);
+        const keywords = this.simplifyQuery(buildIntentSearchQuery(intent));
+        if (intent.llmBacked || intent.language || intent.task || intent.domain || intent.minRows) {
+            console.error(`[JIT] Intent: ${JSON.stringify({ language: intent.language, task: intent.task, domain: intent.domain, minRows: intent.minRows, searchQuery: intent.searchQuery })}`);
+        }
+        else if (keywords.length > 0) {
             console.error(`[JIT] Keywords extracted: ${keywords.join(", ")}`);
         }
         const newDatasets = [];
@@ -81,15 +85,16 @@ export class JITOrchestrator {
             // Get existing dataset IDs to avoid duplicates
             const existing = this.metadataStore.getAllDatasets();
             existing.forEach(ds => existingIds.add(ds.id));
-            // 1. Scrape HuggingFace - try each keyword separately for better results
-            let hfResults = [];
-            for (const keyword of keywords) {
-                if (hfResults.length >= limit)
-                    break;
-                const results = await this.scrapeHuggingFace(keyword, Math.ceil(limit / keywords.length));
-                for (const ds of results) {
-                    if (!hfResults.some(existing => existing.id === ds.id)) {
-                        hfResults.push(ds);
+            let hfResults = await this.scrapeHuggingFace(intent, limit);
+            if (hfResults.length < Math.max(3, Math.floor(limit / 2))) {
+                for (const keyword of keywords) {
+                    if (hfResults.length >= limit)
+                        break;
+                    const results = await this.scrapeHuggingFace(keyword, Math.ceil(limit / Math.max(keywords.length, 1)));
+                    for (const ds of results) {
+                        if (!hfResults.some(existing => existing.id === ds.id)) {
+                            hfResults.push(ds);
+                        }
                     }
                 }
             }
@@ -170,7 +175,6 @@ export class JITOrchestrator {
     async scrapeHuggingFace(query, limit) {
         const scraper = new HuggingFaceScraper();
         try {
-            // Pass the query as a general search term
             return await scraper.scrape(limit, true, query);
         }
         catch (error) {

package/build/search/query-intent.js

@@ -0,0 +1,509 @@
+import { classifyDomain } from "../metadata/domain.js";
+const STOP_WORDS = new Set([
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "as", "is", "was", "are", "were", "been",
+    "be", "have", "has", "had", "do", "does", "did", "will", "would",
+    "could", "should", "may", "might", "must", "shall", "can", "need",
+    "about", "into", "through", "during", "before", "after", "above",
+    "below", "between", "under", "again", "further", "then", "once",
+    "here", "there", "when", "where", "why", "how", "all", "each",
+    "few", "more", "most", "other", "some", "such", "no", "nor", "not",
+    "only", "own", "same", "so", "than", "too", "very", "just", "also",
+    "dataset", "datasets", "data", "find", "search", "looking", "need", "want",
+    "give", "show", "me", "please"
+]);
+const LANGUAGE_ALIASES = {
+    english: ["english", "en", "eng"],
+    spanish: ["spanish", "es", "spa"],
+    french: ["french", "fr", "fra"],
+    german: ["german", "de", "deu"],
+    portuguese: ["portuguese", "pt", "por"],
+    chinese: ["chinese", "zh", "cmn"],
+    japanese: ["japanese", "ja", "jpn"],
+    korean: ["korean", "ko", "kor"],
+    arabic: ["arabic", "ar", "ara"],
+    russian: ["russian", "ru", "rus"],
+    hindi: ["hindi", "hi", "hin"],
+    multilingual: ["multilingual", "bilingual", "cross-lingual", "crosslingual"],
+};
+const TASK_PATTERNS = [
+    { task: "translation", patterns: [/\btranslation\b/i, /\bmachine translation\b/i, /\bparallel corpus\b/i] },
+    { task: "question-answering", patterns: [/\bquestion answering\b/i, /\bqa\b/i, /\bq&a\b/i] },
+    { task: "summarization", patterns: [/\bsummarization\b/i, /\bsummary\b/i, /\btl;dr\b/i] },
+    { task: "sentiment-analysis", patterns: [/\bsentiment\b/i, /\bsentiment analysis\b/i] },
+    { task: "text-classification", patterns: [/\bclassification\b/i, /\bclassifier\b/i, /\btext classification\b/i] },
+    { task: "token-classification", patterns: [/\bner\b/i, /\bnamed entity\b/i, /\btoken classification\b/i] },
+    { task: "text-generation", patterns: [/\btext generation\b/i, /\bgenerative\b/i, /\binstruction\b/i, /\bchat\b/i] },
+    { task: "image-classification", patterns: [/\bimage classification\b/i] },
+    { task: "object-detection", patterns: [/\bobject detection\b/i, /\bdetection\b/i] },
+];
+const intentCache = new Map();
+export async function analyzeDatasetQuery(query, requirements) {
+    const cacheKey = `${query || ""}::${requirements || ""}`;
+    const cached = intentCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    const task = (async () => {
+        const heuristic = buildHeuristicIntent(query, requirements);
+        const llmIntent = await tryLlmIntent(heuristic, requirements);
+        return llmIntent ? mergeIntent(heuristic, llmIntent) : heuristic;
+    })();
+    intentCache.set(cacheKey, task);
+    return task;
+}
+export function scoreDatasetAgainstIntent(dataset, intent) {
+    if (!intent)
+        return 0;
+    const text = [
+        dataset.name,
+        dataset.description,
+        dataset.task,
+        dataset.domain || "",
+        dataset.tags.join(" "),
+        dataset.languages.join(" "),
+    ].join(" ").toLowerCase();
+    let score = 0;
+    if (intent.language) {
+        const aliases = getLanguageAliases(intent.language);
+        const datasetLanguages = dataset.languages.map(normalizeToken);
+        const languageMatch = aliases.some(alias => datasetLanguages.includes(alias) || text.includes(alias));
+        const isMultilingualIntent = intent.language === "multilingual";
+        if (languageMatch) {
+            // Check if the dataset is monolingual in the requested language vs multilingual
+            const nonRequestedLanguages = datasetLanguages.filter(lang => !aliases.includes(lang) && lang !== "" && lang !== "unknown");
+            if (nonRequestedLanguages.length === 0 || isMultilingualIntent) {
+                // Purely the requested language (or user wants multilingual) → full boost
+                score += 0.55;
+            }
+            else {
+                // Bilingual/multilingual dataset that CONTAINS the language but isn't exclusive
+                // Penalize proportionally to how many other languages are present
+                const ratio = nonRequestedLanguages.length / Math.max(datasetLanguages.length, 1);
+                score += 0.1 - (ratio * 0.4); // ranges from +0.1 (mostly target lang) to -0.3 (mostly other langs)
+            }
+        }
+        else if (dataset.languages.length > 0) {
+            score -= 0.65;
+        }
+        else {
+            score -= 0.1;
+        }
+    }
+    if (intent.task) {
+        if (matchesTask(dataset, intent.task, text)) {
+            score += 0.35;
+        }
+        else {
+            score -= 0.3;
+        }
+    }
+    if (intent.domain && intent.domain !== "general" && intent.domain !== "unknown") {
+        const datasetDomain = String(dataset.domain || "").toLowerCase();
+        if (datasetDomain === intent.domain || text.includes(intent.domain)) {
+            score += 0.25;
+        }
+        else {
+            score -= 0.2;
+        }
+    }
+    if (intent.minRows && intent.minRows > 0) {
+        const totalExamples = Number(dataset.total_examples || 0);
+        if (totalExamples > 0) {
+            const ratio = totalExamples / intent.minRows;
+            if (ratio >= 1) {
+                score += Math.min(0.45, 0.18 + (Math.log10(ratio + 1) * 0.15));
+            }
+            else if (ratio < 0.05) {
+                score -= 1.2;
+            }
+            else if (ratio < 0.25) {
+                score -= 0.8;
+            }
+            else if (ratio < 0.5) {
+                score -= 0.45;
+            }
+            else {
+                score -= 0.15;
+            }
+        }
+        else {
+            score -= 0.08;
+        }
+    }
+    if (intent.positiveTerms.length > 0) {
+        const matches = intent.positiveTerms.filter(term => text.includes(term)).length;
+        score += Math.min(0.25, matches * 0.06);
+    }
+    if (intent.negativeTerms.some(term => text.includes(term))) {
+        score -= 0.7;
+    }
+    return Math.round(score * 100) / 100;
+}
+export function buildIntentSearchQuery(intent) {
+    return intent.searchQuery;
+}
+/**
+ * Build HuggingFace-compatible filter tags from the parsed intent.
+ * Returns e.g. ["language:en", "task_ids:text-classification"].
+ */
+export function buildHuggingFaceFilterTags(intent) {
+    const tags = [];
+    if (intent.language && intent.language !== "multilingual") {
+        const langCode = LANGUAGE_TO_CODE[intent.language];
+        if (langCode)
+            tags.push(`language:${langCode}`);
+    }
+    if (intent.task) {
+        tags.push(`task_ids:${intent.task}`);
+    }
+    return tags;
+}
+const LANGUAGE_TO_CODE = {
+    english: "en",
+    spanish: "es",
+    french: "fr",
+    german: "de",
+    portuguese: "pt",
+    chinese: "zh",
+    japanese: "ja",
+    korean: "ko",
+    arabic: "ar",
+    russian: "ru",
+    hindi: "hi",
+};
+const BILINGUAL_INDICATORS = [
+    "translation", "parallel", "bilingual", "multilingual",
+    "cross-lingual", "crosslingual", "machine-translation",
+    "aligned", "comparable corpus",
+];
+/**
+ * Hard-exclude a dataset when the user requests a single specific language
+ * and the dataset is bilingual, multilingual, or tagged with other languages.
+ * Returns true if the dataset should be EXCLUDED from results.
+ */
+export function shouldExcludeByLanguage(dataset, intent) {
+    if (!intent?.language || intent.language === "multilingual")
+        return false;
+    const aliases = getLanguageAliases(intent.language);
+    const datasetLanguages = dataset.languages.map(normalizeToken).filter(l => l && l !== "unknown");
+    // If the dataset has language tags and ANY of them are NOT the requested language, exclude
+    if (datasetLanguages.length > 0) {
+        const hasRequestedLang = aliases.some(a => datasetLanguages.includes(a));
+        const hasOtherLangs = datasetLanguages.some(lang => !aliases.includes(lang));
+        if (hasOtherLangs)
+            return true; // bilingual/multilingual → exclude
+        if (!hasRequestedLang)
+            return true; // wrong language entirely
+    }
+    // Check name, description, and tags for bilingual indicators or other language names
+    const text = [
+        dataset.name,
+        dataset.description,
+        dataset.tags.join(" "),
+    ].join(" ").toLowerCase();
+    // Check for bilingual/translation keywords
+    if (BILINGUAL_INDICATORS.some(indicator => text.includes(indicator))) {
+        return true;
+    }
+    // Check if the text mentions other specific languages by name
+    const otherLanguageNames = Object.keys(LANGUAGE_ALIASES).filter(lang => lang !== intent.language && lang !== "multilingual");
+    for (const otherLang of otherLanguageNames) {
+        const otherAliases = LANGUAGE_ALIASES[otherLang];
+        // Only check the full language name (not 2-letter codes which could appear in regular text)
+        if (otherAliases && otherAliases[0] && text.includes(otherAliases[0])) {
+            return true;
+        }
+    }
+    return false;
+}
+function buildHeuristicIntent(query, requirements) {
+    const originalQuery = `${query || ""} ${requirements || ""}`.trim();
+    const normalizedQuery = originalQuery.toLowerCase();
+    const negativeTerms = [...normalizedQuery.matchAll(/(?:^|\s)-([\w-]{2,})/g)].map(match => normalizeToken(match[1]));
+    const positiveTerms = tokenize(normalizedQuery)
+        .filter(token => !negativeTerms.includes(token))
+        .slice(0, 8);
+    const task = detectTask(normalizedQuery);
+    const language = detectLanguage(normalizedQuery);
+    const domain = classifyDomain(normalizedQuery, [], normalizedQuery, task);
+    const minRows = extractRequestedRows(normalizedQuery);
+    const searchTerms = [
+        language,
+        task,
+        domain !== "general" && domain !== "unknown" ? domain : undefined,
+        ...positiveTerms,
+    ].filter((value, index, self) => !!value && self.indexOf(value) === index);
+    return {
+        originalQuery,
+        normalizedQuery,
+        searchQuery: searchTerms.slice(0, 6).join(" ") || normalizedQuery,
+        positiveTerms,
+        negativeTerms,
+        language,
+        task: task || undefined,
+        domain,
+        minRows,
+        llmBacked: false,
+    };
+}
+function mergeIntent(base, llmIntent) {
+    const language = llmIntent.language ? normalizeToken(llmIntent.language) : base.language;
+    const task = llmIntent.task ? normalizeToken(llmIntent.task) : base.task;
+    const domain = llmIntent.domain ? normalizeToken(llmIntent.domain) : base.domain;
+    const minRows = typeof llmIntent.minRows === "number" && Number.isFinite(llmIntent.minRows)
+        ? llmIntent.minRows
+        : base.minRows;
+    const positiveTerms = Array.from(new Set([...(llmIntent.positiveTerms || []), ...base.positiveTerms].map(normalizeToken))).filter(Boolean);
+    const negativeTerms = Array.from(new Set([...(llmIntent.negativeTerms || []), ...base.negativeTerms].map(normalizeToken))).filter(Boolean);
+    const merged = {
+        ...base,
+        language,
+        task,
+        domain,
+        minRows,
+        positiveTerms,
+        negativeTerms,
+        llmBacked: true,
+    };
+    merged.searchQuery = [
+        merged.language,
+        merged.task,
+        merged.domain !== "general" && merged.domain !== "unknown" ? merged.domain : undefined,
+        ...merged.positiveTerms,
+    ].filter((value, index, self) => !!value && self.indexOf(value) === index).slice(0, 6).join(" ") || merged.normalizedQuery;
+    return merged;
+}
+async function tryLlmIntent(base, requirements) {
+    const openAiKey = process.env.OPENAI_API_KEY;
+    if (openAiKey) {
+        return await callOpenAiIntent(base, requirements).catch(() => undefined);
+    }
+    const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
+    if (geminiKey) {
+        return await callGeminiIntent(base, requirements, geminiKey).catch(() => undefined);
+    }
+    return undefined;
+}
+async function callOpenAiIntent(base, requirements) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 5000);
+    try {
+        const response = await fetch("https://api.openai.com/v1/chat/completions", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+            },
+            body: JSON.stringify({
+                model: process.env.OPENAI_MODEL || "gpt-4o-mini",
+                temperature: 0,
+                response_format: { type: "json_object" },
+                messages: [
+                    {
+                        role: "system",
+                        content: "Extract dataset search intent as JSON with keys: language, task, domain, minRows, positiveTerms, negativeTerms. Use null for unknowns.",
+                    },
+                    {
+                        role: "user",
+                        content: JSON.stringify({ query: base.originalQuery, requirements: requirements || null, heuristic: base }),
+                    },
+                ],
+            }),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            return undefined;
+        }
+        const body = await response.json();
+        const content = body?.choices?.[0]?.message?.content;
+        return parseIntentPayload(content);
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+async function callGeminiIntent(base, requirements, apiKey) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 5000);
+    try {
+        const model = process.env.GEMINI_MODEL || "gemini-1.5-flash";
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${encodeURIComponent(apiKey)}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                generationConfig: {
+                    temperature: 0,
+                    responseMimeType: "application/json",
+                },
+                contents: [{
+                        role: "user",
+                        parts: [{
+                                text: `Extract dataset search intent as JSON with keys language, task, domain, minRows, positiveTerms, negativeTerms. Query payload: ${JSON.stringify({ query: base.originalQuery, requirements: requirements || null, heuristic: base })}`,
+                            }],
+                    }],
+            }),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            return undefined;
+        }
+        const body = await response.json();
+        const content = body?.candidates?.[0]?.content?.parts?.[0]?.text;
+        return parseIntentPayload(content);
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+function parseIntentPayload(content) {
+    if (typeof content !== "string" || !content.trim()) {
+        return undefined;
+    }
+    const jsonText = extractJsonObject(content);
+    if (!jsonText) {
+        return undefined;
+    }
+    try {
+        const parsed = JSON.parse(jsonText);
+        return {
+            language: typeof parsed.language === "string" ? parsed.language : undefined,
+            task: typeof parsed.task === "string" ? parsed.task : undefined,
+            domain: typeof parsed.domain === "string" ? parsed.domain : undefined,
+            minRows: typeof parsed.minRows === "number"
+                ? parsed.minRows
+                : typeof parsed.min_rows === "number"
+                    ? parsed.min_rows
+                    : undefined,
+            positiveTerms: Array.isArray(parsed.positiveTerms)
+                ? parsed.positiveTerms.filter((item) => typeof item === "string")
+                : Array.isArray(parsed.positive_terms)
+                    ? parsed.positive_terms.filter((item) => typeof item === "string")
+                    : undefined,
+            negativeTerms: Array.isArray(parsed.negativeTerms)
+                ? parsed.negativeTerms.filter((item) => typeof item === "string")
+                : Array.isArray(parsed.negative_terms)
+                    ? parsed.negative_terms.filter((item) => typeof item === "string")
+                    : undefined,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function extractJsonObject(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
+        return trimmed;
+    }
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+        return trimmed.slice(start, end + 1);
+    }
+    return undefined;
+}
+function detectLanguage(text) {
+    for (const [language, aliases] of Object.entries(LANGUAGE_ALIASES)) {
+        if (aliases.some(alias => new RegExp(`(^|[^a-z])${escapeRegex(alias)}([^a-z]|$)`, "i").test(text))) {
+            return language;
+        }
+    }
+    return undefined;
+}
+function detectTask(text) {
+    const match = TASK_PATTERNS.find(entry => entry.patterns.some(pattern => pattern.test(text)));
+    return match?.task;
+}
+function tokenize(text) {
+    return Array.from(new Set(text
+        .replace(/[^\w\s-]/g, " ")
+        .split(/\s+/)
+        .map(normalizeToken)
+        .filter(token => token.length > 2 && !STOP_WORDS.has(token) && !/^\d+$/.test(token))));
+}
+function normalizeToken(value) {
+    return value.toLowerCase().replace(/^[^a-z0-9]+|[^a-z0-9-]+$/g, "").trim();
+}
+function extractRequestedRows(text) {
+    // Match "1 million", "2.5 billion", "500 thousand" etc.
+    const wordMultipliers = {
+        thousand: 1_000, million: 1_000_000, billion: 1_000_000_000,
+        mil: 1_000_000, bil: 1_000_000_000,
+    };
+    const wordPattern = new RegExp(`(\\d+(?:\\.\\d+)?)\\s*(${Object.keys(wordMultipliers).join("|")})\\b`, "i");
+    const wordMatch = text.match(wordPattern);
+    if (wordMatch) {
+        const base = Number(wordMatch[1]);
+        const multiplier = wordMultipliers[wordMatch[2].toLowerCase()];
+        const value = Math.round(base * multiplier);
+        if (Number.isFinite(value) && value > 0)
+            return value;
+    }
+    const explicit = text.match(/(\d[\d,\s]{1,12})\s*(samples?|rows?|records?)/i);
+    if (explicit) {
+        const value = Number(explicit[1].replace(/[\s,]/g, ""));
+        if (Number.isFinite(value) && value > 0) {
+            return value;
+        }
+    }
+    const humanSized = text.match(/(\d+(?:\.\d+)?)\s*([kmb])\s*(samples?|rows?|records?)/i);
+    if (humanSized) {
+        const base = Number(humanSized[1]);
+        const suffix = humanSized[2].toLowerCase();
+        const multiplier = suffix === "k" ? 1_000 : suffix === "m" ? 1_000_000 : 1_000_000_000;
+        const value = Math.round(base * multiplier);
+        if (Number.isFinite(value) && value > 0) {
+            return value;
+        }
+    }
+    const commaNumbers = [...text.matchAll(/\b\d{1,3}(?:,\d{3})+\b/g)]
+        .map(match => Number(match[0].replace(/,/g, "")))
+        .filter(value => Number.isFinite(value) && value > 0);
+    if (commaNumbers.length > 0) {
+        return Math.max(...commaNumbers);
+    }
+    const humanSizedAnywhere = [...text.matchAll(/\b(\d+(?:\.\d+)?)\s*([kmb])\b/gi)]
+        .map(match => {
+        const base = Number(match[1]);
+        const suffix = match[2].toLowerCase();
+        const multiplier = suffix === "k" ? 1_000 : suffix === "m" ? 1_000_000 : 1_000_000_000;
+        return Math.round(base * multiplier);
+    })
+        .filter(value => Number.isFinite(value) && value > 0);
+    if (humanSizedAnywhere.length > 0) {
+        return Math.max(...humanSizedAnywhere);
+    }
+    const allNumbers = [...text.matchAll(/\b\d{4,9}\b/g)]
+        .map(match => Number(match[0]))
+        .filter(value => Number.isFinite(value) && value > 0);
+    if (allNumbers.length > 0) {
+        return Math.max(...allNumbers);
+    }
+    return undefined;
+}
+function matchesTask(dataset, task, text) {
+    const normalizedTask = normalizeToken(task);
+    const aliases = {
+        "question-answering": ["question-answering", "qa", "question answering"],
+        "text-classification": ["text-classification", "classification", "text classification"],
+        "token-classification": ["token-classification", "ner", "named entity"],
+        "sentiment-analysis": ["sentiment-analysis", "sentiment"],
+        translation: ["translation", "machine-translation", "parallel corpus"],
+        summarization: ["summarization", "summary"],
+        "text-generation": ["text-generation", "generation", "chat", "instruction"],
+        "image-classification": ["image-classification", "image classification"],
+        "object-detection": ["object-detection", "object detection"],
+    };
+    const variants = aliases[normalizedTask] || [normalizedTask];
+    return variants.some(variant => normalizeToken(dataset.task).includes(variant) || text.includes(variant));
+}
+function getLanguageAliases(language) {
+    const normalized = normalizeToken(language);
+    return (LANGUAGE_ALIASES[normalized] || [normalized]).map(normalizeToken);
+}
+function escapeRegex(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}