@vespermcp/mcp-server 1.0.5 → 1.0.8

package/build/quality/analyzer.js

@@ -1,12 +1,44 @@
 import { spawn } from "child_process";
 import path from "path";
+import fs from "fs";
 export class QualityAnalyzer {
     cache;
     pythonPath = "python"; // Assumes python is in PATH
     scriptPath;
-    constructor(cache, projectRoot = process.cwd()) {
+    constructor(cache, buildDir = process.cwd()) {
+        // buildDir is the directory containing the compiled JS (e.g., build/)
+        // Priority: 
+        // 1. ~/.vesper/python (stable synced location)
+        // 2. build/python (production)
+        // 3. src/python (development)
         this.cache = cache;
-        this.scriptPath = path.join(projectRoot, "src", "python", "quality_engine.py");
+        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptPath0 = path.resolve(dataRoot, "python", "quality_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "quality_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "quality_engine.py");
+        const scriptPath3 = path.resolve(buildDir, "..", "python", "quality_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else if (fs.existsSync(scriptPath3)) {
+            this.scriptPath = scriptPath3;
+        }
+        else {
+            // Fallback to stable data path, error will be caught during execution
+            this.scriptPath = scriptPath0;
+            console.error(`[QualityAnalyzer] WARNING: Python script not found!`);
+        }
+        // Detect Python command (Windows may use 'py' instead of 'python')
+        if (process.platform === "win32") {
+            this.pythonPath = "py";
+        }
     }
     /**
      * Run quality analysis on a local file (CSV/Parquet/JSON)
@@ -16,7 +48,7 @@ export class QualityAnalyzer {
         if (this.cache && datasetId) {
             const cached = await this.cache.getReport(datasetId);
             if (cached) {
-                console.log(`[QualityAnalyzer] Cache hit for ${datasetId}`);
+                console.error(`[QualityAnalyzer] Cache hit for ${datasetId}`);
                 return cached;
             }
         }
@@ -32,7 +64,11 @@ export class QualityAnalyzer {
             });
             process.on("close", (code) => {
                 if (code !== 0) {
-                    reject(new Error(`Quality Analyzer failed (code ${code}): ${stderr}`));
+                    const errorDetails = `Quality Analyzer failed (code ${code})
+Command: ${this.pythonPath} ${this.scriptPath} ${filePath}
+Script path exists: ${fs.existsSync(this.scriptPath)}
+Error output: ${stderr}`;
+                    reject(new Error(errorDetails));
                     return;
                 }
                 try {

package/build/quality/image-analyzer.js

@@ -1,10 +1,36 @@
 import { spawn } from "child_process";
 import path from "path";
+import fs from "fs";
 export class ImageAnalyzer {
     pythonPath = "python";
     scriptPath;
-    constructor(projectRoot = process.cwd()) {
-        this.scriptPath = path.join(projectRoot, "src", "python", "image_engine.py");
+    constructor(buildDir = process.cwd()) {
+        // buildDir is the directory containing the compiled JS (e.g., build/)
+        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptPath0 = path.resolve(dataRoot, "python", "image_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "image_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "image_engine.py");
+        const scriptPath3 = path.resolve(buildDir, "..", "python", "image_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else if (fs.existsSync(scriptPath3)) {
+            this.scriptPath = scriptPath3;
+        }
+        else {
+            this.scriptPath = scriptPath0;
+        }
+        // Detect Python command (Windows may use 'py' instead of 'python')
+        if (process.platform === "win32") {
+            this.pythonPath = "py";
+        }
     }
     /**
      * Analyze image quality for a single file or a directory

package/build/quality/media-analyzer.js

@@ -1,10 +1,36 @@
 import { spawn } from "child_process";
 import path from "path";
+import fs from "fs";
 export class MediaAnalyzer {
     pythonPath = "python";
     scriptPath;
-    constructor(projectRoot = process.cwd()) {
-        this.scriptPath = path.join(projectRoot, "src", "python", "media_engine.py");
+    constructor(buildDir = process.cwd()) {
+        // buildDir is the directory containing the compiled JS (e.g., build/)
+        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
+        const dataRoot = path.join(homeDir, ".vesper");
+        const scriptPath0 = path.resolve(dataRoot, "python", "media_engine.py");
+        const scriptPath1 = path.resolve(buildDir, "python", "media_engine.py");
+        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "media_engine.py");
+        const scriptPath3 = path.resolve(buildDir, "..", "python", "media_engine.py");
+        if (fs.existsSync(scriptPath0)) {
+            this.scriptPath = scriptPath0;
+        }
+        else if (fs.existsSync(scriptPath1)) {
+            this.scriptPath = scriptPath1;
+        }
+        else if (fs.existsSync(scriptPath2)) {
+            this.scriptPath = scriptPath2;
+        }
+        else if (fs.existsSync(scriptPath3)) {
+            this.scriptPath = scriptPath3;
+        }
+        else {
+            this.scriptPath = scriptPath0;
+        }
+        // Detect Python command (Windows may use 'py' instead of 'python')
+        if (process.platform === "win32") {
+            this.pythonPath = "py";
+        }
     }
     /**
      * Analyze audio/video quality for a single file or a directory

package/build/scripts/cleanup-kaggle.js

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+/**
+ * Cleanup script to remove all Kaggle datasets from Vesper
+ */
+import { fileURLToPath } from "url";
+import path from "path";
+import { MetadataStore } from "../metadata/store.js";
+import { VectorStore } from "../search/vector-store.js";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const homeDir = process.env.HOME || process.env.USERPROFILE || path.join(__dirname, "..");
+const dataRoot = path.join(homeDir, ".vesper");
+const dbPath = path.join(dataRoot, "data", "metadata.db");
+const vectorPath = path.join(dataRoot, "data", "vectors.json");
+console.log("🧹 Vesper Kaggle Cleanup");
+console.log("========================\n");
+try {
+    const metadataStore = new MetadataStore(dbPath);
+    const vectorStore = new VectorStore(vectorPath);
+    // Get all Kaggle dataset IDs
+    const kaggleIds = metadataStore.getDatasetIdsBySource("kaggle");
+    console.log(`Found ${kaggleIds.length} Kaggle datasets in database`);
+    if (kaggleIds.length === 0) {
+        console.log("✅ No Kaggle datasets to remove");
+        process.exit(0);
+    }
+    // Delete from vector store
+    const vectorsDeleted = vectorStore.deleteMany(kaggleIds);
+    console.log(`🗑️  Deleted ${vectorsDeleted} vectors from vector store`);
+    vectorStore.save();
+    // Delete from metadata database
+    const datasetsDeleted = metadataStore.deleteBySource("kaggle");
+    console.log(`🗑️  Deleted ${datasetsDeleted} datasets from metadata database`);
+    metadataStore.close();
+    console.log("\n✅ Cleanup complete! Kaggle datasets have been removed.");
+    console.log("   You can now search without seeing Kaggle results.");
+}
+catch (error) {
+    console.error("❌ Cleanup failed:", error.message);
+    process.exit(1);
+}

package/build/scripts/repro-bug.js

@@ -0,0 +1,37 @@
+import { JobManager } from "../jobs/manager.js";
+import { MetadataStore } from "../metadata/store.js";
+import fs from "fs";
+async function repro() {
+    const dbPath = "repro_test.db";
+    if (fs.existsSync(dbPath))
+        fs.unlinkSync(dbPath);
+    const store = new MetadataStore(dbPath);
+    const jobManager = JobManager.getInstance(store);
+    console.log("Setting up listener...");
+    jobManager.on("processJob", async (job, execute) => {
+        console.log(`Listener received job ${job.id}`);
+        const task = async () => {
+            console.log("Running task...");
+            return "success";
+        };
+        try {
+            await execute(task);
+            console.log("Execute finished");
+        }
+        catch (e) {
+            console.error("Execute failed in listener:", e.message);
+        }
+    });
+    console.log("Creating job...");
+    const job = jobManager.createJob("prepare", 0, { query: "test" });
+    console.log(`Job created: ${job.id}`);
+    // Wait for a bit
+    await new Promise(r => setTimeout(r, 2000));
+    const finalJob = store.getJob(job.id);
+    console.log("Final job status:", finalJob?.status);
+    console.log("Final job status text:", finalJob?.status_text);
+    store.close();
+    if (fs.existsSync(dbPath))
+        fs.unlinkSync(dbPath);
+}
+repro().catch(console.error);

package/build/scripts/repro-export-bug.js

@@ -0,0 +1,56 @@
+import path from "path";
+import fs from "fs";
+import { spawnSync } from "child_process";
+const pythonPath = "python";
+const scriptPath = path.join(process.cwd(), "src", "python", "cleaner.py");
+const testDir = path.join(process.cwd(), "test_repro");
+if (!fs.existsSync(testDir))
+    fs.mkdirSync(testDir);
+async function runRepro() {
+    console.log("=== Reproducing CSV Export Bug ===\n");
+    const parquetFile = path.join(testDir, "test_nested.parquet");
+    const csvOutput = path.join(testDir, "test_nested_cleaned.csv");
+    // 1. Create a Parquet file with nested data (Lists/Structs) using Python
+    console.log("Creating nested Parquet file...");
+    const createScript = `
+import polars as pl
+df = pl.DataFrame({
+    "id": [1, 2, 3],
+    "tags": [["a", "b"], ["c"], []],
+    "meta": [{"score": 0.9, "safe": True}, {"score": 0.4, "safe": False}, {"score": 0.1, "safe": True}]
+})
+df.write_parquet(r"${parquetFile}")
+`;
+    fs.writeFileSync(path.join(testDir, "create_data.py"), createScript);
+    spawnSync(pythonPath, [path.join(testDir, "create_data.py")], { stdio: 'inherit' });
+    // 2. Call cleaner.py to convert to CSV
+    console.log("Calling cleaner.py to convert to CSV...");
+    const result = spawnSync(pythonPath, [
+        scriptPath,
+        parquetFile,
+        "[]",
+        "csv"
+    ]);
+    console.log("Exit Code:", result.status);
+    console.log("Stdout:", result.stdout?.toString());
+    console.log("Stderr:", result.stderr?.toString());
+    if (result.status === 0) {
+        try {
+            const data = JSON.parse(result.stdout.toString());
+            if (data.success) {
+                console.log("SUCCESS! Output file:", data.output_path);
+                if (fs.existsSync(data.output_path)) {
+                    console.log("File exists on disk.");
+                }
+            }
+            else {
+                console.error("cleaner.py reported failure:", data.error);
+            }
+        }
+        catch (e) {
+            console.error("Failed to parse JSON output:", e);
+        }
+    }
+}
+runRepro().catch(console.error);
+runRepro().catch(console.error);

package/build/scripts/test-mcp-v5.js

@@ -47,16 +47,18 @@ async function testPhase5Tools() {
     // Create job (Logic from index.ts)
     const job = jobManager.createJob("prepare", 0, { query });
     console.log(` - Job Created: ${job.id}`);
-    // Start background task
-    console.log(" - Starting autonomous preparation...");
-    const jobPromise = jobManager.runJob(job.id, async (update) => {
-        update({ progress: 20, status_text: "Searching..." });
-        await new Promise(r => setTimeout(r, 1000));
-        update({ progress: 50, status_text: "Analyzing quality..." });
-        await new Promise(r => setTimeout(r, 1000));
-        update({ progress: 80, status_text: "Formatting export..." });
-        await new Promise(r => setTimeout(r, 1000));
-        return "data/exports/prepared_dataset.parquet";
+    // Register listener for simulated work (Logic from index.ts)
+    jobManager.on("processJob", async (currJob, execute) => {
+        if (currJob.id !== job.id)
+            return;
+        await execute(async () => {
+            console.log(" - [Worker] Starting autonomous preparation task...");
+            await new Promise(r => setTimeout(r, 1000));
+            console.log(" - [Worker] Phase 1: Search complete");
+            await new Promise(r => setTimeout(r, 1000));
+            console.log(" - [Worker] Phase 2: Quality analysis complete");
+            return "data/exports/prepared_dataset.parquet";
+        });
     });
     // 4. Test check_job_status (Polling)
     console.log("\nStep 4: Polling Job Status (Simulating UI Check)...");
@@ -67,7 +69,6 @@ async function testPhase5Tools() {
             break;
         await new Promise(r => setTimeout(r, 800));
     }
-    await jobPromise;
     console.log("\n Phase 5 tools logic verified.");
 }
 testPhase5Tools().catch(console.error);

package/build/scripts/test-production-sync.js

@@ -0,0 +1,36 @@
+import { JobManager } from "../jobs/manager.js";
+import { MetadataStore } from "../metadata/store.js";
+import fs from "fs";
+async function testSync() {
+    console.log("Starting Production Sync Test...");
+    const dbPath = "prod_sync.db";
+    if (fs.existsSync(dbPath))
+        fs.unlinkSync(dbPath);
+    const store = new MetadataStore(dbPath);
+    const jobManager = JobManager.getInstance(store);
+    console.log("Attaching listener (same as index.ts)...");
+    jobManager.on("processJob", async (job, execute) => {
+        console.log(`[Listener 1] Received job ${job.id}`);
+        if (typeof execute !== 'function') {
+            console.error(`[Listener 1] ERROR: execute is not a function! It is: ${typeof execute}`);
+            return;
+        }
+        const task = async () => {
+            console.log("[Listener 1] Task running...");
+            return "ok";
+        };
+        await execute(task);
+        console.log("[Listener 1] Task finished.");
+    });
+    console.log("Emitting job...");
+    const job = jobManager.createJob("prepare", 0, { query: "test" });
+    // Wait for the background loop
+    await new Promise(r => setTimeout(r, 1000));
+    const finalJob = store.getJob(job.id);
+    console.log(`Job Result: ${finalJob?.status} - ${finalJob?.status_text}`);
+    store.close();
+    if (fs.existsSync(dbPath))
+        fs.unlinkSync(dbPath);
+    console.log("Test Complete.");
+}
+testSync().catch(console.error);

package/build/scripts/test-target-detector.js

@@ -0,0 +1,29 @@
+import { TargetDetector } from "../preparation/target-detector.js";
+import path from "path";
+async function testDetector() {
+    // 1. Test existing build dir
+    const detector = new TargetDetector(path.join(process.cwd(), "build"));
+    // 2. Create a dummy CSV for testing
+    const testFile = path.join(process.cwd(), "test_target.csv");
+    const fs = (await import("fs")).default;
+    // Test Case 1: SalePrice (Regression)
+    console.log("--- Test Case 1: SalePrice ---");
+    fs.writeFileSync(testFile, "id,feature1,feature2,SalePrice\n1,10,20,100000\n2,11,21,120000\n3,12,22,110000");
+    let result = await detector.detectTarget(testFile);
+    console.log("Detection:", result.target_column, result.confidence);
+    if (result.target_column) {
+        let val = await detector.validateTarget(testFile, result.target_column);
+        console.log("Validation:", val.problem_type, val.valid);
+    }
+    // Test Case 2: diagnosis (Classification)
+    console.log("\n--- Test Case 2: diagnosis ---");
+    fs.writeFileSync(testFile, "id,age,diagnosis\n1,50,M\n2,60,B\n3,45,M");
+    result = await detector.detectTarget(testFile);
+    console.log("Detection:", result.target_column, result.confidence);
+    if (result.target_column) {
+        let val = await detector.validateTarget(testFile, result.target_column);
+        console.log("Validation:", val.problem_type, val.valid);
+    }
+    fs.unlinkSync(testFile);
+}
+testDetector().catch(console.error);

package/build/scripts/test-write.js

@@ -0,0 +1,14 @@
+import fs from 'fs';
+const target = "C:\\Users\\нурбулан\\AppData\\Roaming\\Code\\User\\mcp.json";
+const content = JSON.stringify({ mcpServers: { test: { command: "node" } } }, null, 2);
+try {
+    console.log(`Testing write to: ${target}`);
+    fs.writeFileSync(target, content, "utf8");
+    const stat = fs.statSync(target);
+    console.log(`Success! File size: ${stat.size} bytes`);
+    const readBack = fs.readFileSync(target, "utf8");
+    console.log("Read back content:", readBack);
+}
+catch (e) {
+    console.error("Failed to write:", e);
+}

package/build/scripts/verify-integration.js

@@ -0,0 +1,57 @@
+import { TargetDetector } from "../preparation/target-detector.js";
+import path from "path";
+import fs from "fs";
+async function verifyIntegration() {
+    // 1. Create a dummy CSV with a clear target
+    const testFile = path.join(process.cwd(), "data", "raw", "integration_test.csv");
+    const testId = "integration_test";
+    if (!fs.existsSync(path.dirname(testFile))) {
+        fs.mkdirSync(path.dirname(testFile), { recursive: true });
+    }
+    console.log("Creating test file:", testFile);
+    fs.writeFileSync(testFile, "id,feature1,feature2,SalePrice\n1,10,20,100000\n2,11,21,100000\n3,12,22,110000");
+    // 2. Call the preview_cleaning tool (simulated by calling valid request handler logic or via MCP client if possible)
+    // Since we can't easily call the MCP server from here without a client, we will simulate 
+    // the logic we added to index.ts to ensure it runs without error.
+    try {
+        const { QualityAnalyzer } = await import("../quality/analyzer.js");
+        const { CleaningPlanner } = await import("../cleaning/planner.js");
+        const { CacheService, MockRedisProvider } = await import("../cache/service.js");
+        console.log("Initializing services...");
+        const cacheService = new CacheService(new MockRedisProvider());
+        // Use build/ directory to simulate runtime environment
+        const buildDir = path.join(process.cwd(), "build");
+        const qualityAnalyzer = new QualityAnalyzer(cacheService, buildDir);
+        const cleaningPlanner = new CleaningPlanner(cacheService, buildDir);
+        console.log("Running analysis...");
+        const report = await qualityAnalyzer.analyze(testFile);
+        console.log("Running target detection...");
+        const detector = new TargetDetector(buildDir);
+        const targetResult = await detector.detectTarget(testFile);
+        console.log("Detected:", targetResult);
+        const targetInfo = targetResult.target_column ? {
+            target: targetResult.target_column,
+            confidence: targetResult.confidence
+        } : undefined;
+        console.log("Generating plan...");
+        const plan = await cleaningPlanner.generatePlan(testId, report, undefined, targetInfo);
+        console.log("Plan Operations:", JSON.stringify(plan.operations, null, 2));
+        const hasRename = plan.operations.some(op => op.type === "RenameTarget");
+        if (hasRename) {
+            console.log("✅ SUCCESS: RenameTarget operation found in plan!");
+        }
+        else {
+            console.error("❌ FAILURE: RenameTarget operation NOT found.");
+            process.exit(1);
+        }
+    }
+    catch (e) {
+        console.error("Error during verification:", e);
+        process.exit(1);
+    }
+    finally {
+        if (fs.existsSync(testFile))
+            fs.unlinkSync(testFile);
+    }
+}
+verifyIntegration();

package/build/scripts/verify-priority.js

@@ -0,0 +1,33 @@
+import path from "path";
+import { MetadataStore } from "../metadata/store.js";
+import { VectorStore } from "../search/vector-store.js";
+import { Embedder } from "../search/embedder.js";
+import { SearchEngine } from "../search/engine.js";
+import { formatSearchResults } from "../tools/formatter.js";
+import fs from "fs";
+const query = process.argv[2] || "anime";
+// Use the actual .vesper data path if it exists, otherwise use local data/
+const homeDir = process.env.HOME || process.env.USERPROFILE || process.cwd();
+const vesperDataRoot = path.join(homeDir, ".vesper");
+let dbPath = path.join(vesperDataRoot, "data", "metadata.db");
+let vectorPath = path.join(vesperDataRoot, "data", "vectors.json");
+if (!fs.existsSync(dbPath)) {
+    console.error("Using local project data directory as fallback...");
+    dbPath = path.join(process.cwd(), "data", "metadata.db");
+    vectorPath = path.join(process.cwd(), "data", "vectors.json");
+}
+const metadataStore = new MetadataStore(dbPath);
+const vectorStore = new VectorStore(vectorPath);
+const embedder = Embedder.getInstance();
+const searchEngine = new SearchEngine(metadataStore, vectorStore, embedder);
+async function run() {
+    console.log(`\n=== VERIFYING SOURCE PRIORITIZATION [Query: "${query}"] ===\n`);
+    const results = await searchEngine.search(query, { limit: 5 });
+    if (results.length === 0) {
+        console.log("No results found. Run a search that triggers JIT first!");
+        return;
+    }
+    // Print formatted results to show badges
+    console.log(formatSearchResults(results));
+}
+run().catch(console.error);

package/build/search/engine.js

@@ -96,12 +96,23 @@ export class SearchEngine {
             if (lexicalScore === 0 && positiveKeywords.length > 1) {
                 penalty += 0.2;
             }
+            // D. Accessibility Bonuses (Prioritize low-friction sources)
+            let bonus = 0;
+            const sourceBonuses = {
+                "huggingface": 0.1,
+                "uci": 0.1,
+                "github": 0.1,
+                "worldbank": 0.1,
+                "nasa": 0.1
+            };
+            bonus = sourceBonuses[metadata.source] || 0;
             // Final Combined Score
-            // 70% Vector, 30% Lexical, minus Penalties
-            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty;
+            // 70% Vector, 30% Lexical, minus Penalties, plus Bonuses
+            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus;
             metadata.relevance_score = Math.round(finalScore * 100) / 100;
             metadata.vector_score = Math.round(vectorScore * 100) / 100;
             metadata.lexical_score = Math.round(lexicalScore * 100) / 100;
+            metadata.accessibility_bonus = bonus;
             results.push(metadata);
         }
         // Sort by final score and limit

package/build/search/jit-orchestrator.js

@@ -1,5 +1,4 @@
 import { HuggingFaceScraper } from "../metadata/scraper.js";
-import { KaggleMetadataScraper } from "../metadata/kaggle-scraper.js";
 import { UCIScraper } from "../metadata/uci-scraper.js";
 import { GitHubScraper } from "../metadata/github-scraper.js";
 import { WorldBankScraper, NASAScraper } from "../metadata/institutional-scrapers.js";
@@ -45,7 +44,7 @@ export class JITOrchestrator {
             // Get existing dataset IDs to avoid duplicates
             const existing = this.metadataStore.getAllDatasets();
             existing.forEach(ds => existingIds.add(ds.id));
-            // 1. Scrape HuggingFace
+            // 1. Scrape HuggingFace (Open Access)
             const hfResults = await this.scrapeHuggingFace(query, limit);
             console.error(`  HuggingFace: Found ${hfResults.length} datasets`);
             for (const ds of hfResults) {
@@ -54,21 +53,7 @@ export class JITOrchestrator {
                     existingIds.add(ds.id);
                 }
             }
-            // 2. Scrape Kaggle (if credentials available)
-            const kaggleUser = process.env.KAGGLE_USERNAME;
-            const kaggleKey = process.env.KAGGLE_KEY;
-            if (kaggleUser && kaggleKey) {
-                const kaggleResults = await this.scrapeKaggle(query, Math.floor(limit / 2));
-                console.error(`  Kaggle: Found ${kaggleResults.length} datasets`);
-                for (const ds of kaggleResults) {
-                    ds.id = `kaggle:${ds.id}`;
-                    if (!existingIds.has(ds.id)) {
-                        newDatasets.push(ds);
-                        existingIds.add(ds.id);
-                    }
-                }
-            }
-            // 3. Scrape UCI
+            // 2. Scrape UCI (Open Access)
             const uciResults = await this.scrapeUCI(query, Math.floor(limit / 2));
             console.error(`  UCI: Found ${uciResults.length} datasets`);
             for (const ds of uciResults) {
@@ -77,7 +62,7 @@ export class JITOrchestrator {
                     existingIds.add(ds.id);
                 }
             }
-            // 4. Scrape GitHub
+            // 3. Scrape GitHub (Open Access)
             const githubResults = await this.scrapeGitHub(query, Math.floor(limit / 2));
             console.error(`  GitHub: Found ${githubResults.length} datasets`);
             for (const ds of githubResults) {
@@ -86,7 +71,7 @@ export class JITOrchestrator {
                     existingIds.add(ds.id);
                 }
             }
-            // 5. Scrape World Bank
+            // 4. Scrape World Bank (Open Access)
             const wbResults = await this.scrapeWorldBank(query, Math.floor(limit / 2));
             console.error(`  World Bank: Found ${wbResults.length} datasets`);
             for (const ds of wbResults) {
@@ -95,7 +80,7 @@ export class JITOrchestrator {
                     existingIds.add(ds.id);
                 }
             }
-            // 6. Scrape NASA
+            // 5. Scrape NASA (Open Access)
             const nasaResults = await this.scrapeNASA(query, Math.floor(limit / 2));
             console.error(`  NASA: Found ${nasaResults.length} datasets`);
             for (const ds of nasaResults) {
@@ -125,8 +110,7 @@ export class JITOrchestrator {
     async scrapeHuggingFace(query, limit) {
         const scraper = new HuggingFaceScraper();
         try {
-            // Use the query as a domain filter for now
-            // In the future, we can add a freeTextSearch parameter to the scraper
+            // Pass the query as a general search term
             return await scraper.scrape(limit, true, query);
         }
         catch (error) {
@@ -134,24 +118,6 @@ export class JITOrchestrator {
             return [];
         }
     }
-    /**
-     * Scrape Kaggle with search query
-     */
-    async scrapeKaggle(query, limit) {
-        const kaggleUser = process.env.KAGGLE_USERNAME;
-        const kaggleKey = process.env.KAGGLE_KEY;
-        if (!kaggleUser || !kaggleKey) {
-            return [];
-        }
-        try {
-            const scraper = new KaggleMetadataScraper(kaggleUser, kaggleKey);
-            return await scraper.scrape(query, limit);
-        }
-        catch (error) {
-            console.error(`  ERROR: Kaggle scrape failed: ${error.message}`);
-            return [];
-        }
-    }
     /**
      * Scrape UCI
      */

package/build/search/vector-store.js

@@ -74,6 +74,24 @@ export class VectorStore {
     add(id, vector) {
         this.idToVector.set(id, vector instanceof Float32Array ? vector : new Float32Array(vector));
     }
+    /**
+     * Delete a vector by ID
+     */
+    delete(id) {
+        return this.idToVector.delete(id);
+    }
+    /**
+     * Delete multiple vectors by IDs
+     */
+    deleteMany(ids) {
+        let count = 0;
+        for (const id of ids) {
+            if (this.idToVector.delete(id)) {
+                count++;
+            }
+        }
+        return count;
+    }
     search(queryVector, limit = 10) {
         const q = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
         const results = [];