@vespermcp/mcp-server 1.0.5 → 1.0.6

package/build/index.js

@@ -23,30 +23,94 @@ import { CacheService, MockRedisProvider } from "./cache/service.js";
 import { ImageAnalyzer } from "./quality/image-analyzer.js";
 import { MediaAnalyzer } from "./quality/media-analyzer.js";
 import { QualityOrchestrator } from "./quality/quality-orchestrator.js";
+import { ConfigManager } from "./config/config-manager.js";
+import { Selector } from "./utils/selector.js";
 // Determine absolute paths relative to the compiled script
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
-// We are in /build/index.js, so project root is one level up
-const projectRoot = path.join(__dirname, "..");
-const dbPath = path.join(projectRoot, "data", "metadata.db");
-const vectorPath = path.join(projectRoot, "data", "vectors.json");
+// appRoot: Where the source code/scripts are (inside node_modules or source)
+const appRoot = path.join(__dirname, "..");
+// dataRoot: Where database and user data live (in user home) to prevent data loss on update and running from node_modules
+const homeDir = process.env.HOME || process.env.USERPROFILE || appRoot;
+const dataRoot = path.join(homeDir, ".vesper");
+// Ensure data directory exists
+if (!fs.existsSync(dataRoot))
+    fs.mkdirSync(dataRoot, { recursive: true });
+const dbPath = path.join(dataRoot, "data", "metadata.db");
+const vectorPath = path.join(dataRoot, "data", "vectors.json");
+const errorLogPath = path.join(dataRoot, "vesper_errors.log");
+function logError(err, context) {
+    const timestamp = new Date().toISOString();
+    const stack = err.stack || String(err);
+    const msg = `[${timestamp}] ERROR in ${context}:\n${stack}\n${"-".repeat(50)}\n`;
+    fs.appendFileSync(errorLogPath, msg);
+    console.error(`[Vesper] Critical error logged to ${errorLogPath}`);
+}
+/**
+ * Sync Python scripts from the application package to the stable data directory (~/.vesper/python)
+ */
+function syncPythonScripts(appRoot, dataRoot) {
+    const pythonDest = path.join(dataRoot, "python");
+    if (!fs.existsSync(pythonDest))
+        fs.mkdirSync(pythonDest, { recursive: true });
+    // Sources to check for Python scripts
+    const sources = [
+        path.join(appRoot, "src", "python"),
+        path.join(appRoot, "build", "python"),
+        path.join(appRoot, "python")
+    ];
+    let syncedCount = 0;
+    for (const src of sources) {
+        if (fs.existsSync(src)) {
+            const files = fs.readdirSync(src);
+            for (const file of files) {
+                if (file.endsWith(".py")) {
+                    const srcPath = path.join(src, file);
+                    const destPath = path.join(pythonDest, file);
+                    // Only copy if file doesn't exist or is different size (basic sync)
+                    const srcStat = fs.statSync(srcPath);
+                    let shouldCopy = true;
+                    if (fs.existsSync(destPath)) {
+                        const destStat = fs.statSync(destPath);
+                        if (srcStat.size === destStat.size)
+                            shouldCopy = false;
+                    }
+                    if (shouldCopy) {
+                        fs.copyFileSync(srcPath, destPath);
+                        syncedCount++;
+                    }
+                }
+            }
+        }
+    }
+    if (syncedCount > 0) {
+        console.error(`[Vesper] Synced ${syncedCount} Python scripts to ${pythonDest}`);
+    }
+}
+// Sync scripts immediately
+syncPythonScripts(appRoot, dataRoot);
 const metadataStore = new MetadataStore(dbPath);
 const vectorStore = new VectorStore(vectorPath);
 const embedder = Embedder.getInstance();
 const searchEngine = new SearchEngine(metadataStore, vectorStore, embedder);
 const jobManager = JobManager.getInstance(metadataStore);
-const dataIngestor = new DataIngestor(projectRoot, metadataStore);
-const installService = new InstallService(projectRoot, metadataStore);
+// Use dataRoot for storage services (persistence)
+const dataIngestor = new DataIngestor(dataRoot, metadataStore);
+const installService = new InstallService(dataRoot, metadataStore);
 const cacheService = new CacheService(new MockRedisProvider());
-const qualityAnalyzer = new QualityAnalyzer(cacheService, projectRoot);
-const cleaningPlanner = new CleaningPlanner(cacheService);
-const dataCleaner = new DataCleaner(projectRoot);
-const pipelineExecutor = new PipelineExecutor(projectRoot);
-const dataSplitter = new DataSplitter(projectRoot);
-const dataExporter = new DataExporter(projectRoot);
-const imageAnalyzer = new ImageAnalyzer(projectRoot);
-const mediaAnalyzer = new MediaAnalyzer(projectRoot);
-const qualityOrchestrator = new QualityOrchestrator(projectRoot);
+const dataCleaner = new DataCleaner(__dirname);
+const pipelineExecutor = new PipelineExecutor(dataRoot, __dirname);
+const dataSplitter = new DataSplitter(__dirname);
+const dataExporter = new DataExporter(__dirname);
+// CRITICAL FIX: Pass __dirname (build directory) to analyzers
+// Python scripts are in build/python/, so analyzers should look relative to build/
+// NOT relative to project root (appRoot)
+process.env.PYTHONIOENCODING = "utf-8";
+const qualityAnalyzer = new QualityAnalyzer(cacheService, __dirname);
+const cleaningPlanner = new CleaningPlanner(cacheService, __dirname); // Pass __dirname for TargetDetector
+const imageAnalyzer = new ImageAnalyzer(__dirname);
+const mediaAnalyzer = new MediaAnalyzer(__dirname);
+const qualityOrchestrator = new QualityOrchestrator(__dirname);
 // Subscribe to job updates for real-time streaming to the UI
 jobManager.on("jobUpdated", (job) => {
     const level = job.status === "failed" ? "error" : "info";
@@ -58,12 +122,91 @@ jobManager.on("jobUpdated", (job) => {
     });
 });
 // IMPORTANT: Execute jobs when the manager emits them
-// This connects the queue logic to the execution context
 jobManager.on("processJob", async (job, execute) => {
-    // The JobManager controls concurrency, so if we receive this event,
-    // we should execute the job immediately.
-    await execute();
+    console.error(`[Vesper] Listener RECEIVED job: ${job?.id}, execute type: ${typeof execute}`);
+    if (typeof execute !== 'function') {
+        console.error(`[CRITICAL] execute is NOT a function! It is: ${typeof execute}`);
+        logError(new Error(`execute is ${typeof execute}`), "listener:execute_check");
+        return;
+    }
+    const prepareDatasetTask = async () => {
+        console.error(`[Vesper] Starting prepareDatasetTask for job ${job.id}...`);
+        const metadata = job.metadata ? JSON.parse(job.metadata) : {};
+        switch (job.type) {
+            case "prepare": return await handlePrepareJob(job.id, metadata.query);
+            case "clean": return await handleCleanJob(job.id, metadata.datasetId, metadata.ops);
+            default: throw new Error(`Unhandled job type: ${job.type}`);
+        }
+    };
+    try {
+        console.error(`[Vesper] Calling execute(prepareDatasetTask) for ${job.id}...`);
+        await execute(prepareDatasetTask);
+        console.error(`[Vesper] execute(prepareDatasetTask) COMPLETED for ${job.id}`);
+    }
+    catch (e) {
+        logError(e, `processJob:${job.type}:${job.id}`);
+        console.error(`[Vesper] Error in execute wrapper for ${job.id}: ${e.message}`);
+    }
 });
+/**
+ * Logic for preparing a dataset (Search + Ingest + Process)
+ */
+async function handlePrepareJob(jobId, query) {
+    const update = (updates) => jobManager.updateJob(jobId, updates);
+    update({ progress: 10, status_text: "Searching for best dataset matching query..." });
+    const results = await searchEngine.search(query, { limit: 1 });
+    if (results.length === 0) {
+        throw new Error("No datasets found matching the query. Try refining your search terms.");
+    }
+    const topDataset = results[0];
+    update({
+        progress: 20,
+        status_text: `Matched: ${topDataset.name} (${topDataset.source})`
+    });
+    const source = topDataset.source;
+    // Pre-check credentials for Kaggle
+    if (source === "kaggle") {
+        if (!process.env.KAGGLE_USERNAME || !process.env.KAGGLE_KEY ||
+            process.env.KAGGLE_USERNAME === "YOUR_KAGGLE_USERNAME") {
+            throw new Error("Kaggle credentials not set. Use 'kaggle login' or set KAGGLE_USERNAME/KAGGLE_KEY.");
+        }
+    }
+    update({ progress: 30, status_text: `Starting download from ${source}...` });
+    // ensureData handles download and returns path to the raw file
+    const rawFilePath = await dataIngestor.ensureData(topDataset.id, source, (msg, prog) => {
+        update({ status_text: msg, progress: 30 + (prog ? Math.floor(prog * 0.4) : 0) });
+    });
+    update({ progress: 70, status_text: "Analyzing dataset quality..." });
+    const report = await qualityAnalyzer.analyze(rawFilePath);
+    // Update local metadata with quality info
+    metadataStore.saveDataset({
+        ...topDataset,
+        quality_score: report.overall_score
+    });
+    update({ progress: 85, status_text: "Installing dataset into project..." });
+    const installPath = await installService.install(topDataset.id, rawFilePath);
+    update({ progress: 100, status_text: "Preparation complete!" });
+    return installPath;
+}
+/**
+ * Logic for cleaning a dataset
+ */
+async function handleCleanJob(jobId, datasetId, ops) {
+    const update = (updates) => jobManager.updateJob(jobId, updates);
+    let filePath = path.join(dataRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
+    if (datasetId === "demo" || !fs.existsSync(filePath)) {
+        const demoPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
+        if (fs.existsSync(demoPath))
+            filePath = demoPath;
+        else
+            throw new Error(`Data file not found for ${datasetId}`);
+    }
+    update({ status_text: "Cleaning dataset..." });
+    const result = await dataCleaner.clean(filePath, ops);
+    if (!result.success)
+        throw new Error(result.error);
+    return result.output_path;
+}
 // Create the server
 const server = new Server({
     name: "vesper",
@@ -308,10 +451,10 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
         case "analyze_quality": {
             const datasetId = String(request.params.arguments?.dataset_id);
-            let filePath = path.join(projectRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
+            let filePath = path.join(dataRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
             // Demo Fallback for easy testing
             if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                const demoPath = path.join(projectRoot, "e2e_demo_output", "raw_data.csv");
+                const demoPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
                 if (fs.existsSync(demoPath)) {
                     filePath = demoPath;
                 }
@@ -329,9 +472,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
         case "preview_cleaning": {
             const datasetId = String(request.params.arguments?.dataset_id);
-            let filePath = path.join(projectRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
+            let filePath = path.join(dataRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
             if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                const demoPath = path.join(projectRoot, "e2e_demo_output", "raw_data.csv");
+                const demoPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
                 if (fs.existsSync(demoPath)) {
                     filePath = demoPath;
                 }
@@ -340,8 +483,43 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 }
             }
             const report = await qualityAnalyzer.analyze(filePath);
-            const plan = await cleaningPlanner.generatePlan(datasetId, report);
+            // Phase 1: Target Detection
+            // We use the same TargetDetector instance inside CleaningPlanner now? 
+            // Actually, we instantiated it inside CleaningPlanner, so we just need to pass the file path 
+            // OR let the planner handle it if we update its signature to accept filePath.
+            // Let's check `CleaningPlanner.generatePlan` signature again.
+            // We updated it to accept `targetInfo`.
+            // So we need to run detection HERE and pass it.
+            // But `TargetDetector` is not exposed in `index.ts` scope yet.
+            // Let's create a global instance or use the one inside planner if exposed (it's private).
+            // Better approach: Instantiate TargetDetector here in index.ts for the tool content.
+            // Quick fix: Instantiate local detector or make global.
+            // I'll make a global `targetDetector` constant in index.ts
+            // But wait, I updated `CleaningPlanner` to instantiate its own detector.
+            // Does `CleaningPlanner` use it? No, I commented out the logic because it needed `filePath`.
+            // RETRY STRATEGY: 
+            // 1. Instantiate `targetDetector` in `index.ts`.
+            // 2. Run `detectTarget(filePath)`.
+            // 3. Pass result to `cleaningPlanner.generatePlan(..., targetInfo)`.
+            // I need to add `const targetDetector = new TargetDetector(__dirname);` to imports/init section first.
+            // But since I'm in this tool, I can't look back.
+            // I will assume I can add it, or just do it inside the case for now.
+            // To do it properly, I should have added `targetDetector` to the global scope in previous step.
+            // Let's do that in a separate step if needed. 
+            // For now, I'll instantiate it here.
+            const { TargetDetector } = await import("./preparation/target-detector.js");
+            const detector = new TargetDetector(__dirname);
+            const targetResult = await detector.detectTarget(filePath);
+            const targetInfo = targetResult.target_column ? {
+                target: targetResult.target_column,
+                confidence: targetResult.confidence
+            } : undefined;
+            const plan = await cleaningPlanner.generatePlan(datasetId, report, undefined, targetInfo);
             let explanation = `### 📋 Cleaning Plan for ${datasetId}\n\n`;
+            if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
+                explanation += `🎯 **Target Detected**: '${targetInfo.target}' (Confidence: ${targetInfo.confidence.toFixed(2)})\n`;
+                explanation += `   - **Action**: Renaming to 'target' for consistency.\n\n`;
+            }
             explanation += `Estimated Quality Improvement: +${plan.estimated_impact.quality_score_improvement} points\n\n`;
             if (plan.operations.length === 0) {
                 explanation += "✅ No cleaning operations required.";
@@ -358,25 +536,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         case "custom_clean": {
             const datasetId = String(request.params.arguments?.dataset_id);
             const ops = request.params.arguments?.operations;
-            let filePath = path.join(projectRoot, "data", "raw", `${datasetId.replace(/\//g, "_")}.csv`);
-            if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                const demoPath = path.join(projectRoot, "e2e_demo_output", "raw_data.csv");
-                if (fs.existsSync(demoPath)) {
-                    filePath = demoPath;
-                }
-                else {
-                    throw new McpError(ErrorCode.InvalidParams, `Local data file not found for ${datasetId}`);
-                }
-            }
             const job = jobManager.createJob("clean", 0, { datasetId, ops });
-            // Run in background
-            jobManager.runJob(job.id, async (update) => {
-                update({ status_text: "Cleaning dataset..." });
-                const result = await dataCleaner.clean(filePath, ops);
-                if (!result.success)
-                    throw new Error(result.error);
-                return result.output_path;
-            });
             return {
                 content: [{ type: "text", text: `Job started successfully. ID: ${job.id}. Use check_job_status to monitor progress.` }]
             };
@@ -384,56 +544,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         case "prepare_dataset": {
             const query = String(request.params.arguments?.query);
             const job = jobManager.createJob("prepare", 0, { query });
-            // Orchestrated Background Task
-            jobManager.runJob(job.id, async (update) => {
-                try {
-                    update({ progress: 10, status_text: "Searching for best dataset matching query..." });
-                    const results = await searchEngine.search(query, { limit: 1 });
-                    if (results.length === 0) {
-                        throw new Error("No datasets found matching the query. Try refining your search terms.");
-                    }
-                    const topDataset = results[0];
-                    // Phase 6: Real Ingestion
-                    update({
-                        progress: 20,
-                        status_text: `Matched: ${topDataset.name} (${topDataset.source})`
-                    });
-                    const source = topDataset.source;
-                    // Pre-check credentials for Kaggle to fail fast with helpful message
-                    if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
-                        const errorMsg = dataIngestor.getKaggleCredentialError();
-                        throw new Error(errorMsg);
-                    }
-                    const filePath = await dataIngestor.ensureData(topDataset.id, source, (msg, prog) => {
-                        update({
-                            status_text: msg,
-                            progress: prog !== undefined ? 20 + Math.floor(prog * 0.3) : undefined // 20% -> 50%
-                        });
-                    });
-                    update({ progress: 55, status_text: "Analyzing dataset quality..." });
-                    const quality = await qualityAnalyzer.analyze(filePath);
-                    const pipelineResult = await pipelineExecutor.runPipeline(topDataset.id, filePath, "csv", (msg) => {
-                        update({ status_text: msg });
-                    });
-                    update({ progress: 90, status_text: "Installing dataset into codebase..." });
-                    const installPath = await installService.install(topDataset.id, pipelineResult.final_output_path);
-                    update({ progress: 100, status_text: "Preparation complete!" });
-                    const message = `✅ Preparation complete for ${topDataset.name}.\n` +
-                        `📦 Dataset installed to: ${installPath}\n` +
-                        `🚀 You can now use this dataset for training your models.`;
-                    return message;
-                }
-                catch (error) {
-                    // Re-throw with enhanced error message for credential issues
-                    const errorMessage = error.message || String(error);
-                    if (errorMessage.includes("Kaggle credentials") || errorMessage.includes("KAGGLE")) {
-                        throw new Error(errorMessage);
-                    }
-                    throw error;
-                }
-            });
             return {
-                content: [{ type: "text", text: `Autonomous preparation job started. ID: ${job.id}. Use check_job_status to monitor progress.` }]
+                content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you.` }]
             };
         }
         case "compare_datasets": {
@@ -479,7 +591,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             // Check if we need conversion
             const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
             if (currentExt !== requestedFormat) {
-                console.log(`[Export] Format mismatch (${currentExt} vs ${requestedFormat}). Converting...`);
+                console.error(`[Export] Format mismatch (${currentExt} vs ${requestedFormat}). Converting...`);
                 try {
                     const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, requestedFormat);
                     sourcePath = pipelineResult.final_output_path;
@@ -645,9 +757,74 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
     }
 });
 async function main() {
+    const args = process.argv.slice(2);
+    const isSetup = args.includes("--setup") || args.includes("setup");
+    const isSilent = args.includes("--silent");
+    // If run in setup mode OR in a terminal without args (human call), show setup wizard
+    if (isSetup || (process.stdin.isTTY && args.length === 0)) {
+        await runSetupWizard(isSilent);
+        return;
+    }
+    // Otherwise proceed to server mode (for IDEs/Agents)
+    console.error(`[Vesper] Starting server...`);
+    console.error(`[Vesper] dataRoot: ${dataRoot}`);
+    console.error(`[Vesper] dbPath: ${dbPath}`);
     const transport = new StdioServerTransport();
     await server.connect(transport);
     console.error("Vesper MCP server running on stdio");
+    console.error("Tip: To configure Vesper for your IDE, run: npx @vespermcp/mcp-server --setup");
+    console.log("[Vesper] Main loop finished");
+}
+async function runSetupWizard(silent = false) {
+    const configManager = new ConfigManager();
+    if (!silent) {
+        console.log(`\n🚀 Welcome to Vesper MCP Setup!`);
+        console.log(`--------------------------------`);
+        console.log(`Searching for IDE configurations...`);
+    }
+    const ides = configManager.detectIDEs();
+    if (ides.length === 0) {
+        if (!silent) {
+            console.log("\n❌ No supported IDEs detected.");
+            console.log("I checked for:");
+            console.log(" - Cursor (Global & Project)");
+            console.log(" - Claude Desktop");
+            console.log(" - VS Code (Standard MCP, Copilot Chat, Cline, Roo Code)");
+            console.log("\nIf you are using VS Code or Cursor, please make sure they are installed.");
+            console.log("For project-specific setup, run this command inside your project folder.");
+        }
+        return;
+    }
+    if (silent) {
+        for (const ide of ides) {
+            await configManager.installTo(ide);
+        }
+        return;
+    }
+    console.log(`\nFound ${ides.length} potential application(s):`);
+    const selector = new Selector("Select applications to configure for Vesper:", ides.map(ide => ({
+        name: ide.name,
+        value: ide,
+        selected: true
+    })));
+    const selectedIDEs = await selector.run();
+    if (selectedIDEs.length > 0) {
+        console.log(`\nInstalling to ${selectedIDEs.length} application(s)...\n`);
+        for (const ide of selectedIDEs) {
+            process.stdout.write(`Installing to ${ide.name}... `);
+            const success = await configManager.installTo(ide);
+            if (success) {
+                process.stdout.write("✅\n");
+            }
+            else {
+                process.stdout.write("❌ (Check permissions or if file is in use)\n");
+            }
+        }
+        console.log("\n✨ Setup complete! Please RESTART your IDE(s) to apply changes.");
+    }
+    else {
+        console.log("\nSetup skipped. No applications selected.");
+    }
 }
 main().catch((error) => {
     console.error("Server error:", error);

package/build/ingestion/ingestor.js

@@ -40,7 +40,7 @@ export class DataIngestor {
             return status.local_path;
         }
         if (status && status.status === 'downloading') {
-            console.log(`[Ingestor] Dataset ${datasetId} status is 'downloading'. Attempting to resume/concurrently monitor.`);
+            console.error(`[Ingestor] Dataset ${datasetId} status is 'downloading'. Attempting to resume/concurrently monitor.`);
             // In a better system we'd use a lock, but for now we let it resume 
             // the RobustDownloader handles the actual file locking/range logic.
         }
@@ -67,27 +67,10 @@ export class DataIngestor {
             }
         }
         else if (source === "kaggle") {
-            // Check credentials early and provide helpful error
-            if (!this.kaggleDownloader.hasCredentials()) {
-                const errorMsg = this.kaggleDownloader.getCredentialError();
-                this.failDownload(datasetId, errorMsg);
-                throw new Error(errorMsg);
-            }
-            const safeId = datasetId.replace("kaggle:", "").replace(/\//g, "_");
-            const targetDir = path.join(this.rawDataDir, `kaggle_${safeId}`);
-            this.store.registerDownload(datasetId, targetDir, "downloading");
-            try {
-                const primaryFile = await this.kaggleDownloader.download(datasetId.replace("kaggle:", ""), targetDir, (progress) => {
-                    onProgress?.("Downloading Kaggle archive...", progress);
-                });
-                const stats = fs.statSync(primaryFile);
-                this.completeDownload(datasetId, primaryFile, stats.size);
-                return primaryFile;
-            }
-            catch (e) {
-                this.failDownload(datasetId, e.message);
-                throw e;
-            }
+            // Kaggle support has been disabled
+            const errorMsg = "Kaggle datasets are no longer supported. Please use HuggingFace or other open-access sources.";
+            this.failDownload(datasetId, errorMsg);
+            throw new Error(errorMsg);
         }
         throw new Error(`Download logic for ${source} not yet implemented`);
     }

package/build/install/install-service.js

@@ -35,7 +35,7 @@ export class InstallService {
         // Update metadata
         const absolutePath = path.resolve(targetPath);
         this.metadataStore.updateInstallPath(datasetId, absolutePath);
-        console.log(`[InstallService] Dataset ${datasetId} installed to ${absolutePath}`);
+        console.error(`[InstallService] Dataset ${datasetId} installed to ${absolutePath}`);
         return absolutePath;
     }
 }

package/build/jobs/manager.js

@@ -64,11 +64,22 @@ export class JobManager extends EventEmitter {
         this.activeWorkers++;
         this.updateJob(job.id, { status: "running", status_text: "Picked up by worker" });
         const startTime = Date.now();
+        const listeners = this.listenerCount("processJob");
+        console.error(`[JobManager] Emitting processJob for ${job.id}. Active listeners: ${listeners}`);
         // In a real system, we'd have a registry of handlers for each JobType.
         // For now, we emit an event so the orchestrator can run it.
-        this.emit("processJob", job, async (task) => {
+        this.emit("processJob", job, async (jobExecutionTask) => {
+            console.error(`[JobManager] Wrapper received jobExecutionTask: ${typeof jobExecutionTask}`);
+            if (typeof jobExecutionTask !== 'function') {
+                console.error(`[JobManager] Error: jobExecutionTask is NOT a function! It is: ${typeof jobExecutionTask}`);
+                this.updateJob(job.id, {
+                    status: "failed",
+                    status_text: "Internal error: jobExecutionTask is not a function"
+                });
+                return;
+            }
             try {
-                const resultUrl = await task();
+                const resultUrl = await jobExecutionTask();
                 const duration = Date.now() - startTime;
                 this.updateJob(job.id, {
                     status: "completed",
@@ -103,14 +114,6 @@ export class JobManager extends EventEmitter {
         // Try to start another worker if capacity allows
         this.processQueue();
     }
-    /**
-     * Helper to run a task as a job with automatic status updates
-     */
-    runJob(id, task) {
-        this.emit("processJob", { id }, async () => {
-            return await task((updates) => this.updateJob(id, updates));
-        });
-    }
     /**
      * Update job status and progress
      */
@@ -118,6 +121,10 @@ export class JobManager extends EventEmitter {
         const job = this.store.getJob(id);
         if (!job)
             return;
+        // Correctly handle metadata update if it's an object
+        if (updates.metadata && typeof updates.metadata !== 'string') {
+            updates.metadata = JSON.stringify(updates.metadata);
+        }
         const updatedJob = {
             ...job,
             ...updates,

package/build/metadata/monitoring-service.js

@@ -90,7 +90,7 @@ export class MonitoringService {
         }
     }
     async sendToWebhook(webhook, diff) {
-        console.log(`[MonitoringService] Sending notification to ${webhook.name} (${webhook.channel}) for dataset ${diff.dataset_id}`);
+        console.error(`[MonitoringService] Sending notification to ${webhook.name} (${webhook.channel}) for dataset ${diff.dataset_id}`);
         // In a real implementation, this would be an HTTP POST
         // For now, we simulate the payload
         const payload = {
@@ -101,7 +101,7 @@ export class MonitoringService {
         // await axios.post(webhook.url, payload);
     }
     async triggerReprocess(datasetId) {
-        console.log(`[MonitoringService] Auto-reprocessing dataset ${datasetId}...`);
+        console.error(`[MonitoringService] Auto-reprocessing dataset ${datasetId}...`);
         // This would call IngestionService or similar
     }
 }

package/build/metadata/scraper.js

@@ -8,8 +8,8 @@ export class HuggingFaceScraper {
      * Bulk discovery: Fetch many datasets quickly without deep details.
      * Hits the 25k target in minutes.
      */
-    async scrapeBulk(limit = 1000, domainFilter) {
-        const filterMsg = domainFilter ? `, domain: ${domainFilter}` : "";
+    async scrapeBulk(limit = 1000, query) {
+        const filterMsg = query ? `, query: ${query}` : "";
         console.error(`[Bulk Scraper] Fetching datasets (target limit: ${limit}${filterMsg})...`);
         const results = [];
         let processed = 0;
@@ -18,7 +18,7 @@ export class HuggingFaceScraper {
             for await (const ds of listDatasets({
                 limit: limit,
                 additionalFields: ["description", "tags", "downloadsAllTime", "createdAt"],
-                search: { query: domainFilter },
+                search: { query: query },
                 ...(hfToken ? { accessToken: hfToken } : {})
             })) {
                 if (results.length >= limit)
@@ -86,9 +86,9 @@ export class HuggingFaceScraper {
         }
         return results;
     }
-    async scrape(limit = 100, applyMVPFilters = true, domainFilter // Optional: filter by domain (medicine, healthcare, security, etc.)
+    async scrape(limit = 100, applyMVPFilters = true, query // Use as general search query
     ) {
-        const filterMsg = domainFilter ? `, domain: ${domainFilter}` : "";
+        const filterMsg = query ? `, query: ${query}` : "";
         console.error(`Fetching datasets (target limit: ${limit}, MVP filters: ${applyMVPFilters}${filterMsg})...`);
         const results = [];
         let processed = 0;
@@ -105,7 +105,7 @@ export class HuggingFaceScraper {
             for await (const ds of listDatasets({
                 limit: fetchLimit,
                 additionalFields: ["description", "tags"],
-                search: { query: domainFilter },
+                search: { query: query },
                 ...(hfToken ? { accessToken: hfToken } : {})
             })) {
                 if (results.length >= limit)
@@ -192,8 +192,8 @@ export class HuggingFaceScraper {
                         const columns = this.extractColumns(cardData, splits);
                         const task = this.extractTask(tags);
                         const domain = classifyDomain(description, tags, repoId, task);
-                        if (domainFilter && domain !== domainFilter)
-                            return;
+                        // REMOVED strict domain filtering that caused search bias
+                        // if (query && domain !== query) return; 
                         const metadata = {
                             id: repoId,
                             source: "huggingface",

package/build/metadata/store.js

@@ -17,7 +17,7 @@ export class MetadataStore {
         // Add install_path if missing
         try {
             this.db.exec("ALTER TABLE datasets ADD COLUMN install_path TEXT");
-            console.log("[MetadataStore] Migrated: Added install_path column");
+            console.error("[MetadataStore] Migrated: Added install_path column");
         }
         catch (e) {
             // Probably already exists
@@ -315,10 +315,25 @@ export class MetadataStore {
      * Perform database maintenance (VACUUM, ANALYZE).
      */
     optimize() {
-        console.log("[MetadataStore] Running database optimization (VACUUM, ANALYZE)...");
+        console.error("[MetadataStore] Running database optimization (VACUUM, ANALYZE)...");
         this.db.exec("VACUUM");
         this.db.exec("ANALYZE");
     }
+    /**
+     * Delete all datasets from a specific source
+     */
+    deleteBySource(source) {
+        const info = this.db.prepare("DELETE FROM datasets WHERE source = ?").run(source);
+        console.error(`[MetadataStore] Deleted ${info.changes} datasets from source: ${source}`);
+        return info.changes;
+    }
+    /**
+     * Get all dataset IDs from a specific source
+     */
+    getDatasetIdsBySource(source) {
+        const rows = this.db.prepare("SELECT id FROM datasets WHERE source = ?").all(source);
+        return rows.map(r => r.id);
+    }
     close() {
         this.db.close();
     }