@toolsdk.ai/registry 1.0.102 → 1.0.103

package/dist/search/search-service.js

@@ -0,0 +1,417 @@
+/**
+ * MeiliSearch Service for Awesome MCP Registry
+ * Handles search indexing and querying for MCP packages
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { MeiliSearch } from "meilisearch";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+class SearchService {
+    constructor(indexName = "mcp-packages") {
+        // MeiliSearch configuration
+        this.host = process.env.MEILI_HTTP_ADDR || "http://localhost:7700";
+        this.apiKey = process.env.MEILI_MASTER_KEY || null;
+        this.indexName = indexName;
+        // Initialize MeiliSearch client
+        this.client = new MeiliSearch({
+            host: this.host,
+            apiKey: this.apiKey || undefined,
+        });
+        this._index = null;
+        this.isInitialized = false;
+    }
+    /**
+     * Get the initialization status
+     */
+    getIsInitialized() {
+        return this.isInitialized;
+    }
+    /**
+     * Get the MeiliSearch client
+     */
+    getClient() {
+        return this.client;
+    }
+    get index() {
+        return this._index;
+    }
+    set index(value) {
+        this._index = value;
+    }
+    async initialize() {
+        try {
+            console.log(`Connecting to MeiliSearch at ${this.host}...`);
+            // Check if MeiliSearch is running
+            await this.client.health();
+            console.log("✅ MeiliSearch is healthy");
+            // Create or get the index
+            try {
+                await this.client.createIndex(this.indexName, { primaryKey: "id" });
+                console.log(`✅ Created new index: ${this.indexName}`);
+            }
+            catch (error) {
+                if (error.message.includes("exists")) {
+                    console.log(`✅ Using existing index: ${this.indexName}`);
+                }
+                else {
+                    throw error;
+                }
+            }
+            // Get the index object
+            this.index = await this.client.getIndex(this.indexName);
+            // Configure search settings
+            await this.configureIndex();
+            this.isInitialized = true;
+            console.log("✅ Search service initialized successfully");
+        }
+        catch (error) {
+            console.error("❌ Failed to initialize search service:", error.message);
+            console.log("💡 Make sure MeiliSearch is running on", this.host);
+            throw error;
+        }
+    }
+    /**
+     * Configure search index settings for optimal MCP package search
+     */
+    async configureIndex() {
+        try {
+            // Configure searchable attributes (ranked by importance)
+            if (this.index) {
+                await this.index.updateSettings({
+                    searchableAttributes: [
+                        "name",
+                        "packageName",
+                        "description",
+                        "tools",
+                        "category",
+                        "author",
+                        "keywords",
+                    ],
+                    // Configure filterable attributes
+                    filterableAttributes: ["category", "validated", "author", "hasTools", "popularity"],
+                    // Configure sortable attributes
+                    sortableAttributes: ["popularity", "name", "category"],
+                    // Configure ranking rules for relevance
+                    rankingRules: [
+                        "words",
+                        "typo",
+                        "proximity",
+                        "attribute",
+                        "sort",
+                        "exactness",
+                        "popularity:desc",
+                    ],
+                    // Configure synonyms for better search
+                    synonyms: {
+                        ai: ["artificial intelligence", "machine learning", "ml"],
+                        db: ["database"],
+                        api: ["rest", "graphql"],
+                        auth: ["authentication", "authorization"],
+                        mcp: ["model context protocol"],
+                    },
+                });
+            }
+            console.log("✅ Search index configured");
+        }
+        catch (error) {
+            console.error("❌ Failed to configure index:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Transform package data for search indexing
+     */
+    transformPackageForIndex(packageName, packageData) {
+        // Extract tools information
+        const tools = packageData.tools || {};
+        const toolNames = Object.keys(tools);
+        const toolDescriptions = Object.values(tools)
+            .map((t) => t.description || "")
+            .join(" ");
+        // Calculate basic popularity score (can be enhanced with real metrics)
+        const popularity = this.calculatePopularityScore(packageData);
+        // Create a safe ID by encoding special characters
+        const safeId = this.createSafeId(packageName);
+        return {
+            id: safeId,
+            name: packageData.name || packageName,
+            packageName: packageName,
+            description: packageData.description || "",
+            category: packageData.category || "uncategorized",
+            validated: packageData.validated || false,
+            author: this.extractAuthor(packageName),
+            tools: `${toolNames.join(" ")} ${toolDescriptions}`,
+            toolCount: toolNames.length,
+            hasTools: toolNames.length > 0,
+            keywords: this.extractKeywords(packageData, packageName),
+            popularity: popularity,
+            path: packageData.path || "",
+        };
+    }
+    /**
+     * Create a safe ID for MeiliSearch (alphanumeric, hyphens, underscores only)
+     */
+    createSafeId(packageName) {
+        return packageName
+            .replace(/@/g, "at-")
+            .replace(/\//g, "-")
+            .replace(/\./g, "_")
+            .replace(/:/g, "-")
+            .replace(/[^a-zA-Z0-9\-_]/g, "_")
+            .substring(0, 500); // Keep under 511 byte limit
+    }
+    /**
+     * Calculate a popularity score for ranking
+     */
+    calculatePopularityScore(packageData) {
+        let score = 0;
+        // Boost for validated packages
+        if (packageData.validated)
+            score += 10;
+        // Boost for packages with tools
+        const toolCount = Object.keys(packageData.tools || {}).length;
+        score += toolCount * 2;
+        // Boost for packages with good descriptions
+        if (packageData.description && packageData.description.length > 50)
+            score += 5;
+        return score;
+    }
+    /**
+     * Extract author from package name
+     */
+    extractAuthor(packageName) {
+        if (packageName.startsWith("@")) {
+            return packageName.split("/")[0].substring(1);
+        }
+        return "";
+    }
+    /**
+     * Extract relevant keywords from package data
+     */
+    extractKeywords(packageData, packageName) {
+        const keywords = [];
+        // Add category as keyword
+        if (packageData.category) {
+            keywords.push(packageData.category.replace(/-/g, " "));
+        }
+        // Extract keywords from description
+        if (packageData.description) {
+            const desc = packageData.description.toLowerCase();
+            const commonKeywords = [
+                "api",
+                "database",
+                "auth",
+                "search",
+                "ai",
+                "ml",
+                "tool",
+                "server",
+                "client",
+            ];
+            commonKeywords.forEach((keyword) => {
+                if (desc.includes(keyword))
+                    keywords.push(keyword);
+            });
+        }
+        // Extract from package name
+        const nameWords = packageName
+            .replace(/[@/\-_]/g, " ")
+            .split(" ")
+            .filter((w) => w.length > 2);
+        keywords.push(...nameWords);
+        return [...new Set(keywords)].join(" ");
+    }
+    /**
+     * Index all packages from the packages-list.json file
+     */
+    async indexPackages() {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            console.log("📥 Loading packages data...");
+            const packagesPath = path.join(__dirname, "..", "..", "indexes", "packages-list.json");
+            const packagesData = await fs.readFile(packagesPath, "utf8");
+            const packages = JSON.parse(packagesData);
+            console.log(`📦 Found ${Object.keys(packages).length} packages to index`);
+            // Transform packages for indexing
+            const documents = Object.entries(packages).map(([packageName, packageData]) => this.transformPackageForIndex(packageName, packageData));
+            console.log("🔄 Indexing packages...");
+            // Add documents to index in batches
+            const batchSize = 1000;
+            const batches = [];
+            for (let i = 0; i < documents.length; i += batchSize) {
+                batches.push(documents.slice(i, i + batchSize));
+            }
+            for (let i = 0; i < batches.length; i++) {
+                const batch = batches[i];
+                if (this.index) {
+                    const task = await this.index.addDocuments(batch);
+                    console.log(`📝 Indexed batch ${i + 1}/${batches.length} (${batch.length} documents) - Task ID: ${task.taskUid}`);
+                    // Wait for task completion
+                    await this.client.waitForTask(task.taskUid);
+                }
+            }
+            // Get final stats
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            const stats = await this.index.getStats();
+            console.log(`✅ Indexing complete! ${stats.numberOfDocuments} documents indexed`);
+            return stats;
+        }
+        catch (error) {
+            console.error("❌ Failed to index packages:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Search packages with advanced options
+     */
+    async search(query, options = {}) {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            const searchOptions = Object.assign({ limit: options.limit || 20, offset: options.offset || 0, filter: options.filter || undefined, sort: options.sort || ["popularity:desc"], attributesToHighlight: ["name", "description"], attributesToCrop: ["description"], cropLength: 100, highlightPreTag: "<mark>", highlightPostTag: "</mark>", matchingStrategy: options.matchingStrategy || "last" }, options);
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            const results = await this.index.search(query, searchOptions);
+            return {
+                hits: results.hits,
+                query: results.query,
+                processingTimeMs: results.processingTimeMs,
+                limit: results.limit,
+                offset: results.offset,
+                estimatedTotalHits: results.estimatedTotalHits,
+            };
+        }
+        catch (error) {
+            console.error("❌ Search failed:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Get search suggestions/autocomplete
+     */
+    async suggest(query, limit = 10) {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            const results = await this.index.search(query, {
+                limit: limit,
+                attributesToRetrieve: ["name", "packageName", "category"],
+                attributesToHighlight: ["name"],
+                cropLength: 0,
+            });
+            return results.hits.map((hit) => {
+                var _a;
+                return ({
+                    name: hit.name || "",
+                    packageName: hit.packageName || "",
+                    category: hit.category || "",
+                    highlighted: ((_a = hit._formatted) === null || _a === void 0 ? void 0 : _a.name) || hit.name || "",
+                });
+            });
+        }
+        catch (error) {
+            console.error("❌ Suggestions failed:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Get faceted search results (for filters)
+     */
+    async getFacets() {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            const results = await this.index.search("", {
+                limit: 0,
+                facets: ["category", "validated", "author"],
+            });
+            return results.facetDistribution;
+        }
+        catch (error) {
+            console.error("❌ Failed to get facets:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Get index statistics
+     */
+    async getStats() {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            return await this.index.getStats();
+        }
+        catch (error) {
+            console.error("❌ Failed to get stats:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Clear the index
+     */
+    async clearIndex() {
+        if (!this.isInitialized) {
+            throw new Error("Search service not initialized. Call initialize() first.");
+        }
+        try {
+            if (!this.index) {
+                throw new Error("Index is not initialized");
+            }
+            const task = await this.index.deleteAllDocuments();
+            await this.client.waitForTask(task.taskUid);
+            console.log("✅ Index cleared");
+        }
+        catch (error) {
+            console.error("❌ Failed to clear index:", error.message);
+            throw error;
+        }
+    }
+    /**
+     * Health check for the search service
+     */
+    async healthCheck() {
+        try {
+            await this.client.health();
+            const stats = this.isInitialized ? await this.getStats() : null;
+            return {
+                status: "healthy",
+                host: this.host,
+                initialized: this.isInitialized,
+                indexName: this.indexName,
+                documentCount: (stats === null || stats === void 0 ? void 0 : stats.numberOfDocuments) || 0,
+            };
+        }
+        catch (_error) {
+            return {
+                status: "unhealthy",
+                host: this.host,
+                initialized: false,
+                indexName: this.indexName,
+                documentCount: 0,
+            };
+        }
+    }
+}
+// Export singleton instance
+const searchService = new SearchService();
+export default searchService;

package/dist/search/search.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/search/search.test.js

@@ -0,0 +1,100 @@
+import { describe, expect, it, vi } from "vitest";
+import searchService from "./search-service";
+// Mock the MeiliSearch client
+vi.mock("meilisearch", () => {
+    return {
+        MeiliSearch: vi.fn().mockImplementation(() => {
+            const mockIndex = {
+                updateSettings: vi.fn().mockResolvedValue({ taskUid: "settings-task" }),
+                addDocuments: vi.fn().mockResolvedValue({ taskUid: "documents-task" }),
+                getStats: vi.fn().mockResolvedValue({
+                    numberOfDocuments: 100,
+                    isIndexing: false,
+                    fieldDistribution: {},
+                }),
+                search: vi.fn().mockResolvedValue({
+                    hits: [{ id: "1", name: "Test Package" }],
+                    query: "test",
+                    processingTimeMs: 10,
+                    limit: 20,
+                    offset: 0,
+                    estimatedTotalHits: 1,
+                }),
+                getTask: vi.fn().mockResolvedValue({ status: "succeeded" }),
+                deleteAllDocuments: vi.fn().mockResolvedValue({ taskUid: "clear-task" }),
+            };
+            return {
+                health: vi.fn().mockResolvedValue({ status: "available" }),
+                createIndex: vi.fn().mockResolvedValue({ uid: "test-task" }),
+                getIndex: vi.fn().mockResolvedValue(mockIndex),
+                index: vi.fn().mockReturnValue(mockIndex),
+                waitForTask: vi.fn().mockResolvedValue({ status: "succeeded" }),
+            };
+        }),
+    };
+});
+// Mock fs module
+vi.mock("node:fs/promises", () => {
+    return {
+        default: {
+            readFile: vi.fn().mockResolvedValue(JSON.stringify({
+                "@test/package": {
+                    name: "Test Package",
+                    description: "A test package",
+                },
+            })),
+        },
+    };
+});
+describe("SearchService - MCP Registry Search Service Test", () => {
+    it("should initialize search service successfully", async () => {
+        const result = await searchService.initialize();
+        expect(result).toBeUndefined();
+    });
+    it("should transform package data for indexing", () => {
+        const packageName = "@test/package";
+        const packageData = {
+            name: "Test Package",
+            description: "A test package",
+            category: "testing",
+            validated: true,
+            tools: {
+                "test-tool": { description: "A test tool" },
+            },
+        };
+        const result = searchService.transformPackageForIndex(packageName, packageData);
+        expect(result).toEqual({
+            id: "at-test-package",
+            name: "Test Package",
+            packageName: "@test/package",
+            description: "A test package",
+            category: "testing",
+            validated: true,
+            author: "test",
+            tools: "test-tool A test tool",
+            toolCount: 1,
+            hasTools: true,
+            keywords: expect.any(String),
+            popularity: expect.any(Number),
+            path: "",
+        });
+    });
+    it("should perform search successfully", async () => {
+        // First initialize the service
+        await searchService.initialize();
+        // Mock the search result
+        const mockResult = {
+            hits: [{ id: "1", name: "Test Package" }],
+            query: "test",
+            processingTimeMs: 10,
+            limit: 20,
+            offset: 0,
+            estimatedTotalHits: 1,
+        };
+        if (searchService.index) {
+            searchService.index.search = vi.fn().mockResolvedValue(mockResult);
+        }
+        const result = await searchService.search("test");
+        expect(result).toEqual(mockResult);
+    });
+});

package/indexes/packages-list.json

@@ -47880,4 +47880,4 @@
     "validated": false,
     "tools": {}
   }
-}
+}