@ignidor/web-search-mcp 1.0.0

package/dist/utils/concurrency.js

@@ -0,0 +1,53 @@
+// Concurrency Control Utilities
+import pLimit from 'p-limit';
+// ============================================================================
+// Concurrency Limiter
+// ============================================================================
+/**
+ * Creates a concurrency limiter for controlling parallel operations.
+ * @param concurrency - Maximum number of concurrent operations (default: 5)
+ * @returns A limit function that wraps async operations
+ */
+export function createConcurrencyLimiter(concurrency = 5) {
+    return pLimit(concurrency);
+}
+// ============================================================================
+// Batch Processing
+// ============================================================================
+/**
+ * Processes items in batches with concurrency control.
+ * @param items - Array of items to process
+ * @param processor - Async function to process each item
+ * @param concurrency - Maximum concurrent operations
+ * @returns Array of results in the same order as input
+ */
+export async function processBatch(items, processor, concurrency = 5) {
+    const limit = createConcurrencyLimiter(concurrency);
+    const tasks = items.map((item, index) => limit(() => processor(item, index)));
+    return Promise.all(tasks);
+}
+// ============================================================================
+// Parallel Processing with Error Handling
+// ============================================================================
+/**
+ * Processes items with graceful error handling.
+ * Failed items return undefined instead of rejecting the entire batch.
+ * @param items - Array of items to process
+ * @param processor - Async function to process each item
+ * @param concurrency - Maximum concurrent operations
+ * @returns Array of successful results or undefined for failed items
+ */
+export async function processBatchWithErrorHandling(items, processor, concurrency = 5) {
+    const limit = createConcurrencyLimiter(concurrency);
+    const tasks = items.map((item, index) => limit(async () => {
+        try {
+            return await processor(item, index);
+        }
+        catch (error) {
+            console.error(`[Concurrency] Item ${index} failed:`, error instanceof Error ? error.message : error);
+            return undefined;
+        }
+    }));
+    return Promise.all(tasks);
+}
+//# sourceMappingURL=concurrency.js.map

package/dist/utils/concurrency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../src/utils/concurrency.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,cAAsB,CAAC;IAC9D,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC;AAC7B,CAAC;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAU,EACV,SAAiD,EACjD,cAAsB,CAAC;IAEvB,MAAM,KAAK,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CACpC,CAAC;IAEF,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,+EAA+E;AAC/E,0CAA0C;AAC1C,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,KAAU,EACV,SAAiD,EACjD,cAAsB,CAAC;IAEvB,MAAM,KAAK,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,sBAAsB,KAAK,UAAU,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACrG,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}

package/dist/utils/validators.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Validates that a URL is safe and properly formatted.
+ * Blocks localhost, private IPs, and non-HTTP protocols to prevent SSRF attacks.
+ */
+export declare function isValidUrl(url: string): boolean;
+/**
+ * Validates search query for security and length limits.
+ */
+export declare function isValidQuery(query: string): {
+    valid: boolean;
+    error?: string;
+};
+/**
+ * Clamps a number between min and max values.
+ */
+export declare function clamp(value: number, min: number, max: number): number;
+/**
+ * Validates an array of URLs, returning only valid URLs.
+ */
+export declare function validateUrls(urls: string[]): string[];
+//# sourceMappingURL=validators.d.ts.map

package/dist/utils/validators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../../src/utils/validators.ts"],"names":[],"mappings":"AAMA;;;GAGG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CA4B/C;AAMD;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAgB9E;AAMD;;GAEG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAErE;AAMD;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAIrD"}

package/dist/utils/validators.js

@@ -0,0 +1,75 @@
+// Validators and Sanitizers
+// ============================================================================
+// URL Validation (SSRF Protection)
+// ============================================================================
+/**
+ * Validates that a URL is safe and properly formatted.
+ * Blocks localhost, private IPs, and non-HTTP protocols to prevent SSRF attacks.
+ */
+export function isValidUrl(url) {
+    if (!url || typeof url !== 'string')
+        return false;
+    try {
+        const parsed = new URL(url);
+        const hostname = parsed.hostname.toLowerCase();
+        // Block localhost variants
+        if (hostname === 'localhost' ||
+            hostname === '127.0.0.1' ||
+            hostname === '0.0.0.0' ||
+            hostname === '[::1]' ||
+            hostname.startsWith('127.')) {
+            return false;
+        }
+        // Block private IP ranges
+        if (hostname.startsWith('192.168.') ||
+            hostname.startsWith('10.') ||
+            hostname.startsWith('172.16.')) {
+            return false;
+        }
+        // Only allow http/https
+        return url.startsWith('http://') || url.startsWith('https://');
+    }
+    catch {
+        return false;
+    }
+}
+// ============================================================================
+// Query Validation
+// ============================================================================
+/**
+ * Validates search query for security and length limits.
+ */
+export function isValidQuery(query) {
+    if (!query || typeof query !== 'string') {
+        return { valid: false, error: 'Query must be a non-empty string' };
+    }
+    const trimmed = query.trim();
+    if (trimmed.length === 0) {
+        return { valid: false, error: 'Query cannot be empty' };
+    }
+    if (trimmed.length > 500) {
+        return { valid: false, error: 'Query too long (maximum 500 characters)' };
+    }
+    return { valid: true };
+}
+// ============================================================================
+// Number Range Validation
+// ============================================================================
+/**
+ * Clamps a number between min and max values.
+ */
+export function clamp(value, min, max) {
+    return Math.min(Math.max(value, min), max);
+}
+// ============================================================================
+// URL List Validation
+// ============================================================================
+/**
+ * Validates an array of URLs, returning only valid URLs.
+ */
+export function validateUrls(urls) {
+    if (!Array.isArray(urls))
+        return [];
+    return urls.filter(url => isValidUrl(url));
+}
+//# sourceMappingURL=validators.js.map

package/dist/utils/validators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validators.js","sourceRoot":"","sources":["../../src/utils/validators.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAE5B,+EAA+E;AAC/E,mCAAmC;AACnC,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,IAAI,QAAQ,KAAK,WAAW;YACxB,QAAQ,KAAK,WAAW;YACxB,QAAQ,KAAK,SAAS;YACtB,QAAQ,KAAK,OAAO;YACpB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC;YAC1B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,wBAAwB;QACxB,OAAO,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,yCAAyC,EAAE,CAAC;IAC5E,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,+EAA+E;AAC/E,0BAA0B;AAC1B,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IAC3D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC"}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@ignidor/web-search-mcp",
+  "version": "1.0.0",
+  "description": "Local, unlimited web-search MCP server with BM25 ranking and Playwright crawling. No Docker, no API keys, no rate limits.",
+  "private": false,
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "ignidor-web-search-mcp": "./bin/web-search-mcp.js"
+  },
+  "files": [
+    "dist",
+    "bin",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "npm run clean && npm run build",
+    "postpack": "echo 'Package created successfully. Install with: npx @ignidor/web-search-mcp'",
+    "test": "echo \"Tests not yet implemented\" && exit 0",
+    "lint": "echo \"Linting not yet implemented\" && exit 0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "mcp",
+    "web-search",
+    "playwright",
+    "bm25",
+    "search-engine",
+    "claude",
+    "ai",
+    "content-extraction",
+    "crawler",
+    "duckduckgo",
+    "unlimited",
+    "free",
+    "no-api-key"
+  ],
+  "author": "Ignidor Team",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/JayaBigDataIsCool/ignidor-web-search-mcp.git"
+  },
+  "bugs": {
+    "url": "https://github.com/JayaBigDataIsCool/ignidor-web-search-mcp/issues"
+  },
+  "homepage": "https://github.com/JayaBigDataIsCool/ignidor-web-search-mcp#readme",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.18.1",
+    "@types/cheerio": "^0.22.35",
+    "axios": "^1.7.9",
+    "cheerio": "^1.0.0",
+    "fast-bm25": "^0.0.5",
+    "p-limit": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  },
+  "peerDependencies": {
+    "playwright": "^1.40.0"
+  },
+  "peerDependenciesMeta": {
+    "playwright": {
+      "optional": true
+    }
+  },
+  "optionalDependencies": {
+    "playwright": "^1.57.0"
+  }
+}