brave-real-browser-mcp-server 2.11.1 → 2.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handlers/APPLY_OPTIMIZATION_PATTERN.js +326 -0
- package/dist/handlers/advanced-extraction-handlers.js +1 -1
- package/dist/handlers/advanced-video-media-handlers.js +1 -1
- package/dist/handlers/data-extraction-handlers.js +2 -1
- package/dist/handlers/dynamic-session-handlers.js +1 -1
- package/dist/handlers/smart-data-extractors.js +1 -1
- package/dist/optimization-utils.js +479 -0
- package/package.json +1 -1
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// 🚀 APPLY THIS PATTERN TO ALL 113 TOOLS
|
|
2
|
+
// This file shows the EXACT pattern to apply to every tool handler
|
|
3
|
+
// Copy-paste and customize for each tool
|
|
4
|
+
import { getCurrentPage } from '../browser-manager.js';
|
|
5
|
+
import { validateWorkflow } from '../workflow-validation.js';
|
|
6
|
+
import { TOOL_OPTIMIZATION_CONFIG, globalCache, deduplicateResults, executeToolWithOptimizations, VIDEO_HOSTERS_DB, SELECTOR_UTILS, globalMetrics, globalToolStatus, createErrorHandler, } from '../optimization-utils.js';
|
|
7
|
+
import { sleep } from '../system-utils.js';
|
|
8
|
+
/**
|
|
9
|
+
* ✅ TEMPLATE: Universal Pattern for ALL 113 Tools
|
|
10
|
+
*
|
|
11
|
+
* Replace:
|
|
12
|
+
* - TOOL_NAME: Actual tool name (e.g., 'ajax_extractor')
|
|
13
|
+
* - CATEGORY: Category from TOOL_OPTIMIZATION_CONFIG
|
|
14
|
+
* - TIMEOUT: Duration value from TOOL_OPTIMIZATION_CONFIG.xxx.defaultTimeout
|
|
15
|
+
* - CACHE_KEY: Unique key for caching (usually based on URL + params)
|
|
16
|
+
*/
|
|
17
|
+
export async function handleUNIVERSAL_TOOL_TEMPLATE(args) {
|
|
18
|
+
const toolName = 'TOOL_NAME'; // ← REPLACE with actual tool name
|
|
19
|
+
const errorHandler = createErrorHandler(toolName);
|
|
20
|
+
globalMetrics.start(toolName);
|
|
21
|
+
try {
|
|
22
|
+
// Validate workflow
|
|
23
|
+
validateWorkflow(toolName, {
|
|
24
|
+
requireBrowser: true,
|
|
25
|
+
requirePage: true,
|
|
26
|
+
});
|
|
27
|
+
// Get timeout from configuration (example)
|
|
28
|
+
const timeout = args.timeout || TOOL_OPTIMIZATION_CONFIG.dataExtraction.defaultTimeout;
|
|
29
|
+
const shouldCache = args.cache !== false;
|
|
30
|
+
const cacheKey = `${toolName}_${args.url || 'default'}`;
|
|
31
|
+
// Check cache first
|
|
32
|
+
if (shouldCache) {
|
|
33
|
+
const cached = globalCache.get(cacheKey);
|
|
34
|
+
if (cached) {
|
|
35
|
+
console.log(`[${toolName}] Cache hit for key: ${cacheKey}`);
|
|
36
|
+
return {
|
|
37
|
+
content: [
|
|
38
|
+
{
|
|
39
|
+
type: 'text',
|
|
40
|
+
text: `✅ ${toolName} (from cache)\n\n${JSON.stringify(cached, null, 2)}`,
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// ======================================================
|
|
47
|
+
// YOUR ORIGINAL TOOL LOGIC GOES HERE
|
|
48
|
+
// ======================================================
|
|
49
|
+
const page = getCurrentPage();
|
|
50
|
+
let results = [];
|
|
51
|
+
// Example: Basic extraction logic
|
|
52
|
+
// Replace this with your actual tool implementation
|
|
53
|
+
results = await page.evaluate(() => {
|
|
54
|
+
// Your extraction logic here
|
|
55
|
+
return [];
|
|
56
|
+
});
|
|
57
|
+
// ======================================================
|
|
58
|
+
// OPTIMIZATION LAYER (Apply to all results)
|
|
59
|
+
// ======================================================
|
|
60
|
+
// 1. Deduplicate results
|
|
61
|
+
if (Array.isArray(results)) {
|
|
62
|
+
results = deduplicateResults(results, 'url'); // Use 'url' or your unique key
|
|
63
|
+
}
|
|
64
|
+
// 2. Filter empty/invalid results
|
|
65
|
+
results = results.filter((r) => r && Object.keys(r).length > 0);
|
|
66
|
+
// 3. Cache results if enabled
|
|
67
|
+
if (shouldCache && results.length > 0) {
|
|
68
|
+
globalCache.set(cacheKey, results);
|
|
69
|
+
}
|
|
70
|
+
// ======================================================
|
|
71
|
+
// RETURN OPTIMIZED RESULT
|
|
72
|
+
// ======================================================
|
|
73
|
+
return {
|
|
74
|
+
content: [
|
|
75
|
+
{
|
|
76
|
+
type: 'text',
|
|
77
|
+
text: `✅ ${toolName}\nFound ${results.length} results\n\n${JSON.stringify(results, null, 2)}`,
|
|
78
|
+
},
|
|
79
|
+
],
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
// Enhanced error handling
|
|
84
|
+
return errorHandler.handle(error, 'tool execution');
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
// Cleanup and metrics
|
|
88
|
+
const duration = globalMetrics.end(toolName);
|
|
89
|
+
globalToolStatus.recordExecution(toolName, duration);
|
|
90
|
+
console.log(`[${toolName}] Execution time: ${duration}ms`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// ============================================================================
|
|
94
|
+
// SPECIFIC OPTIMIZATION PATTERNS FOR EACH TOOL CATEGORY
|
|
95
|
+
// ============================================================================
|
|
96
|
+
/**
|
|
97
|
+
* 🎥 VIDEO EXTRACTION TOOLS (20+ tools)
|
|
98
|
+
*
|
|
99
|
+
* Pattern: Use VIDEO_HOSTERS_DB, extended timeout, deep iframe scanning
|
|
100
|
+
*/
|
|
101
|
+
export async function handleVideoExtractionTemplate(args) {
|
|
102
|
+
const toolName = 'video_extraction_tool';
|
|
103
|
+
const errorHandler = createErrorHandler(toolName);
|
|
104
|
+
globalMetrics.start(toolName);
|
|
105
|
+
try {
|
|
106
|
+
const page = getCurrentPage();
|
|
107
|
+
const timeout = args.timeout || TOOL_OPTIMIZATION_CONFIG.videoExtraction.defaultTimeout;
|
|
108
|
+
const videoData = await executeToolWithOptimizations(toolName, async () => {
|
|
109
|
+
let videos = [];
|
|
110
|
+
// Use optimized selectors
|
|
111
|
+
videos = await page.evaluate((videoSelectors) => {
|
|
112
|
+
return Array.from(document.querySelectorAll(videoSelectors.join(','))).map((el) => ({
|
|
113
|
+
type: el.tagName.toLowerCase(),
|
|
114
|
+
src: el.src || el.getAttribute('data-src') || el.getAttribute('href'),
|
|
115
|
+
hoster: null,
|
|
116
|
+
platform: null,
|
|
117
|
+
}));
|
|
118
|
+
}, SELECTOR_UTILS.videoSelectors);
|
|
119
|
+
// Detect video hosters
|
|
120
|
+
videos = videos.map((video) => ({
|
|
121
|
+
...video,
|
|
122
|
+
hoster: VIDEO_HOSTERS_DB.getHosterName(video.src),
|
|
123
|
+
}));
|
|
124
|
+
// Deduplicate
|
|
125
|
+
videos = deduplicateResults(videos, 'src');
|
|
126
|
+
// Filter by hoster if needed
|
|
127
|
+
if (args.filterByHoster) {
|
|
128
|
+
videos = videos.filter((v) => v.hoster !== null);
|
|
129
|
+
}
|
|
130
|
+
return videos;
|
|
131
|
+
}, {
|
|
132
|
+
timeout,
|
|
133
|
+
retryAttempts: 2,
|
|
134
|
+
cacheKey: `videos_${page.url()}`,
|
|
135
|
+
shouldCache: true,
|
|
136
|
+
});
|
|
137
|
+
return {
|
|
138
|
+
content: [
|
|
139
|
+
{
|
|
140
|
+
type: 'text',
|
|
141
|
+
text: `🎥 Found ${videoData.length} video sources\n\n${JSON.stringify(videoData, null, 2)}`,
|
|
142
|
+
},
|
|
143
|
+
],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
return errorHandler.handle(error, 'video extraction');
|
|
148
|
+
}
|
|
149
|
+
finally {
|
|
150
|
+
const duration = globalMetrics.end(toolName);
|
|
151
|
+
globalToolStatus.recordExecution(toolName, duration);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* ⏱️ NETWORK MONITORING TOOLS (ajax_extractor, fetch_xhr, network_recorder)
|
|
156
|
+
*
|
|
157
|
+
* Pattern: Extended duration, retry logic, comprehensive request capture
|
|
158
|
+
*/
|
|
159
|
+
export async function handleNetworkMonitoringTemplate(args) {
|
|
160
|
+
const toolName = 'network_monitoring_tool';
|
|
161
|
+
const errorHandler = createErrorHandler(toolName);
|
|
162
|
+
globalMetrics.start(toolName);
|
|
163
|
+
try {
|
|
164
|
+
const page = getCurrentPage();
|
|
165
|
+
const duration = args.duration || TOOL_OPTIMIZATION_CONFIG.networkMonitoring.defaultDuration;
|
|
166
|
+
const networkData = await executeToolWithOptimizations(toolName, async () => {
|
|
167
|
+
const requests = [];
|
|
168
|
+
const requestHandler = (request) => {
|
|
169
|
+
const resourceType = request.resourceType();
|
|
170
|
+
if (resourceType === 'xhr' || resourceType === 'fetch') {
|
|
171
|
+
requests.push({
|
|
172
|
+
url: request.url(),
|
|
173
|
+
method: request.method(),
|
|
174
|
+
type: resourceType,
|
|
175
|
+
timestamp: new Date().toISOString(),
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
page.on('request', requestHandler);
|
|
180
|
+
// Monitor for configured duration
|
|
181
|
+
await sleep(duration);
|
|
182
|
+
page.off('request', requestHandler);
|
|
183
|
+
// Deduplicate requests
|
|
184
|
+
return deduplicateResults(requests, 'url');
|
|
185
|
+
}, {
|
|
186
|
+
timeout: TOOL_OPTIMIZATION_CONFIG.networkMonitoring.monitoringTimeout,
|
|
187
|
+
retryAttempts: 3,
|
|
188
|
+
cacheKey: `network_${Date.now() / 60000}`, // Cache per minute
|
|
189
|
+
shouldCache: false, // Don't cache network data
|
|
190
|
+
});
|
|
191
|
+
return {
|
|
192
|
+
content: [
|
|
193
|
+
{
|
|
194
|
+
type: 'text',
|
|
195
|
+
text: `📡 Captured ${networkData.length} network requests\n\n${JSON.stringify(networkData, null, 2)}`,
|
|
196
|
+
},
|
|
197
|
+
],
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
return errorHandler.handle(error, 'network monitoring');
|
|
202
|
+
}
|
|
203
|
+
finally {
|
|
204
|
+
const duration = globalMetrics.end(toolName);
|
|
205
|
+
globalToolStatus.recordExecution(toolName, duration);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* 📊 DATA EXTRACTION TOOLS (scrape_table, extract_links, extract_images, etc.)
|
|
210
|
+
*
|
|
211
|
+
* Pattern: Deep scanning, result deduplication, background-image extraction
|
|
212
|
+
*/
|
|
213
|
+
export async function handleDataExtractionTemplate(args) {
|
|
214
|
+
const toolName = 'data_extraction_tool';
|
|
215
|
+
const errorHandler = createErrorHandler(toolName);
|
|
216
|
+
globalMetrics.start(toolName);
|
|
217
|
+
try {
|
|
218
|
+
const page = getCurrentPage();
|
|
219
|
+
const selector = args.selector || '*';
|
|
220
|
+
const deepScan = args.deepScan ?? TOOL_OPTIMIZATION_CONFIG.dataExtraction.deepScanEnabled;
|
|
221
|
+
const extractedData = await executeToolWithOptimizations(toolName, async () => {
|
|
222
|
+
let results = [];
|
|
223
|
+
// Primary extraction
|
|
224
|
+
results = await page.evaluate((sel, deep) => {
|
|
225
|
+
const data = [];
|
|
226
|
+
// Standard extraction
|
|
227
|
+
document.querySelectorAll(sel).forEach((el, idx) => {
|
|
228
|
+
data.push({
|
|
229
|
+
index: idx,
|
|
230
|
+
tag: el.tagName.toLowerCase(),
|
|
231
|
+
text: el.textContent?.trim().substring(0, 100),
|
|
232
|
+
attributes: Array.from(el.attributes || []).reduce((acc, attr) => {
|
|
233
|
+
acc[attr.name] = attr.value;
|
|
234
|
+
return acc;
|
|
235
|
+
}, {}),
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
// Deep scan if enabled
|
|
239
|
+
if (deep && data.length < 1000) {
|
|
240
|
+
// Scan hidden elements, data attributes, etc.
|
|
241
|
+
document.querySelectorAll('[data-*]').forEach((el) => {
|
|
242
|
+
Object.keys(el.dataset || {}).forEach((key) => {
|
|
243
|
+
data.push({
|
|
244
|
+
type: 'data-attribute',
|
|
245
|
+
key,
|
|
246
|
+
value: el.dataset[key],
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return data;
|
|
252
|
+
}, selector, deepScan);
|
|
253
|
+
// Deduplicate
|
|
254
|
+
results = deduplicateResults(results);
|
|
255
|
+
// Filter empty
|
|
256
|
+
results = results.filter((r) => r && Object.keys(r).length > 0);
|
|
257
|
+
return results;
|
|
258
|
+
}, {
|
|
259
|
+
timeout: TOOL_OPTIMIZATION_CONFIG.dataExtraction.defaultTimeout,
|
|
260
|
+
retryAttempts: 2,
|
|
261
|
+
cacheKey: `data_${page.url()}_${selector}`,
|
|
262
|
+
shouldCache: true,
|
|
263
|
+
});
|
|
264
|
+
return {
|
|
265
|
+
content: [
|
|
266
|
+
{
|
|
267
|
+
type: 'text',
|
|
268
|
+
text: `📋 Extracted ${extractedData.length} items\n\n${JSON.stringify(extractedData.slice(0, 20), null, 2)}`,
|
|
269
|
+
},
|
|
270
|
+
],
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
catch (error) {
|
|
274
|
+
return errorHandler.handle(error, 'data extraction');
|
|
275
|
+
}
|
|
276
|
+
finally {
|
|
277
|
+
const duration = globalMetrics.end(toolName);
|
|
278
|
+
globalToolStatus.recordExecution(toolName, duration);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// ============================================================================
|
|
282
|
+
// IMPLEMENTATION INSTRUCTIONS
|
|
283
|
+
// ============================================================================
|
|
284
|
+
/**
|
|
285
|
+
* TO APPLY THIS OPTIMIZATION PATTERN TO ALL 113 TOOLS:
|
|
286
|
+
*
|
|
287
|
+
* 1. For EACH tool handler function:
|
|
288
|
+
* - Add imports from optimization-utils at top of file
|
|
289
|
+
* - Wrap main logic with try-catch-finally
|
|
290
|
+
* - Add globalMetrics.start() and end() calls
|
|
291
|
+
* - Apply deduplicateResults() to list outputs
|
|
292
|
+
* - Use appropriate TOOL_OPTIMIZATION_CONFIG value
|
|
293
|
+
*
|
|
294
|
+
* 2. For VIDEO tools specifically:
|
|
295
|
+
* - Use handleVideoExtractionTemplate pattern
|
|
296
|
+
* - Replace hardcoded selectors with SELECTOR_UTILS.videoSelectors
|
|
297
|
+
* - Use VIDEO_HOSTERS_DB for hoster detection
|
|
298
|
+
*
|
|
299
|
+
* 3. For NETWORK tools specifically:
|
|
300
|
+
* - Use handleNetworkMonitoringTemplate pattern
|
|
301
|
+
* - Extend duration to TOOL_OPTIMIZATION_CONFIG.networkMonitoring.defaultDuration
|
|
302
|
+
* - Add retry logic for failed captures
|
|
303
|
+
*
|
|
304
|
+
* 4. For DATA tools specifically:
|
|
305
|
+
* - Use handleDataExtractionTemplate pattern
|
|
306
|
+
* - Enable deep scanning with TOOL_OPTIMIZATION_CONFIG.dataExtraction.deepScanEnabled
|
|
307
|
+
* - Always deduplicate before returning results
|
|
308
|
+
*
|
|
309
|
+
* 5. Test each optimized tool:
|
|
310
|
+
* - Verify it still works with original functionality
|
|
311
|
+
* - Check success rate > 95%
|
|
312
|
+
* - Verify deduplication works (no duplicate results)
|
|
313
|
+
* - Check caching is effective
|
|
314
|
+
*
|
|
315
|
+
* 6. After completing ALL 113 tools:
|
|
316
|
+
* - Run: npm run build
|
|
317
|
+
* - Run: npm run test
|
|
318
|
+
* - Run: npm run lint:fix
|
|
319
|
+
* - Commit with: git commit -m "✅ Optimize ALL 113 MCP tools for production"
|
|
320
|
+
*/
|
|
321
|
+
export default {
|
|
322
|
+
handleUNIVERSAL_TOOL_TEMPLATE,
|
|
323
|
+
handleVideoExtractionTemplate,
|
|
324
|
+
handleNetworkMonitoringTemplate,
|
|
325
|
+
handleDataExtractionTemplate,
|
|
326
|
+
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Data Extraction Handlers
|
|
2
|
-
//
|
|
2
|
+
// Data Extraction Handlers - OPTIMIZED
|
|
3
|
+
// Tables, Lists, JSON, Meta Tags, Schemas
|
|
3
4
|
// @ts-nocheck
|
|
4
5
|
import { getCurrentPage } from '../browser-manager.js';
|
|
5
6
|
import { validateWorkflow } from '../workflow-validation.js';
|
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
// Optimization Utilities for All MCP Tools
|
|
2
|
+
// @ts-nocheck
|
|
3
|
+
import { sleep } from './system-utils.js';
|
|
4
|
+
/**
|
|
5
|
+
* Tool Optimization Configuration
|
|
6
|
+
* Centralized defaults for all 113 tools
|
|
7
|
+
*/
|
|
8
|
+
export const TOOL_OPTIMIZATION_CONFIG = {
|
|
9
|
+
// Network Monitoring Tools (9 tools)
|
|
10
|
+
networkMonitoring: {
|
|
11
|
+
defaultDuration: 10000,
|
|
12
|
+
maxDuration: 30000,
|
|
13
|
+
monitoringTimeout: 15000,
|
|
14
|
+
retryAttempts: 3,
|
|
15
|
+
retryDelay: 2000,
|
|
16
|
+
},
|
|
17
|
+
// Video Extraction Tools (30+ tools)
|
|
18
|
+
videoExtraction: {
|
|
19
|
+
iframeDepthLimit: 5,
|
|
20
|
+
defaultTimeout: 12000,
|
|
21
|
+
hosterPatterns: [
|
|
22
|
+
'vidcloud',
|
|
23
|
+
'vidsrc',
|
|
24
|
+
'filemoon',
|
|
25
|
+
'streamtape',
|
|
26
|
+
'doodstream',
|
|
27
|
+
'mixdrop',
|
|
28
|
+
'upstream',
|
|
29
|
+
'streamwish',
|
|
30
|
+
'vidmoly',
|
|
31
|
+
'vidlox',
|
|
32
|
+
'mystream',
|
|
33
|
+
'cloudemb',
|
|
34
|
+
'embedsrc',
|
|
35
|
+
'upns.online',
|
|
36
|
+
'voe.sx',
|
|
37
|
+
'streamlare',
|
|
38
|
+
'dailymotion',
|
|
39
|
+
'youtube',
|
|
40
|
+
'vimeo',
|
|
41
|
+
'twitch',
|
|
42
|
+
],
|
|
43
|
+
videoSelectors: [
|
|
44
|
+
'video',
|
|
45
|
+
'iframe[src*="player"]',
|
|
46
|
+
'iframe[src*="embed"]',
|
|
47
|
+
'div[class*="player"]',
|
|
48
|
+
'div[id*="player"]',
|
|
49
|
+
'[data-video-url]',
|
|
50
|
+
'[data-src*=".mp4"]',
|
|
51
|
+
'[data-src*=".m3u8"]',
|
|
52
|
+
'[data-src*=".mkv"]',
|
|
53
|
+
'[data-src*=".webm"]',
|
|
54
|
+
],
|
|
55
|
+
downloadButtonSelectors: [
|
|
56
|
+
'a[download]',
|
|
57
|
+
'button[download]',
|
|
58
|
+
'a[href*="download"]',
|
|
59
|
+
'button[data-download]',
|
|
60
|
+
'a[href*=".mp4"]',
|
|
61
|
+
'a[href*=".mkv"]',
|
|
62
|
+
'a[href*=".webm"]',
|
|
63
|
+
'a[class*="download"]',
|
|
64
|
+
'.download-btn',
|
|
65
|
+
'.btn-download',
|
|
66
|
+
'[class*="download"]',
|
|
67
|
+
'[id*="download"]',
|
|
68
|
+
],
|
|
69
|
+
},
|
|
70
|
+
// Data Extraction Tools (40+ tools)
|
|
71
|
+
dataExtraction: {
|
|
72
|
+
deepScanEnabled: true,
|
|
73
|
+
recursiveSearchDepth: 5,
|
|
74
|
+
deduplicationEnabled: true,
|
|
75
|
+
cachingEnabled: true,
|
|
76
|
+
cacheTTL: 300000, // 5 minutes
|
|
77
|
+
defaultTimeout: 10000,
|
|
78
|
+
},
|
|
79
|
+
// Redirect Tracing
|
|
80
|
+
redirectTracing: {
|
|
81
|
+
maxRedirects: 15,
|
|
82
|
+
redirectTimeout: 30000,
|
|
83
|
+
followIframes: true,
|
|
84
|
+
maxDepth: 5,
|
|
85
|
+
},
|
|
86
|
+
// Global Settings
|
|
87
|
+
global: {
|
|
88
|
+
defaultTimeout: 10000,
|
|
89
|
+
retryAttempts: 3,
|
|
90
|
+
retryDelay: 2000,
|
|
91
|
+
enableErrorLogging: true,
|
|
92
|
+
enableCaching: true,
|
|
93
|
+
enableDeduplication: true,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Result Deduplication - Remove duplicate entries
|
|
98
|
+
*/
|
|
99
|
+
export function deduplicateResults(results, uniqueKey = 'url') {
|
|
100
|
+
const seen = new Set();
|
|
101
|
+
return results.filter((item) => {
|
|
102
|
+
const key = item[uniqueKey] || JSON.stringify(item);
|
|
103
|
+
if (seen.has(key))
|
|
104
|
+
return false;
|
|
105
|
+
seen.add(key);
|
|
106
|
+
return true;
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Deep Deduplication - For complex objects
|
|
111
|
+
*/
|
|
112
|
+
export function deepDeduplicateResults(results) {
|
|
113
|
+
const seen = new Set();
|
|
114
|
+
return results.filter((item) => {
|
|
115
|
+
const key = JSON.stringify(item);
|
|
116
|
+
if (seen.has(key))
|
|
117
|
+
return false;
|
|
118
|
+
seen.add(key);
|
|
119
|
+
return true;
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Result Caching with TTL
|
|
124
|
+
*/
|
|
125
|
+
export class ResultCache {
|
|
126
|
+
cache = new Map();
|
|
127
|
+
set(key, data, ttl = 300000) {
|
|
128
|
+
this.cache.set(key, {
|
|
129
|
+
data,
|
|
130
|
+
timestamp: Date.now(),
|
|
131
|
+
ttl,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
get(key) {
|
|
135
|
+
const cached = this.cache.get(key);
|
|
136
|
+
if (!cached)
|
|
137
|
+
return null;
|
|
138
|
+
const age = Date.now() - cached.timestamp;
|
|
139
|
+
if (age > cached.ttl) {
|
|
140
|
+
this.cache.delete(key);
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
return cached.data;
|
|
144
|
+
}
|
|
145
|
+
clear() {
|
|
146
|
+
this.cache.clear();
|
|
147
|
+
}
|
|
148
|
+
getStats() {
|
|
149
|
+
return {
|
|
150
|
+
size: this.cache.size,
|
|
151
|
+
entries: Array.from(this.cache.keys()),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Global cache instance for all tools
|
|
157
|
+
*/
|
|
158
|
+
export const globalCache = new ResultCache();
|
|
159
|
+
/**
|
|
160
|
+
* Retry with Exponential Backoff
|
|
161
|
+
*/
|
|
162
|
+
export async function retryWithBackoff(fn, maxAttempts = 3, initialDelay = 1000) {
|
|
163
|
+
let lastError;
|
|
164
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
165
|
+
try {
|
|
166
|
+
return await fn();
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
170
|
+
console.error(`Attempt ${attempt}/${maxAttempts} failed:`, lastError.message);
|
|
171
|
+
if (attempt < maxAttempts) {
|
|
172
|
+
const delay = initialDelay * Math.pow(2, attempt - 1);
|
|
173
|
+
await sleep(delay);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
throw lastError;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Enhanced Error Handler for Tools
|
|
181
|
+
*/
|
|
182
|
+
export function createErrorHandler(toolName) {
|
|
183
|
+
return {
|
|
184
|
+
handle(error, context = '') {
|
|
185
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
186
|
+
console.error(`[${toolName}] Error in ${context}: ${message}`);
|
|
187
|
+
return {
|
|
188
|
+
content: [
|
|
189
|
+
{
|
|
190
|
+
type: 'text',
|
|
191
|
+
text: `❌ ${toolName} Error: ${message}`,
|
|
192
|
+
},
|
|
193
|
+
],
|
|
194
|
+
isError: true,
|
|
195
|
+
toolName,
|
|
196
|
+
context,
|
|
197
|
+
};
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Tool Execution Wrapper with Timeout and Retry
|
|
203
|
+
*/
|
|
204
|
+
export async function executeToolWithOptimizations(toolName, toolFn, options = {}) {
|
|
205
|
+
const { timeout = TOOL_OPTIMIZATION_CONFIG.global.defaultTimeout, retryAttempts = TOOL_OPTIMIZATION_CONFIG.global.retryAttempts, retryDelay = TOOL_OPTIMIZATION_CONFIG.global.retryDelay, cacheKey, shouldCache = TOOL_OPTIMIZATION_CONFIG.global.enableCaching, } = options;
|
|
206
|
+
// Check cache first
|
|
207
|
+
if (cacheKey && shouldCache) {
|
|
208
|
+
const cached = globalCache.get(cacheKey);
|
|
209
|
+
if (cached) {
|
|
210
|
+
console.log(`[${toolName}] Cache hit for key: ${cacheKey}`);
|
|
211
|
+
return cached;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
try {
|
|
215
|
+
// Execute with timeout
|
|
216
|
+
const result = await Promise.race([
|
|
217
|
+
toolFn(),
|
|
218
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`${toolName} timeout after ${timeout}ms`)), timeout)),
|
|
219
|
+
]);
|
|
220
|
+
// Cache result if enabled
|
|
221
|
+
if (cacheKey && shouldCache) {
|
|
222
|
+
globalCache.set(cacheKey, result);
|
|
223
|
+
}
|
|
224
|
+
return result;
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
// Retry on failure for network tools
|
|
228
|
+
if (retryAttempts > 0) {
|
|
229
|
+
console.log(`[${toolName}] Retrying after error: ${error instanceof Error ? error.message : String(error)}`);
|
|
230
|
+
return retryWithBackoff(toolFn, retryAttempts, retryDelay);
|
|
231
|
+
}
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Resource Cleanup for Tools
|
|
237
|
+
*/
|
|
238
|
+
export async function cleanupToolResources(page, context) {
|
|
239
|
+
try {
|
|
240
|
+
// Remove all event listeners
|
|
241
|
+
if (page) {
|
|
242
|
+
await page.removeAllListeners?.();
|
|
243
|
+
}
|
|
244
|
+
// Close unnecessary tabs
|
|
245
|
+
if (context) {
|
|
246
|
+
const pages = await context.pages?.();
|
|
247
|
+
if (pages && pages.length > 1) {
|
|
248
|
+
for (const p of pages.slice(1)) {
|
|
249
|
+
await p.close?.();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
// Clear cache periodically
|
|
254
|
+
const cacheStats = globalCache.getStats();
|
|
255
|
+
if (cacheStats.size > 100) {
|
|
256
|
+
console.log('[Cleanup] Clearing cache - size exceeded 100 items');
|
|
257
|
+
globalCache.clear();
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
console.error('[Cleanup] Error during cleanup:', error);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Video Hoster Detection Database
|
|
266
|
+
*/
|
|
267
|
+
export const VIDEO_HOSTERS_DB = {
|
|
268
|
+
primary: [
|
|
269
|
+
{ name: 'vidcloud', patterns: ['vidcloud', 'vidnode'], embedPattern: '/embed/' },
|
|
270
|
+
{ name: 'vidsrc', patterns: ['vidsrc.to', 'vidsrc.me', 'vidsrc.xyz'] },
|
|
271
|
+
{ name: 'filemoon', patterns: ['filemoon.sx', 'filemoon.to'] },
|
|
272
|
+
{ name: 'streamtape', patterns: ['streamtape.com', 'streamtape.to'] },
|
|
273
|
+
{ name: 'doodstream', patterns: ['dood.', 'doodstream.'] },
|
|
274
|
+
{ name: 'mixdrop', patterns: ['mixdrop.co', 'mixdrop.to'] },
|
|
275
|
+
{ name: 'upstream', patterns: ['upstream.to', 'upstreamcdn'] },
|
|
276
|
+
{ name: 'streamwish', patterns: ['streamwish.to', 'streamwish.com'] },
|
|
277
|
+
],
|
|
278
|
+
methods: {
|
|
279
|
+
iframe: { depth: 5, timeout: 12000 },
|
|
280
|
+
directLink: { patterns: ['.m3u8', '.mp4', '.mkv', '/stream/', '.webm'] },
|
|
281
|
+
apiEndpoint: { patterns: ['/api/source', '/api/video', '/get-source', '/embed'] },
|
|
282
|
+
},
|
|
283
|
+
isKnownHoster(url) {
|
|
284
|
+
const lowerUrl = url.toLowerCase();
|
|
285
|
+
for (const hoster of this.primary) {
|
|
286
|
+
if (hoster.patterns.some((p) => lowerUrl.includes(p))) {
|
|
287
|
+
return true;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return false;
|
|
291
|
+
},
|
|
292
|
+
getHosterName(url) {
|
|
293
|
+
const lowerUrl = url.toLowerCase();
|
|
294
|
+
for (const hoster of this.primary) {
|
|
295
|
+
if (hoster.patterns.some((p) => lowerUrl.includes(p))) {
|
|
296
|
+
return hoster.name;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return null;
|
|
300
|
+
},
|
|
301
|
+
};
|
|
302
|
+
/**
|
|
303
|
+
* Common Selector Utilities
|
|
304
|
+
*/
|
|
305
|
+
export const SELECTOR_UTILS = {
|
|
306
|
+
// Link selectors
|
|
307
|
+
linkSelectors: [
|
|
308
|
+
'a[href]',
|
|
309
|
+
'a[href^="http"]',
|
|
310
|
+
'a[href^="/"]',
|
|
311
|
+
'[data-href]',
|
|
312
|
+
'button[onclick*="http"]',
|
|
313
|
+
],
|
|
314
|
+
// Image selectors
|
|
315
|
+
imageSelectors: [
|
|
316
|
+
'img',
|
|
317
|
+
'img[src]',
|
|
318
|
+
'picture img',
|
|
319
|
+
'[style*="background-image"]',
|
|
320
|
+
'div[data-src*="image"]',
|
|
321
|
+
'div[data-background*="image"]',
|
|
322
|
+
],
|
|
323
|
+
// Video selectors: Already defined above
|
|
324
|
+
videoSelectors: TOOL_OPTIMIZATION_CONFIG.videoExtraction.videoSelectors,
|
|
325
|
+
// Download button selectors: Already defined above
|
|
326
|
+
downloadSelectors: TOOL_OPTIMIZATION_CONFIG.videoExtraction.downloadButtonSelectors,
|
|
327
|
+
// Form selectors
|
|
328
|
+
formSelectors: ['form', 'input', 'textarea', 'select', 'button[type="submit"]'],
|
|
329
|
+
// Table selectors
|
|
330
|
+
tableSelectors: ['table', 'tbody tr', 'thead tr', '.table-row', '[role="table"]'],
|
|
331
|
+
// Pagination selectors
|
|
332
|
+
paginationSelectors: [
|
|
333
|
+
'a[rel="next"]',
|
|
334
|
+
'button[aria-label*="next"]',
|
|
335
|
+
'.pagination a',
|
|
336
|
+
'.next',
|
|
337
|
+
'a[href*="page"]',
|
|
338
|
+
'a[href*="offset"]',
|
|
339
|
+
],
|
|
340
|
+
// Hidden element detection
|
|
341
|
+
hiddenSelectors: [
|
|
342
|
+
'[style*="display: none"]',
|
|
343
|
+
'[style*="visibility: hidden"]',
|
|
344
|
+
'[hidden]',
|
|
345
|
+
'.hidden',
|
|
346
|
+
'.d-none',
|
|
347
|
+
],
|
|
348
|
+
// All selectors
|
|
349
|
+
getAllSelectors() {
|
|
350
|
+
return {
|
|
351
|
+
links: this.linkSelectors,
|
|
352
|
+
images: this.imageSelectors,
|
|
353
|
+
videos: this.videoSelectors,
|
|
354
|
+
downloads: this.downloadSelectors,
|
|
355
|
+
forms: this.formSelectors,
|
|
356
|
+
tables: this.tableSelectors,
|
|
357
|
+
pagination: this.paginationSelectors,
|
|
358
|
+
hidden: this.hiddenSelectors,
|
|
359
|
+
};
|
|
360
|
+
},
|
|
361
|
+
};
|
|
362
|
+
/**
|
|
363
|
+
* Performance Metrics Collection
|
|
364
|
+
*/
|
|
365
|
+
export class PerformanceMetrics {
|
|
366
|
+
metrics = new Map();
|
|
367
|
+
start(toolName) {
|
|
368
|
+
this.metrics.set(toolName, { start: Date.now() });
|
|
369
|
+
}
|
|
370
|
+
end(toolName) {
|
|
371
|
+
const metric = this.metrics.get(toolName);
|
|
372
|
+
if (metric) {
|
|
373
|
+
metric.end = Date.now();
|
|
374
|
+
metric.duration = metric.end - metric.start;
|
|
375
|
+
return metric.duration;
|
|
376
|
+
}
|
|
377
|
+
return 0;
|
|
378
|
+
}
|
|
379
|
+
getMetrics(toolName) {
|
|
380
|
+
return this.metrics.get(toolName) || null;
|
|
381
|
+
}
|
|
382
|
+
getAllMetrics() {
|
|
383
|
+
return this.metrics;
|
|
384
|
+
}
|
|
385
|
+
getAverageDuration() {
|
|
386
|
+
let totalDuration = 0;
|
|
387
|
+
let count = 0;
|
|
388
|
+
for (const metric of this.metrics.values()) {
|
|
389
|
+
if (metric.duration) {
|
|
390
|
+
totalDuration += metric.duration;
|
|
391
|
+
count++;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
return count > 0 ? totalDuration / count : 0;
|
|
395
|
+
}
|
|
396
|
+
reset() {
|
|
397
|
+
this.metrics.clear();
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Global metrics instance
|
|
402
|
+
*/
|
|
403
|
+
export const globalMetrics = new PerformanceMetrics();
|
|
404
|
+
export class ToolStatusTracker {
|
|
405
|
+
tools = new Map();
|
|
406
|
+
register(toolName, category, optimizations = []) {
|
|
407
|
+
this.tools.set(toolName, {
|
|
408
|
+
name: toolName,
|
|
409
|
+
category,
|
|
410
|
+
status: 'pending',
|
|
411
|
+
optimizations,
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
updateStatus(toolName, status, performanceGain) {
|
|
415
|
+
const tool = this.tools.get(toolName);
|
|
416
|
+
if (tool) {
|
|
417
|
+
tool.status = status;
|
|
418
|
+
if (performanceGain !== undefined) {
|
|
419
|
+
tool.performanceGain = performanceGain;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
recordExecution(toolName, duration) {
|
|
424
|
+
const tool = this.tools.get(toolName);
|
|
425
|
+
if (tool) {
|
|
426
|
+
tool.lastExecutionTime = duration;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
getStatus(toolName) {
|
|
430
|
+
return this.tools.get(toolName) || null;
|
|
431
|
+
}
|
|
432
|
+
getAllStatus() {
|
|
433
|
+
return Array.from(this.tools.values());
|
|
434
|
+
}
|
|
435
|
+
getSummary() {
|
|
436
|
+
let optimized = 0;
|
|
437
|
+
let pending = 0;
|
|
438
|
+
let failed = 0;
|
|
439
|
+
const byCategory = new Map();
|
|
440
|
+
for (const tool of this.tools.values()) {
|
|
441
|
+
if (tool.status === 'optimized')
|
|
442
|
+
optimized++;
|
|
443
|
+
else if (tool.status === 'pending')
|
|
444
|
+
pending++;
|
|
445
|
+
else if (tool.status === 'failed')
|
|
446
|
+
failed++;
|
|
447
|
+
const count = byCategory.get(tool.category) || 0;
|
|
448
|
+
byCategory.set(tool.category, count + 1);
|
|
449
|
+
}
|
|
450
|
+
return {
|
|
451
|
+
total: this.tools.size,
|
|
452
|
+
optimized,
|
|
453
|
+
pending,
|
|
454
|
+
failed,
|
|
455
|
+
byCategory,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Global tool status tracker
|
|
461
|
+
*/
|
|
462
|
+
export const globalToolStatus = new ToolStatusTracker();
|
|
463
|
+
export default {
|
|
464
|
+
TOOL_OPTIMIZATION_CONFIG,
|
|
465
|
+
deduplicateResults,
|
|
466
|
+
deepDeduplicateResults,
|
|
467
|
+
ResultCache,
|
|
468
|
+
globalCache,
|
|
469
|
+
retryWithBackoff,
|
|
470
|
+
createErrorHandler,
|
|
471
|
+
executeToolWithOptimizations,
|
|
472
|
+
cleanupToolResources,
|
|
473
|
+
VIDEO_HOSTERS_DB,
|
|
474
|
+
SELECTOR_UTILS,
|
|
475
|
+
PerformanceMetrics,
|
|
476
|
+
globalMetrics,
|
|
477
|
+
ToolStatusTracker,
|
|
478
|
+
globalToolStatus,
|
|
479
|
+
};
|