@purepageio/fetch-engines 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +205 -0
- package/dist/FetchEngine.d.ts +46 -0
- package/dist/FetchEngine.d.ts.map +1 -0
- package/dist/FetchEngine.js +137 -0
- package/dist/FetchEngine.js.map +1 -0
- package/dist/FetchEngine.test.d.ts +2 -0
- package/dist/FetchEngine.test.d.ts.map +1 -0
- package/dist/FetchEngine.test.js +44 -0
- package/dist/FetchEngine.test.js.map +1 -0
- package/dist/HybridEngine.d.ts +15 -0
- package/dist/HybridEngine.d.ts.map +1 -0
- package/dist/HybridEngine.js +45 -0
- package/dist/HybridEngine.js.map +1 -0
- package/dist/IEngine.d.ts +22 -0
- package/dist/IEngine.d.ts.map +1 -0
- package/dist/IEngine.js +2 -0
- package/dist/IEngine.js.map +1 -0
- package/dist/PlaywrightEngine.d.ts +88 -0
- package/dist/PlaywrightEngine.d.ts.map +1 -0
- package/dist/PlaywrightEngine.js +484 -0
- package/dist/PlaywrightEngine.js.map +1 -0
- package/dist/PlaywrightEngine.test.d.ts +2 -0
- package/dist/PlaywrightEngine.test.d.ts.map +1 -0
- package/dist/PlaywrightEngine.test.js +299 -0
- package/dist/PlaywrightEngine.test.js.map +1 -0
- package/dist/PuppeteerEngine.d.ts +21 -0
- package/dist/PuppeteerEngine.d.ts.map +1 -0
- package/dist/PuppeteerEngine.js +412 -0
- package/dist/PuppeteerEngine.js.map +1 -0
- package/dist/browser/BrowserPool.d.ts +29 -0
- package/dist/browser/BrowserPool.d.ts.map +1 -0
- package/dist/browser/BrowserPool.js +378 -0
- package/dist/browser/BrowserPool.js.map +1 -0
- package/dist/browser/PlaywrightBrowserPool.d.ts +78 -0
- package/dist/browser/PlaywrightBrowserPool.d.ts.map +1 -0
- package/dist/browser/PlaywrightBrowserPool.js +429 -0
- package/dist/browser/PlaywrightBrowserPool.js.map +1 -0
- package/dist/browser/PlaywrightBrowserPool.test.d.ts +2 -0
- package/dist/browser/PlaywrightBrowserPool.test.d.ts.map +1 -0
- package/dist/browser/PlaywrightBrowserPool.test.js +422 -0
- package/dist/browser/PlaywrightBrowserPool.test.js.map +1 -0
- package/dist/errors.d.ts +20 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +30 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/types.d.ts +151 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +72 -0
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.PuppeteerEngine = void 0;
|
|
7
|
+
const BrowserPool_1 = require("./browser/BrowserPool");
|
|
8
|
+
const p_queue_1 = __importDefault(require("p-queue"));
|
|
9
|
+
function delay(time) {
|
|
10
|
+
// Added return type
|
|
11
|
+
return new Promise((resolve) => setTimeout(resolve, time));
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* PuppeteerEngine - A headless browser engine that uses Puppeteer to render JavaScript-heavy pages
|
|
15
|
+
* Ideal for SPAs and sites with anti-scraping measures
|
|
16
|
+
*/
|
|
17
|
+
class PuppeteerEngine {
|
|
18
|
+
constructor(concurrentPages = 3) {
|
|
19
|
+
this.cache = new Map();
|
|
20
|
+
this.cacheTTL = 15 * 60 * 1000; // 15 minutes cache TTL
|
|
21
|
+
this.isInitializing = false; // Flag to prevent concurrent initialization
|
|
22
|
+
this.queue = new p_queue_1.default({ concurrency: concurrentPages });
|
|
23
|
+
// Initialize pool lazily on first fetchHTML call instead of constructor
|
|
24
|
+
// this.initializeBrowserPool(); // Removed from constructor
|
|
25
|
+
// Graceful shutdown handling
|
|
26
|
+
const cleanup = async () => {
|
|
27
|
+
console.log("Shutting down PuppeteerEngine...");
|
|
28
|
+
await this.cleanup();
|
|
29
|
+
// Let the calling process handle exit
|
|
30
|
+
// process.exit(0); // Removed process.exit
|
|
31
|
+
};
|
|
32
|
+
// Consider removing these global handlers if the engine is used as a library
|
|
33
|
+
// The application using the library should manage its lifecycle.
|
|
34
|
+
process.on("SIGTERM", cleanup);
|
|
35
|
+
process.on("SIGINT", cleanup);
|
|
36
|
+
}
|
|
37
|
+
async initializeBrowserPool() {
|
|
38
|
+
// Made async, added return type
|
|
39
|
+
if (PuppeteerEngine.browserPool || this.isInitializing) {
|
|
40
|
+
// Wait if initialization is already in progress
|
|
41
|
+
while (this.isInitializing) {
|
|
42
|
+
await delay(100);
|
|
43
|
+
}
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
this.isInitializing = true;
|
|
47
|
+
try {
|
|
48
|
+
console.log("Initializing Puppeteer Browser Pool...");
|
|
49
|
+
PuppeteerEngine.browserPool = new BrowserPool_1.BrowserPool(2, // maxBrowsers
|
|
50
|
+
6, // maxPagesPerBrowser
|
|
51
|
+
60 * 60 * 1000, // maxBrowserAge (1 hour)
|
|
52
|
+
60 * 1000 // healthCheckInterval (1 minute)
|
|
53
|
+
);
|
|
54
|
+
await PuppeteerEngine.browserPool.initialize();
|
|
55
|
+
console.log("Puppeteer Browser Pool initialized.");
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
console.error("Failed to initialize Puppeteer browser pool:", error);
|
|
59
|
+
PuppeteerEngine.browserPool = null; // Ensure pool is null on failure
|
|
60
|
+
throw error; // Re-throw error
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
this.isInitializing = false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async fetchHTML(url) {
|
|
67
|
+
// Check cache first
|
|
68
|
+
const cachedResult = this.checkCache(url);
|
|
69
|
+
if (cachedResult) {
|
|
70
|
+
console.log(`Cache hit for ${url}`);
|
|
71
|
+
return cachedResult;
|
|
72
|
+
}
|
|
73
|
+
// Ensure browser pool is initialized before processing
|
|
74
|
+
await this.initializeBrowserPool();
|
|
75
|
+
if (!PuppeteerEngine.browserPool) {
|
|
76
|
+
throw new Error("Browser pool is not available after initialization attempt.");
|
|
77
|
+
}
|
|
78
|
+
const fetchPromise = this.queue.add(async () => {
|
|
79
|
+
const metrics = [];
|
|
80
|
+
const totalStart = performance.now();
|
|
81
|
+
let page = null; // Initialize page as null
|
|
82
|
+
try {
|
|
83
|
+
// Fall back to Puppeteer
|
|
84
|
+
// Get page from pool
|
|
85
|
+
const pageStart = performance.now();
|
|
86
|
+
if (!PuppeteerEngine.browserPool)
|
|
87
|
+
throw new Error("Browser pool is null"); // Guard against null pool
|
|
88
|
+
page = await PuppeteerEngine.browserPool.acquirePage();
|
|
89
|
+
metrics.push({
|
|
90
|
+
operation: "acquire_page",
|
|
91
|
+
duration: performance.now() - pageStart,
|
|
92
|
+
});
|
|
93
|
+
// Pre-navigation setup
|
|
94
|
+
const setupStart = performance.now();
|
|
95
|
+
// Set a realistic user agent
|
|
96
|
+
await page.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
|
97
|
+
// Set extra headers
|
|
98
|
+
await page.setExtraHTTPHeaders({
|
|
99
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
100
|
+
Referer: "https://www.google.com/",
|
|
101
|
+
"Sec-Ch-Ua": '"Chromium";v="122", "Google Chrome";v="122", "Not(A:Brand";v="24"',
|
|
102
|
+
"Sec-Ch-Ua-Mobile": "?0",
|
|
103
|
+
"Sec-Ch-Ua-Platform": '"macOS"',
|
|
104
|
+
});
|
|
105
|
+
// Enable JavaScript
|
|
106
|
+
await page.setJavaScriptEnabled(true);
|
|
107
|
+
// Set cookies (consider making domain more specific if possible)
|
|
108
|
+
try {
|
|
109
|
+
const hostname = new URL(url).hostname;
|
|
110
|
+
const domain = hostname.startsWith("www.") ? hostname.substring(4) : hostname;
|
|
111
|
+
await page.setCookie({
|
|
112
|
+
name: "cookie_consent",
|
|
113
|
+
value: "accepted",
|
|
114
|
+
domain: `.${domain}`, // Use extracted domain
|
|
115
|
+
path: "/",
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
catch (e) {
|
|
119
|
+
console.warn(`Could not set cookie for URL ${url}: ${e instanceof Error ? e.message : String(e)}`);
|
|
120
|
+
}
|
|
121
|
+
// Adaptive request interception based on URL content guess
|
|
122
|
+
const isLikelyMediaSite = /music|record|audio|video|stream|artist|label|release|track|album/i.test(url);
|
|
123
|
+
await page.setRequestInterception(true);
|
|
124
|
+
const requestHandler = (request /* puppeteer.HTTPRequest */) => {
|
|
125
|
+
const resourceType = request.resourceType();
|
|
126
|
+
const requestUrl = request.url();
|
|
127
|
+
// Block common tracking/analytics/ads/fonts/images unless it's a media site
|
|
128
|
+
if (!isLikelyMediaSite &&
|
|
129
|
+
(["font", "image", "media", "stylesheet"].includes(resourceType) ||
|
|
130
|
+
/google-analytics\.com|googletagmanager\.com|doubleclick\.net|facebook\.net|twitter\.com/.test(requestUrl))) {
|
|
131
|
+
request.abort().catch((e) => console.warn(`Failed to abort request: ${e.message}`));
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
request.continue().catch((e) => console.warn(`Failed to continue request: ${e.message}`));
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
page.on("request", requestHandler); // Assign handler
|
|
138
|
+
await page.setDefaultNavigationTimeout(45000); // Increased timeout
|
|
139
|
+
metrics.push({
|
|
140
|
+
operation: "page_setup",
|
|
141
|
+
duration: performance.now() - setupStart,
|
|
142
|
+
});
|
|
143
|
+
// Navigation
|
|
144
|
+
const navigationStart = performance.now();
|
|
145
|
+
let response;
|
|
146
|
+
try {
|
|
147
|
+
if (isLikelyMediaSite) {
|
|
148
|
+
// Relaxed approach for media sites
|
|
149
|
+
response = await page.goto(url, {
|
|
150
|
+
waitUntil: ["domcontentloaded", "networkidle2"], // Wait for DOM and less strict network idle
|
|
151
|
+
timeout: 40000,
|
|
152
|
+
});
|
|
153
|
+
// Basic interactions to trigger content loading
|
|
154
|
+
await page
|
|
155
|
+
.evaluate(() => {
|
|
156
|
+
window.scrollTo(0, document.body.scrollHeight / 2);
|
|
157
|
+
const buttons = Array.from(document.querySelectorAll("button")).filter((b) => /cookie|accept|agree/i.test(b.textContent || ""));
|
|
158
|
+
buttons.forEach((b) => b.click());
|
|
159
|
+
})
|
|
160
|
+
.catch((e) => console.warn(`Media site interaction error: ${e.message}`));
|
|
161
|
+
await delay(3000); // Wait for potential dynamic content
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
// Cautious approach for other sites
|
|
165
|
+
response = await page.goto(url, {
|
|
166
|
+
waitUntil: "domcontentloaded", // Faster initial load
|
|
167
|
+
timeout: 30000,
|
|
168
|
+
});
|
|
169
|
+
// Check for Cloudflare/challenge pages
|
|
170
|
+
const isChallengePage = await page.evaluate(() => {
|
|
171
|
+
const title = document.title || "";
|
|
172
|
+
const bodyText = document.body?.textContent || "";
|
|
173
|
+
return (/Just a moment|Attention Required|Security check|Enable JavaScript and cookies|Please wait while we verify|Please enable Cookies|DDoS protection|Cloudflare/i.test(title + bodyText) || document.querySelector("#challenge-form, #cf-challenge-running") !== null);
|
|
174
|
+
});
|
|
175
|
+
if (isChallengePage) {
|
|
176
|
+
console.log(`Detected challenge page for ${url}, waiting...`);
|
|
177
|
+
try {
|
|
178
|
+
// Wait for title change or specific element disappearance
|
|
179
|
+
await page.waitForFunction(() => {
|
|
180
|
+
const title = document.title;
|
|
181
|
+
return (!/Just a moment|Attention Required|Security check/i.test(title) &&
|
|
182
|
+
!document.querySelector("#challenge-form, #cf-challenge-running"));
|
|
183
|
+
}, { timeout: 25000 } // Increased wait time
|
|
184
|
+
);
|
|
185
|
+
console.log("Challenge appears resolved, continuing...");
|
|
186
|
+
await delay(5000); // Extra wait after challenge resolution
|
|
187
|
+
// Refresh response object after potential navigation/reload by challenge
|
|
188
|
+
response = page.mainFrame().Mresponse(); // Get current response
|
|
189
|
+
if (!response) {
|
|
190
|
+
// Reload if no response object found
|
|
191
|
+
response = await page.reload({ waitUntil: "networkidle2", timeout: 30000 });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
catch (challengeError) {
|
|
195
|
+
console.warn(`Challenge wait timed out or failed for ${url}:`, challengeError);
|
|
196
|
+
// Try to interact with potential challenge elements as fallback
|
|
197
|
+
await page
|
|
198
|
+
.evaluate(() => {
|
|
199
|
+
document
|
|
200
|
+
.querySelectorAll('button, input[type="submit"], .cf-button, input[type="checkbox"]')
|
|
201
|
+
.forEach((el) => {
|
|
202
|
+
try {
|
|
203
|
+
el.click();
|
|
204
|
+
}
|
|
205
|
+
catch (e) { }
|
|
206
|
+
if (el.type === "checkbox")
|
|
207
|
+
el.checked = true;
|
|
208
|
+
});
|
|
209
|
+
})
|
|
210
|
+
.catch((e) => console.warn(`Challenge interaction error: ${e.message}`));
|
|
211
|
+
await delay(3000);
|
|
212
|
+
try {
|
|
213
|
+
response = await page.reload({ waitUntil: "domcontentloaded", timeout: 20000 });
|
|
214
|
+
}
|
|
215
|
+
catch (reloadError) {
|
|
216
|
+
console.warn(`Reload after challenge interaction failed for ${url}:`, reloadError);
|
|
217
|
+
// Proceed with whatever state the page is in
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
else {
|
|
222
|
+
// If not a challenge page, wait for network idle or a timeout
|
|
223
|
+
await Promise.race([
|
|
224
|
+
page.waitForNavigation({ waitUntil: "networkidle2", timeout: 15000 }).catch(() => { }),
|
|
225
|
+
delay(5000), // Min wait time
|
|
226
|
+
]);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
catch (navigationError) {
|
|
231
|
+
console.error(`Primary navigation failed for ${url}:`, navigationError);
|
|
232
|
+
const errorMsg = navigationError instanceof Error ? navigationError.message : String(navigationError);
|
|
233
|
+
// Handle frame detached errors specifically if needed, otherwise rethrow or fallback
|
|
234
|
+
if (/frame was detached|context destroyed|detached Frame/i.test(errorMsg)) {
|
|
235
|
+
console.warn(`Frame detached during navigation for ${url}, attempting final content grab.`);
|
|
236
|
+
// Attempt to grab content immediately before potentially closing the page
|
|
237
|
+
const html = await page.content().catch(() => "");
|
|
238
|
+
const title = await page.title().catch(() => "Content Unavailable");
|
|
239
|
+
if (html) {
|
|
240
|
+
return { html, title, url: page.url() };
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
throw new Error(`Navigation failed (Frame Detached) for ${url}`); // Rethrow if content grab fails
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
throw navigationError; // Rethrow other navigation errors
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
page.off("request", requestHandler); // Remove listener
|
|
252
|
+
await page.setRequestInterception(false); // Turn off interception
|
|
253
|
+
}
|
|
254
|
+
metrics.push({
|
|
255
|
+
operation: "navigation_complete",
|
|
256
|
+
duration: performance.now() - navigationStart,
|
|
257
|
+
});
|
|
258
|
+
if (!response) {
|
|
259
|
+
console.warn(`No valid response object after navigation for ${url}. Trying to get content anyway.`);
|
|
260
|
+
// Attempt to proceed even without a response object, page might still have content
|
|
261
|
+
}
|
|
262
|
+
else if (!response.ok()) {
|
|
263
|
+
console.warn(`Navigation response not OK for ${url}: Status ${response.status()}`);
|
|
264
|
+
// Decide whether to throw or try to extract content based on status code
|
|
265
|
+
if (response.status() >= 400 && response.status() < 500) {
|
|
266
|
+
// Client errors
|
|
267
|
+
throw new Error(`Client error fetching ${url}: Status ${response.status()}`);
|
|
268
|
+
}
|
|
269
|
+
// For server errors (5xx), we might still attempt content extraction
|
|
270
|
+
}
|
|
271
|
+
// Wait for potential SPA content rendering
|
|
272
|
+
const spaWaitStart = performance.now();
|
|
273
|
+
try {
|
|
274
|
+
// Wait for a combination of potential content indicators or a timeout
|
|
275
|
+
await Promise.race([
|
|
276
|
+
page
|
|
277
|
+
.waitForSelector('h1, h2, main, article, .container, [class*="content"], [id*="content"]', {
|
|
278
|
+
timeout: 15000,
|
|
279
|
+
visible: true,
|
|
280
|
+
})
|
|
281
|
+
.catch(() => { }),
|
|
282
|
+
page
|
|
283
|
+
.waitForFunction(() => (document.body?.textContent || "").length > 200, { timeout: 15000 })
|
|
284
|
+
.catch(() => { }),
|
|
285
|
+
delay(5000), // Minimum wait
|
|
286
|
+
]);
|
|
287
|
+
}
|
|
288
|
+
catch (waitError) {
|
|
289
|
+
console.warn(`SPA content wait potentially failed for ${url}:`, waitError);
|
|
290
|
+
}
|
|
291
|
+
finally {
|
|
292
|
+
metrics.push({
|
|
293
|
+
operation: "spa_content_wait",
|
|
294
|
+
duration: performance.now() - spaWaitStart,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
// Content extraction
|
|
298
|
+
const extractStart = performance.now();
|
|
299
|
+
let html, title;
|
|
300
|
+
// Generic extraction attempt focused on main content areas
|
|
301
|
+
const pageData = await page.evaluate(() => {
|
|
302
|
+
const mainContent = document.querySelector("main, article, #main-content, .main-content, #content, .content")?.innerHTML;
|
|
303
|
+
return {
|
|
304
|
+
html: mainContent || document.documentElement.outerHTML, // Fallback to full HTML
|
|
305
|
+
title: document.title || document.querySelector("h1")?.textContent || "",
|
|
306
|
+
};
|
|
307
|
+
});
|
|
308
|
+
html = pageData.html;
|
|
309
|
+
title = pageData.title;
|
|
310
|
+
// Fallback if extraction yields minimal content
|
|
311
|
+
if (!html || html.length < 150) {
|
|
312
|
+
console.warn(`Minimal content extracted for ${url}, falling back to full page content.`);
|
|
313
|
+
html = await page.content();
|
|
314
|
+
title = await page.title(); // Re-fetch title as it might have updated
|
|
315
|
+
}
|
|
316
|
+
metrics.push({
|
|
317
|
+
operation: "content_extraction",
|
|
318
|
+
duration: performance.now() - extractStart,
|
|
319
|
+
});
|
|
320
|
+
metrics.push({
|
|
321
|
+
operation: "total_time",
|
|
322
|
+
duration: performance.now() - totalStart,
|
|
323
|
+
});
|
|
324
|
+
console.log("Puppeteer fetch successful:", {
|
|
325
|
+
url,
|
|
326
|
+
status: response?.status() ?? "N/A",
|
|
327
|
+
metrics: metrics.map((m) => `${m.operation}: ${m.duration.toFixed(0)}ms`).join(", "), // Simplified metrics logging
|
|
328
|
+
});
|
|
329
|
+
const finalResult = {
|
|
330
|
+
html,
|
|
331
|
+
title,
|
|
332
|
+
url: page.url(), // Get final URL after potential redirects
|
|
333
|
+
};
|
|
334
|
+
// Cache the successful result
|
|
335
|
+
this.cacheResult(url, finalResult);
|
|
336
|
+
return finalResult;
|
|
337
|
+
}
|
|
338
|
+
catch (error) {
|
|
339
|
+
console.error(`PuppeteerEngine fetch failed for ${url}:`, error instanceof Error ? error.message : String(error), { metrics });
|
|
340
|
+
// Decide if the error is critical enough to invalidate the page/browser? Maybe not here.
|
|
341
|
+
throw error; // Re-throw the error to be handled by the caller or queue
|
|
342
|
+
}
|
|
343
|
+
finally {
|
|
344
|
+
if (page) {
|
|
345
|
+
// Ensure page is closed even if errors occurred
|
|
346
|
+
await page.close().catch((e) => console.warn(`Error closing page for ${url}: ${e.message}`));
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
// Type assertion as PQueue returns Promise<unknown> by default
|
|
351
|
+
return fetchPromise;
|
|
352
|
+
}
|
|
353
|
+
checkCache(url) {
|
|
354
|
+
const cached = this.cache.get(url);
|
|
355
|
+
if (!cached)
|
|
356
|
+
return null;
|
|
357
|
+
const now = Date.now();
|
|
358
|
+
if (now - cached.timestamp > this.cacheTTL) {
|
|
359
|
+
// Cache expired
|
|
360
|
+
this.cache.delete(url);
|
|
361
|
+
console.log(`Cache expired for ${url}`);
|
|
362
|
+
return null;
|
|
363
|
+
}
|
|
364
|
+
return cached.result;
|
|
365
|
+
}
|
|
366
|
+
cacheResult(url, result) {
|
|
367
|
+
this.cache.set(url, {
|
|
368
|
+
result,
|
|
369
|
+
timestamp: Date.now(),
|
|
370
|
+
});
|
|
371
|
+
console.log(`Cached result for ${url}`);
|
|
372
|
+
// Simple cache cleanup strategy: clean oldest if cache exceeds size limit
|
|
373
|
+
if (this.cache.size > 100) {
|
|
374
|
+
// Set a reasonable cache size limit
|
|
375
|
+
const oldestUrl = this.cache.keys().next().value;
|
|
376
|
+
if (oldestUrl) {
|
|
377
|
+
this.cache.delete(oldestUrl);
|
|
378
|
+
console.log(`Cache limit reached, removed oldest entry: ${oldestUrl}`);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
// Removed cleanupCache method as logic is now inline in cacheResult
|
|
383
|
+
async cleanup() {
|
|
384
|
+
console.log("Cleaning up PuppeteerEngine...");
|
|
385
|
+
try {
|
|
386
|
+
// Wait for queue to become idle (all active tasks finished)
|
|
387
|
+
await this.queue.onIdle();
|
|
388
|
+
// Clear any pending tasks in the queue
|
|
389
|
+
this.queue.clear();
|
|
390
|
+
console.log("Task queue cleared.");
|
|
391
|
+
// Clean up browser pool
|
|
392
|
+
if (PuppeteerEngine.browserPool) {
|
|
393
|
+
console.log("Cleaning up browser pool...");
|
|
394
|
+
await PuppeteerEngine.browserPool.cleanup();
|
|
395
|
+
PuppeteerEngine.browserPool = null; // Ensure pool is null after cleanup
|
|
396
|
+
console.log("Browser pool cleaned up.");
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
console.log("Browser pool was already null.");
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
console.error("Error during PuppeteerEngine cleanup:", error);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
getMetrics() {
|
|
407
|
+
return PuppeteerEngine.browserPool?.getMetrics() || [];
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
exports.PuppeteerEngine = PuppeteerEngine;
|
|
411
|
+
PuppeteerEngine.browserPool = null;
|
|
412
|
+
//# sourceMappingURL=PuppeteerEngine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PuppeteerEngine.js","sourceRoot":"","sources":["../src/PuppeteerEngine.ts"],"names":[],"mappings":";;;;;;AAEA,uDAAoD;AACpD,sDAA6B;AAG7B,SAAS,KAAK,CAAC,IAAY;IACzB,oBAAoB;IACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAC7D,CAAC;AAQD;;;GAGG;AACH,MAAa,eAAe;IAO1B,YAAY,eAAe,GAAG,CAAC;QAJd,UAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;QAC3C,aAAQ,GAAW,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,uBAAuB;QACnE,mBAAc,GAAY,KAAK,CAAC,CAAC,4CAA4C;QAGnF,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAM,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC;QAC1D,wEAAwE;QACxE,4DAA4D;QAE5D,6BAA6B;QAC7B,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,sCAAsC;YACtC,2CAA2C;QAC7C,CAAC,CAAC;QAEF,6EAA6E;QAC7E,iEAAiE;QACjE,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,gCAAgC;QAChC,IAAI,eAAe,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACvD,gDAAgD;YAChD,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC3B,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;YACtD,eAAe,CAAC,WAAW,GAAG,IAAI,yBAAW,CAC3C,CAAC,EAAE,cAAc;YACjB,CAAC,EAAE,qBAAqB;YACxB,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,yBAAyB;YACzC,EAAE,GAAG,IAAI,CAAC,iCAAiC;aAC5C,CAAC;YACF,MAAM,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,eAAe,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,iCAAiC;YACrE,MAAM,KAAK,CAAC,CAAC,iBAAiB;QAChC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,oBAAoB;QACpB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;YACpC,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,uDAAuD;QACvD,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAA8B,EAAE;YACvE,MAAM,OAAO,GAA8C,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACrC,IAAI,IAAI,GAAgB,IAAI,CAAC,CAAC,0BAA0B;YAExD,IAAI,CAAC;gBACH,yBAAyB;gBACzB,qBAAqB;gBACrB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,0BAA0B;gBACrG,IAAI,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,cAAc;oBACzB,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;iBACxC,CAAC,CAAC;gBAEH,uBAAuB;gBACvB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAErC,6BAA6B;gBAC7B,MAAM,IAAI,CAAC,YAAY,CACrB,uHAAuH,CACxH,CAAC;gBAEF,oBAAoB;gBACpB,MAAM,IAAI,CAAC,mBAAmB,CAAC;oBAC7B,iBAAiB,EAAE,gBAAgB;oBACnC,OAAO,EAAE,yBAAyB;oBAClC,WAAW,EAAE,mEAAmE;oBAChF,kBAAkB,EAAE,IAAI;oBACxB,oBAAoB,EAAE,SAAS;iBAChC,CAAC,CAAC;gBAEH,oBAAoB;gBACpB,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEtC,iEAAiE;gBACjE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;oBACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAC9E,MAAM,IAAI,CAAC,SAAS,CAAC;wBACnB,IAAI,EAAE,gBAAgB;wBACtB,KAAK,EAAE,UAAU;wBACjB,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,uBAAuB;wBAC7C,IAAI,EAAE,GAAG;qBACV,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,gCAAgC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrG,CAAC;gBAED,2DAA2D;gBAC3D,MAAM,iBAAiB,GAAG,mEAAmE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAExG,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;gBACxC,MAAM,cAAc,GAAG,CAAC,OAAY,CAAC,2BAA2B,EAAE,EAAE;oBAClE,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;oBAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBAEjC,4EAA4E;oBAC5E,IACE,CAAC,iBAAiB;wBAClB,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;4BAC9D,yFAAyF,CAAC,IAAI,CAC5F,UAAU,CACX,CAAC,EACJ,CAAC;wBACD,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBAC7F,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACnG,CAAC;gBACH,CAAC,CAAC;gBACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,iBAAiB;gBAErD,MAAM,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAC,oBAAoB;gBACnE,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU;iBACzC,CAAC,CAAC;gBAEH,aAAa;gBACb,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAC1C,IAAI,QAAQ,CAAC;gBAEb,IAAI,CAAC;oBACH,IAAI,iBAAiB,EAAE,CAAC;wBACtB,mCAAmC;wBACnC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;4BAC9B,SAAS,EAAE,CAAC,kBAAkB,EAAE,cAAc,CAAC,EAAE,4CAA4C;4BAC7F,OAAO,EAAE,KAAK;yBACf,CAAC,CAAC;wBACH,gDAAgD;wBAChD,MAAM,IAAI;6BACP,QAAQ,CAAC,GAAG,EAAE;4BACb,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;4BACnD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAoB,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9F,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CACjD,CAAC;4BACF,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;wBACpC,CAAC,CAAC;6BACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;wBAC5E,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,qCAAqC;oBAC1D,CAAC;yBAAM,CAAC;wBACN,oCAAoC;wBACpC,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;4BAC9B,SAAS,EAAE,kBAAkB,EAAE,sBAAsB;4BACrD,OAAO,EAAE,KAAK;yBACf,CAAC,CAAC;wBAEH,uCAAuC;wBACvC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;4BAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;4BACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;4BAClD,OAAO,CACL,6JAA6J,CAAC,IAAI,CAChK,KAAK,GAAG,QAAQ,CACjB,IAAI,QAAQ,CAAC,aAAa,CAAC,wCAAwC,CAAC,KAAK,IAAI,CAC/E,CAAC;wBACJ,CAAC,CAAC,CAAC;wBAEH,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,GAAG,CAAC,+BAA+B,GAAG,cAAc,CAAC,CAAC;4BAC9D,IAAI,CAAC;gCACH,0DAA0D;gCAC1D,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;oCACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;oCAC7B,OAAO,CACL,CAAC,kDAAkD,CAAC,IAAI,CAAC,KAAK,CAAC;wCAC/D,CAAC,QAAQ,CAAC,aAAa,CAAC,wCAAwC,CAAC,CAClE,CAAC;gCACJ,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,sBAAsB;iCAC1C,CAAC;gCACF,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;gCACzD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,wCAAwC;gCAC3D,yEAAyE;gCACzE,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,uBAAuB;gCAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;oCACd,qCAAqC;oCACrC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gCAC9E,CAAC;4BACH,CAAC;4BAAC,OAAO,cAAc,EAAE,CAAC;gCACxB,OAAO,CAAC,IAAI,CAAC,0CAA0C,GAAG,GAAG,EAAE,cAAc,CAAC,CAAC;gCAC/E,gEAAgE;gCAChE,MAAM,IAAI;qCACP,QAAQ,CAAC,GAAG,EAAE;oCACb,QAAQ;yCACL,gBAAgB,CAAc,kEAAkE,CAAC;yCACjG,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;wCACd,IAAI,CAAC;4CACH,EAAE,CAAC,KAAK,EAAE,CAAC;wCACb,CAAC;wCAAC,OAAO,CAAC,EAAE,CAAC,CAAA,CAAC;wCACd,IAAK,EAAuB,CAAC,IAAI,KAAK,UAAU;4CAAG,EAAuB,CAAC,OAAO,GAAG,IAAI,CAAC;oCAC5F,CAAC,CAAC,CAAC;gCACP,CAAC,CAAC;qCACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gCAC3E,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gCAClB,IAAI,CAAC;oCACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gCAClF,CAAC;gCAAC,OAAO,WAAW,EAAE,CAAC;oCACrB,OAAO,CAAC,IAAI,CAAC,iDAAiD,GAAG,GAAG,EAAE,WAAW,CAAC,CAAC;oCACnF,6CAA6C;gCAC/C,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,8DAA8D;4BAC9D,MAAM,OAAO,CAAC,IAAI,CAAC;gCACjB,IAAI,CAAC,iBAAiB,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;gCACrF,KAAK,CAAC,IAAI,CAAC,EAAE,gBAAgB;6BAC9B,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,eAAwB,EAAE,CAAC;oBAClC,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,GAAG,EAAE,eAAe,CAAC,CAAC;oBACxE,MAAM,QAAQ,GAAG,eAAe,YAAY,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;oBACtG,qFAAqF;oBACrF,IAAI,sDAAsD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC1E,OAAO,CAAC,IAAI,CAAC,wCAAwC,GAAG,kCAAkC,CAAC,CAAC;wBAC5F,0EAA0E;wBAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;wBAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,CAAC;wBACpE,IAAI,IAAI,EAAE,CAAC;4BACT,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;wBAC1C,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC,CAAC,gCAAgC;wBACpG,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,eAAe,CAAC,CAAC,kCAAkC;oBAC3D,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,kBAAkB;oBACvD,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,wBAAwB;gBACpE,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,qBAAqB;oBAChC,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,eAAe;iBAC9C,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,OAAO,CAAC,IAAI,CAAC,iDAAiD,GAAG,iCAAiC,CAAC,CAAC;oBACpG,mFAAmF;gBACrF,CAAC;qBAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;oBAC1B,OAAO,CAAC,IAAI,CAAC,kCAAkC,GAAG,YAAY,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBACnF,yEAAyE;oBACzE,IAAI,QAAQ,CAAC,MAAM,EAAE,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;wBACxD,gBAAgB;wBAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,YAAY,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBAC/E,CAAC;oBACD,qEAAqE;gBACvE,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBACvC,IAAI,CAAC;oBACH,sEAAsE;oBACtE,MAAM,OAAO,CAAC,IAAI,CAAC;wBACjB,IAAI;6BACD,eAAe,CAAC,wEAAwE,EAAE;4BACzF,OAAO,EAAE,KAAK;4BACd,OAAO,EAAE,IAAI;yBACd,CAAC;6BACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;wBAClB,IAAI;6BACD,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;6BAC1F,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;wBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,eAAe;qBAC7B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,CAAC,IAAI,CAAC,2CAA2C,GAAG,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC7E,CAAC;wBAAS,CAAC;oBACT,OAAO,CAAC,IAAI,CAAC;wBACX,SAAS,EAAE,kBAAkB;wBAC7B,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY;qBAC3C,CAAC,CAAC;gBACL,CAAC;gBAED,qBAAqB;gBACrB,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBACvC,IAAI,IAAI,EAAE,KAAK,CAAC;gBAEhB,2DAA2D;gBAC3D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;oBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CACxC,iEAAiE,CAClE,EAAE,SAAS,CAAC;oBACb,OAAO;wBACL,IAAI,EAAE,WAAW,IAAI,QAAQ,CAAC,eAAe,CAAC,SAAS,EAAE,wBAAwB;wBACjF,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,IAAI,EAAE;qBACzE,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACrB,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;gBAEvB,gDAAgD;gBAChD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBAC/B,OAAO,CAAC,IAAI,CAAC,iCAAiC,GAAG,sCAAsC,CAAC,CAAC;oBACzF,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC5B,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,0CAA0C;gBACxE,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,oBAAoB;oBAC/B,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY;iBAC3C,CAAC,CAAC;gBAEH,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU;iBACzC,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE;oBACzC,GAAG;oBACH,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,KAAK;oBACnC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,6BAA6B;iBACpH,CAAC,CAAC;gBAEH,MAAM,WAAW,GAAoB;oBACnC,IAAI;oBACJ,KAAK;oBACL,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,0CAA0C;iBAC5D,CAAC;gBAEF,8BAA8B;gBAC9B,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBACnC,OAAO,WAAW,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CACX,oCAAoC,GAAG,GAAG,EAC1C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EACtD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACF,yFAAyF;gBACzF,MAAM,KAAK,CAAC,CAAC,0DAA0D;YACzE,CAAC;oBAAS,CAAC;gBACT,IAAI,IAAI,EAAE,CAAC;oBACT,gDAAgD;oBAChD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAC/F,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,+DAA+D;QAC/D,OAAO,YAAwC,CAAC;IAClD,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,gBAAgB;YAChB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,MAAuB;QACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,MAAM;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;QAExC,0EAA0E;QAC1E,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC;YAC1B,oCAAoC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YACjD,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IAED,oEAAoE;IAEpE,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAE1B,uCAAuC;YACvC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YAEnC,wBAAwB;YACxB,IAAI,eAAe,CAAC,WAAW,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;gBAC3C,MAAM,eAAe,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC5C,eAAe,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,oCAAoC;gBACxE,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,UAAU;QACR,OAAO,eAAe,CAAC,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACzD,CAAC;;AArbH,0CAsbC;AArbgB,2BAAW,GAAuB,IAAI,AAA3B,CAA4B"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type Page } from "playwright";
|
|
2
|
+
import type { BrowserMetrics } from "../types.js";
|
|
3
|
+
export declare class BrowserPool {
|
|
4
|
+
private pool;
|
|
5
|
+
private readonly maxBrowsers;
|
|
6
|
+
private readonly maxPagesPerBrowser;
|
|
7
|
+
private readonly maxBrowserAge;
|
|
8
|
+
private readonly healthCheckInterval;
|
|
9
|
+
private healthCheckTimer?;
|
|
10
|
+
private readonly maxIdleTime;
|
|
11
|
+
private isCleaningUp;
|
|
12
|
+
constructor(maxBrowsers?: number, maxPagesPerBrowser?: number, // Revisit how this limit is applied with Playwright contexts
|
|
13
|
+
maxBrowserAge?: number, // 30 minutes
|
|
14
|
+
healthCheckInterval?: number);
|
|
15
|
+
private createBrowser;
|
|
16
|
+
private closeAndRemoveBrowser;
|
|
17
|
+
private healthCheck;
|
|
18
|
+
private ensureMinimumBrowsers;
|
|
19
|
+
private startHealthChecks;
|
|
20
|
+
/**
|
|
21
|
+
* Asynchronously initializes the browser pool, creating the first browser instance
|
|
22
|
+
* and starting health checks. Should be called before acquiring pages.
|
|
23
|
+
*/
|
|
24
|
+
initialize(): Promise<void>;
|
|
25
|
+
acquirePage(): Promise<Page>;
|
|
26
|
+
cleanup(): Promise<void>;
|
|
27
|
+
getMetrics(): BrowserMetrics[];
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=BrowserPool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BrowserPool.d.ts","sourceRoot":"","sources":["../../src/browser/BrowserPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAE/D,OAAO,KAAK,EAAmB,cAAc,EAAE,MAAM,aAAa,CAAC;AAInE,qBAAa,WAAW;IACtB,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,gBAAgB,CAAC,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyB;IACrD,OAAO,CAAC,YAAY,CAAkB;gBAGpC,WAAW,SAAI,EACf,kBAAkB,SAAI,EAAE,6DAA6D;IACrF,aAAa,SAAiB,EAAE,aAAa;IAC7C,mBAAmB,SAAY;YAUnB,aAAa;YAkFb,qBAAqB;YA+BrB,WAAW;YAiFX,qBAAqB;IA4BnC,OAAO,CAAC,iBAAiB;IAezB;;;OAGG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAY3B,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IA2F5B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAqC9B,UAAU,IAAI,cAAc,EAAE;CAStC"}
|