arn-browser 0.1.44 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +0 -0
- package/package.json +6 -6
- package/src/utility/playwright/routes/pwRoute.d.ts +6 -0
- package/src/utility/playwright/routes/pwRoute.js +129 -75
- package/src/utility/proxy-utility/proxy-chain.js +2 -2
- package/src/utility/puppeteer/routes/ppRoute.d.ts +6 -0
- package/src/utility/puppeteer/routes/ppRoute.js +129 -74
package/bin/cli.js
CHANGED
|
File without changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "arn-browser",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.46",
|
|
4
4
|
"description": "A lightweight, browser autmation helper.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"test": "test"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@aws-sdk/client-ec2": "^3.
|
|
16
|
-
"@ghostery/adblocker": "^2.
|
|
15
|
+
"@aws-sdk/client-ec2": "^3.1053.0",
|
|
16
|
+
"@ghostery/adblocker": "^2.17.3",
|
|
17
17
|
"arn-knexjs": "^0.0.6",
|
|
18
18
|
"camoufox-js": "^0.10.2",
|
|
19
19
|
"devtools-detector": "^2.0.25",
|
|
@@ -24,8 +24,8 @@
|
|
|
24
24
|
"node-cache": "^5.1.2",
|
|
25
25
|
"node-fetch": "^3.3.2",
|
|
26
26
|
"playwright-core": "1.42.1",
|
|
27
|
-
"proxy-chain": "^
|
|
28
|
-
"puppeteer-core": "^
|
|
27
|
+
"proxy-chain": "^3.0.0",
|
|
28
|
+
"puppeteer-core": "^25.0.4",
|
|
29
29
|
"randomstring": "^1.3.1",
|
|
30
30
|
"socks-proxy-agent": "^10.0.0",
|
|
31
31
|
"speakeasy": "^2.0.0",
|
|
@@ -44,4 +44,4 @@
|
|
|
44
44
|
"author": "ARNDESK",
|
|
45
45
|
"license": "ISC",
|
|
46
46
|
"type": "module"
|
|
47
|
-
}
|
|
47
|
+
}
|
|
@@ -62,6 +62,12 @@ export interface PwRouteOptions {
|
|
|
62
62
|
* Playwright network stack will be used instead.
|
|
63
63
|
*/
|
|
64
64
|
skipGotPatterns?: string[];
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Intercept XHR/Fetch requests.
|
|
68
|
+
* Can be boolean (true/false) or a custom proxy string/object used only for XHR/Fetch requests.
|
|
69
|
+
*/
|
|
70
|
+
xhr?: boolean | string | { type?: string; host: string; port: number; user?: string; pass?: string } | null;
|
|
65
71
|
}
|
|
66
72
|
|
|
67
73
|
/**
|
|
@@ -7,6 +7,67 @@ import { SocksProxyAgent } from "socks-proxy-agent";
|
|
|
7
7
|
import NodeCache from "node-cache";
|
|
8
8
|
|
|
9
9
|
let AdBlockEngine;
|
|
10
|
+
const routeConfigs = new WeakMap();
|
|
11
|
+
|
|
12
|
+
const DEFAULTS = {
|
|
13
|
+
logger: false,
|
|
14
|
+
blockAds: true,
|
|
15
|
+
blockImage: true,
|
|
16
|
+
useGot: false,
|
|
17
|
+
useFullUrl: true,
|
|
18
|
+
useCache: true,
|
|
19
|
+
stripGotHeaders: true,
|
|
20
|
+
stripGotLogger: false,
|
|
21
|
+
proxy: null,
|
|
22
|
+
m4w_send_on_post: null,
|
|
23
|
+
m4w_send_on_message: null,
|
|
24
|
+
allowImagePatterns: [],
|
|
25
|
+
skipGotPatterns: [],
|
|
26
|
+
xhr: false,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Helper to update derived configuration values from active user options.
|
|
31
|
+
*/
|
|
32
|
+
function updateDerivedConfig(config) {
|
|
33
|
+
// 1. Defaults/merges for allowImagePatterns
|
|
34
|
+
const allowImagePatterns = config.allowImagePatterns || [];
|
|
35
|
+
config.finalImagePatterns = ["cdn-cgi/challenge-platform", ...allowImagePatterns];
|
|
36
|
+
|
|
37
|
+
// 2. Defaults/merges for skipGotPatterns
|
|
38
|
+
const skipGotPatterns = config.skipGotPatterns || [];
|
|
39
|
+
config.finalSkipHosts = new Set(
|
|
40
|
+
skipGotPatterns.map((entry) => {
|
|
41
|
+
try {
|
|
42
|
+
if (entry.includes("://")) return new URL(entry).hostname;
|
|
43
|
+
} catch {}
|
|
44
|
+
return entry;
|
|
45
|
+
})
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
// 3. Proxies
|
|
49
|
+
config.proxyUrl = formatProxyUrl(config.proxy);
|
|
50
|
+
config.proxyAgent = createProxyAgent(config.proxyUrl);
|
|
51
|
+
|
|
52
|
+
// Support for separate XHR proxy
|
|
53
|
+
if (config.xhr && typeof config.xhr !== "boolean") {
|
|
54
|
+
config.xhrProxyUrl = formatProxyUrl(config.xhr);
|
|
55
|
+
config.xhrProxyAgent = createProxyAgent(config.xhrProxyUrl);
|
|
56
|
+
} else {
|
|
57
|
+
config.xhrProxyUrl = null;
|
|
58
|
+
config.xhrProxyAgent = null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// 4. Intercepted resource types
|
|
62
|
+
const types = ["stylesheet", "script", "font"];
|
|
63
|
+
if (!config.blockImage) {
|
|
64
|
+
types.push("image");
|
|
65
|
+
}
|
|
66
|
+
if (config.xhr) {
|
|
67
|
+
types.push("xhr", "fetch");
|
|
68
|
+
}
|
|
69
|
+
config.interceptedResourceTypes = types;
|
|
70
|
+
}
|
|
10
71
|
|
|
11
72
|
// Create a NodeCache instance for caching responses
|
|
12
73
|
// This helps reduce bandwidth usage and speed up repeated requests
|
|
@@ -238,71 +299,59 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
238
299
|
* @param {Object} options.m4w_send_on_message - Custom handler data for Doublelist messages
|
|
239
300
|
* @param {Array<string>} options.allowImagePatterns - Array of strings/patterns. If a URL contains any of these, it will NOT be blocked even if blockImage is true.
|
|
240
301
|
* @param {Array<string>} options.skipGotPatterns - Array of strings/patterns. If a URL contains any of these, it will skip the custom Superagent fetch.
|
|
302
|
+
* @param {boolean|string|Object|null} options.xhr - Enable XHR/Fetch request interception, optionally using a custom proxy just for XHR/Fetch
|
|
241
303
|
*/
|
|
242
|
-
export async function pwRoute({
|
|
243
|
-
context = null
|
|
244
|
-
page = null
|
|
245
|
-
|
|
246
|
-
blockAds = true,
|
|
247
|
-
blockImage = true,
|
|
248
|
-
useGot = false,
|
|
249
|
-
useFullUrl = true,
|
|
250
|
-
useCache = true,
|
|
251
|
-
stripGotHeaders = true,
|
|
252
|
-
stripGotLogger = false,
|
|
253
|
-
proxy = null,
|
|
254
|
-
m4w_send_on_post = null,
|
|
255
|
-
m4w_send_on_message = null,
|
|
256
|
-
allowImagePatterns = [], // Default empty, merged inside
|
|
257
|
-
skipGotPatterns = [], // Default empty, merged inside
|
|
258
|
-
}) {
|
|
304
|
+
export async function pwRoute(options = {}) {
|
|
305
|
+
const context = options.context || null;
|
|
306
|
+
const page = options.page || null;
|
|
307
|
+
|
|
259
308
|
// Validation: Ensure we have a target to attach the route to
|
|
260
309
|
if (!context && !page) {
|
|
261
310
|
throw new Error("Either context or page must be provided.");
|
|
262
311
|
}
|
|
263
312
|
const contextOrPage = context ? context : page;
|
|
264
313
|
|
|
265
|
-
//
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
314
|
+
// Check if we already have a route config for this context/page
|
|
315
|
+
let config = routeConfigs.get(contextOrPage);
|
|
316
|
+
if (config) {
|
|
317
|
+
// Update the existing configuration with new options!
|
|
318
|
+
Object.assign(config, options);
|
|
319
|
+
updateDerivedConfig(config);
|
|
320
|
+
|
|
321
|
+
// Handle ad blocking initialization if newly enabled
|
|
322
|
+
if (config.blockAds && !AdBlockEngine) {
|
|
323
|
+
AdBlockEngine = await FiltersEngine.fromPrebuiltAdsAndTracking(fetch);
|
|
324
|
+
if (!AdBlockEngine) {
|
|
325
|
+
throw new Error("Failed to initialize AdBlockEngine.");
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// First time registration:
|
|
332
|
+
config = {
|
|
333
|
+
...DEFAULTS,
|
|
334
|
+
...options,
|
|
335
|
+
};
|
|
336
|
+
routeConfigs.set(contextOrPage, config);
|
|
337
|
+
updateDerivedConfig(config);
|
|
282
338
|
|
|
283
339
|
// Initialize ad blocking AdBlockEngine if enabled and not already loaded
|
|
284
|
-
if (blockAds && !AdBlockEngine) {
|
|
285
|
-
// console.log("Initializing AdBlockEngine..............................");
|
|
340
|
+
if (config.blockAds && !AdBlockEngine) {
|
|
286
341
|
AdBlockEngine = await FiltersEngine.fromPrebuiltAdsAndTracking(fetch);
|
|
287
342
|
if (!AdBlockEngine) {
|
|
288
343
|
throw new Error("Failed to initialize AdBlockEngine.");
|
|
289
344
|
}
|
|
290
345
|
}
|
|
291
346
|
|
|
292
|
-
// Define resource types to intercept for custom fetching (useGot)
|
|
293
|
-
const interceptedResourceTypes = ["stylesheet", "script", "font"];
|
|
294
|
-
|
|
295
|
-
// Create proxy agent once (reused for all requests in this route)
|
|
296
|
-
const proxyUrl = formatProxyUrl(proxy);
|
|
297
|
-
const proxyAgent = createProxyAgent(proxyUrl);
|
|
298
|
-
|
|
299
|
-
// If images are NOT blocked, we generally want to intercept/cache them too.
|
|
300
|
-
if (!blockImage) {
|
|
301
|
-
interceptedResourceTypes.push("image");
|
|
302
|
-
}
|
|
303
|
-
|
|
304
347
|
// Set up the global route interception
|
|
305
348
|
await contextOrPage.route("**/*", async (route, request) => {
|
|
349
|
+
const currentConfig = routeConfigs.get(contextOrPage);
|
|
350
|
+
if (!currentConfig) {
|
|
351
|
+
await route.continue();
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
|
|
306
355
|
const url = request.url();
|
|
307
356
|
const method = request.method();
|
|
308
357
|
const resourceType = request.resourceType();
|
|
@@ -310,9 +359,9 @@ export async function pwRoute({
|
|
|
310
359
|
// ============================================================
|
|
311
360
|
// Group 1: Image Blocking
|
|
312
361
|
// ============================================================
|
|
313
|
-
if (blockImage && resourceType === "image") {
|
|
362
|
+
if (currentConfig.blockImage && resourceType === "image") {
|
|
314
363
|
// Check against the merged list (defaults + user input)
|
|
315
|
-
const isAllowed = finalImagePatterns.some((pattern) => url.includes(pattern));
|
|
364
|
+
const isAllowed = currentConfig.finalImagePatterns.some((pattern) => url.includes(pattern));
|
|
316
365
|
|
|
317
366
|
if (!isAllowed) {
|
|
318
367
|
route.abort();
|
|
@@ -323,7 +372,7 @@ export async function pwRoute({
|
|
|
323
372
|
// ============================================================
|
|
324
373
|
// Group 2: Ad Blocking
|
|
325
374
|
// ============================================================
|
|
326
|
-
if (blockAds && AdBlockEngine) {
|
|
375
|
+
if (currentConfig.blockAds && AdBlockEngine) {
|
|
327
376
|
const adsBlockResult = AdBlockEngine.match(
|
|
328
377
|
Request.fromRawDetails({
|
|
329
378
|
url: url,
|
|
@@ -347,23 +396,23 @@ export async function pwRoute({
|
|
|
347
396
|
// ============================================================
|
|
348
397
|
// Group 4: m4w_send_on_message Handling (Doublelist API)
|
|
349
398
|
// ============================================================
|
|
350
|
-
if (m4w_send_on_message && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
399
|
+
if (currentConfig.m4w_send_on_message && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
351
400
|
const headers = request.headers();
|
|
352
401
|
const postData = request.postData();
|
|
353
402
|
const urlParams = new URLSearchParams(postData);
|
|
354
403
|
const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
|
|
355
404
|
const parsedMessageJSON = JSON.parse(messageJSON);
|
|
356
405
|
|
|
357
|
-
m4w_send_on_message.sender_id = parsedMessageJSON.sender_id;
|
|
406
|
+
currentConfig.m4w_send_on_message.sender_id = parsedMessageJSON.sender_id;
|
|
358
407
|
const authorizationHeader = headers["authorization"];
|
|
359
|
-
m4w_send_on_message.token_value = authorizationHeader.replace("Bearer ", "");
|
|
408
|
+
currentConfig.m4w_send_on_message.token_value = authorizationHeader.replace("Bearer ", "");
|
|
360
409
|
|
|
361
410
|
console.log("posts_send_message Blocked (Data Extracted)");
|
|
362
411
|
route.abort();
|
|
363
412
|
return;
|
|
364
413
|
}
|
|
365
414
|
|
|
366
|
-
if (m4w_send_on_message && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
415
|
+
if (currentConfig.m4w_send_on_message && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
367
416
|
console.log("posts_send_message Blocked");
|
|
368
417
|
route.abort();
|
|
369
418
|
return;
|
|
@@ -372,23 +421,23 @@ export async function pwRoute({
|
|
|
372
421
|
// ============================================================
|
|
373
422
|
// Group 5: m4w_send_on_post Handling (Doublelist API)
|
|
374
423
|
// ============================================================
|
|
375
|
-
if (m4w_send_on_post && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
424
|
+
if (currentConfig.m4w_send_on_post && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
376
425
|
if (
|
|
377
|
-
m4w_send_on_post.current_post_id &&
|
|
378
|
-
m4w_send_on_post.current_post_id !== m4w_send_on_post.last_post_id &&
|
|
379
|
-
m4w_send_on_post.current_send_to &&
|
|
380
|
-
m4w_send_on_post.current_send_to != m4w_send_on_post.last_send_to
|
|
426
|
+
currentConfig.m4w_send_on_post.current_post_id &&
|
|
427
|
+
currentConfig.m4w_send_on_post.current_post_id !== currentConfig.m4w_send_on_post.last_post_id &&
|
|
428
|
+
currentConfig.m4w_send_on_post.current_send_to &&
|
|
429
|
+
currentConfig.m4w_send_on_post.current_send_to != currentConfig.m4w_send_on_post.last_send_to
|
|
381
430
|
) {
|
|
382
|
-
m4w_send_on_post.last_send_to = m4w_send_on_post.current_send_to;
|
|
383
|
-
m4w_send_on_post.last_post_id = m4w_send_on_post.current_post_id;
|
|
431
|
+
currentConfig.m4w_send_on_post.last_send_to = currentConfig.m4w_send_on_post.current_send_to;
|
|
432
|
+
currentConfig.m4w_send_on_post.last_post_id = currentConfig.m4w_send_on_post.current_post_id;
|
|
384
433
|
|
|
385
434
|
let postData = request.postData();
|
|
386
435
|
const urlParams = new URLSearchParams(postData);
|
|
387
436
|
const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
|
|
388
437
|
let parsedMessageJSON = JSON.parse(messageJSON);
|
|
389
438
|
|
|
390
|
-
parsedMessageJSON.chat_user = m4w_send_on_post.current_send_to;
|
|
391
|
-
parsedMessageJSON.post_id = m4w_send_on_post.current_post_id;
|
|
439
|
+
parsedMessageJSON.chat_user = currentConfig.m4w_send_on_post.current_send_to;
|
|
440
|
+
parsedMessageJSON.post_id = currentConfig.m4w_send_on_post.current_post_id;
|
|
392
441
|
|
|
393
442
|
urlParams.set("messageJSON", JSON.stringify(parsedMessageJSON));
|
|
394
443
|
const final_postData = urlParams.toString();
|
|
@@ -404,13 +453,13 @@ export async function pwRoute({
|
|
|
404
453
|
}
|
|
405
454
|
}
|
|
406
455
|
|
|
407
|
-
if (m4w_send_on_post && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
456
|
+
if (currentConfig.m4w_send_on_post && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
408
457
|
console.log("posts_send_message Blocked");
|
|
409
458
|
await route.abort();
|
|
410
459
|
return;
|
|
411
460
|
}
|
|
412
461
|
|
|
413
|
-
if (m4w_send_on_post && url.includes("doublelist.com/messages/")) {
|
|
462
|
+
if (currentConfig.m4w_send_on_post && url.includes("doublelist.com/messages/")) {
|
|
414
463
|
console.log("doublelist.com/messages/ Redirected");
|
|
415
464
|
await route.fulfill({
|
|
416
465
|
status: 302,
|
|
@@ -424,27 +473,32 @@ export async function pwRoute({
|
|
|
424
473
|
// ============================================================
|
|
425
474
|
// Group 6: Resource Interception (Custom Fetch/Cache)
|
|
426
475
|
// ============================================================
|
|
427
|
-
if (useGot && interceptedResourceTypes.includes(resourceType) && !url.startsWith("data:")) {
|
|
476
|
+
if (currentConfig.useGot && currentConfig.interceptedResourceTypes.includes(resourceType) && !url.startsWith("data:")) {
|
|
428
477
|
// Check against the normalized host list (defaults + user input)
|
|
429
478
|
let shouldSkipGot = false;
|
|
430
479
|
try {
|
|
431
|
-
shouldSkipGot = finalSkipHosts.has(new URL(url).hostname);
|
|
480
|
+
shouldSkipGot = currentConfig.finalSkipHosts.has(new URL(url).hostname);
|
|
432
481
|
} catch {}
|
|
433
482
|
|
|
434
483
|
if (!shouldSkipGot) {
|
|
435
484
|
const requestHeaders = request.headers();
|
|
436
485
|
const requestMethod = request.method();
|
|
437
486
|
|
|
487
|
+
const isXhrOrFetch = resourceType === "xhr" || resourceType === "fetch";
|
|
488
|
+
const agentToUse = (isXhrOrFetch && currentConfig.xhrProxyAgent)
|
|
489
|
+
? currentConfig.xhrProxyAgent
|
|
490
|
+
: currentConfig.proxyAgent;
|
|
491
|
+
|
|
438
492
|
const response = await fetchWithClient(
|
|
439
|
-
useCache,
|
|
493
|
+
currentConfig.useCache,
|
|
440
494
|
url,
|
|
441
495
|
requestHeaders,
|
|
442
496
|
requestMethod,
|
|
443
|
-
useFullUrl,
|
|
444
|
-
logger,
|
|
445
|
-
|
|
446
|
-
stripGotHeaders,
|
|
447
|
-
stripGotLogger
|
|
497
|
+
currentConfig.useFullUrl,
|
|
498
|
+
currentConfig.logger,
|
|
499
|
+
agentToUse,
|
|
500
|
+
currentConfig.stripGotHeaders,
|
|
501
|
+
currentConfig.stripGotLogger
|
|
448
502
|
);
|
|
449
503
|
|
|
450
504
|
if (response) {
|
|
@@ -455,7 +509,7 @@ export async function pwRoute({
|
|
|
455
509
|
});
|
|
456
510
|
return;
|
|
457
511
|
} else {
|
|
458
|
-
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
512
|
+
if (currentConfig.logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
459
513
|
await route.continue();
|
|
460
514
|
return;
|
|
461
515
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Server } from "proxy-chain";
|
|
2
2
|
import net from "net";
|
|
3
3
|
import https from "https";
|
|
4
4
|
import http from "http";
|
|
@@ -269,7 +269,7 @@ export async function startProxyServer({
|
|
|
269
269
|
return null;
|
|
270
270
|
}
|
|
271
271
|
|
|
272
|
-
server = new
|
|
272
|
+
server = new Server({
|
|
273
273
|
port: candidatePort,
|
|
274
274
|
host: "127.0.0.1",
|
|
275
275
|
verbose: debug,
|
|
@@ -59,6 +59,12 @@ export interface PpRouteOptions {
|
|
|
59
59
|
* Playwright network stack will be used instead.
|
|
60
60
|
*/
|
|
61
61
|
skipGotPatterns?: string[];
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Intercept XHR/Fetch requests.
|
|
65
|
+
* Can be boolean (true/false) or a custom proxy string/object used only for XHR/Fetch requests.
|
|
66
|
+
*/
|
|
67
|
+
xhr?: boolean | string | { type?: string; host: string; port: number; user?: string; pass?: string } | null;
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
/**
|
|
@@ -7,6 +7,67 @@ import { SocksProxyAgent } from "socks-proxy-agent";
|
|
|
7
7
|
import NodeCache from "node-cache";
|
|
8
8
|
|
|
9
9
|
let AdBlockEngine;
|
|
10
|
+
const routeConfigs = new WeakMap();
|
|
11
|
+
|
|
12
|
+
const DEFAULTS = {
|
|
13
|
+
logger: false,
|
|
14
|
+
blockAds: true,
|
|
15
|
+
blockImage: true,
|
|
16
|
+
useGot: false,
|
|
17
|
+
useFullUrl: true,
|
|
18
|
+
useCache: true,
|
|
19
|
+
stripGotHeaders: true,
|
|
20
|
+
stripGotLogger: false,
|
|
21
|
+
proxy: null,
|
|
22
|
+
m4w_send_on_post: null,
|
|
23
|
+
m4w_send_on_message: null,
|
|
24
|
+
allowImagePatterns: [],
|
|
25
|
+
skipGotPatterns: [],
|
|
26
|
+
xhr: false,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Helper to update derived configuration values from active user options.
|
|
31
|
+
*/
|
|
32
|
+
function updateDerivedConfig(config) {
|
|
33
|
+
// 1. Defaults/merges for allowImagePatterns
|
|
34
|
+
const allowImagePatterns = config.allowImagePatterns || [];
|
|
35
|
+
config.finalImagePatterns = ["cdn-cgi/challenge-platform", ...allowImagePatterns];
|
|
36
|
+
|
|
37
|
+
// 2. Defaults/merges for skipGotPatterns
|
|
38
|
+
const skipGotPatterns = config.skipGotPatterns || [];
|
|
39
|
+
config.finalSkipHosts = new Set(
|
|
40
|
+
skipGotPatterns.map((entry) => {
|
|
41
|
+
try {
|
|
42
|
+
if (entry.includes("://")) return new URL(entry).hostname;
|
|
43
|
+
} catch {}
|
|
44
|
+
return entry;
|
|
45
|
+
})
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
// 3. Proxies
|
|
49
|
+
config.proxyUrl = formatProxyUrl(config.proxy);
|
|
50
|
+
config.proxyAgent = createProxyAgent(config.proxyUrl);
|
|
51
|
+
|
|
52
|
+
// Support for separate XHR proxy
|
|
53
|
+
if (config.xhr && typeof config.xhr !== "boolean") {
|
|
54
|
+
config.xhrProxyUrl = formatProxyUrl(config.xhr);
|
|
55
|
+
config.xhrProxyAgent = createProxyAgent(config.xhrProxyUrl);
|
|
56
|
+
} else {
|
|
57
|
+
config.xhrProxyUrl = null;
|
|
58
|
+
config.xhrProxyAgent = null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// 4. Intercepted resource types
|
|
62
|
+
const types = ["stylesheet", "script", "font"];
|
|
63
|
+
if (!config.blockImage) {
|
|
64
|
+
types.push("image");
|
|
65
|
+
}
|
|
66
|
+
if (config.xhr) {
|
|
67
|
+
types.push("xhr", "fetch");
|
|
68
|
+
}
|
|
69
|
+
config.interceptedResourceTypes = types;
|
|
70
|
+
}
|
|
10
71
|
|
|
11
72
|
// Create a NodeCache instance for caching responses
|
|
12
73
|
// This helps reduce bandwidth usage and speed up repeated requests
|
|
@@ -240,67 +301,50 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
240
301
|
* @param {Object} options.m4w_send_on_message - Custom handler data for Doublelist messages
|
|
241
302
|
* @param {Array<string>} options.allowImagePatterns - Array of strings/patterns. If a URL contains any of these, it will NOT be blocked even if blockImage is true.
|
|
242
303
|
* @param {Array<string>} options.skipGotPatterns - Array of strings/patterns. If a URL contains any of these, it will skip the custom Superagent fetch.
|
|
304
|
+
* @param {boolean|string|Object|null} options.xhr - Enable XHR/Fetch request interception, optionally using a custom proxy just for XHR/Fetch
|
|
243
305
|
*/
|
|
244
|
-
export async function ppRoute({
|
|
245
|
-
page = null
|
|
246
|
-
|
|
247
|
-
blockAds = true,
|
|
248
|
-
blockImage = true,
|
|
249
|
-
useGot = false,
|
|
250
|
-
useFullUrl = true,
|
|
251
|
-
useCache = true,
|
|
252
|
-
stripGotHeaders = true,
|
|
253
|
-
stripGotLogger = false,
|
|
254
|
-
proxy = null,
|
|
255
|
-
m4w_send_on_post = null,
|
|
256
|
-
m4w_send_on_message = null,
|
|
257
|
-
allowImagePatterns = [], // Default empty, merged inside
|
|
258
|
-
skipGotPatterns = [], // Default empty, merged inside
|
|
259
|
-
}) {
|
|
306
|
+
export async function ppRoute(options = {}) {
|
|
307
|
+
const page = options.page || null;
|
|
308
|
+
|
|
260
309
|
// Validation: Ensure we have a page
|
|
261
310
|
if (!page) {
|
|
262
311
|
throw new Error("A Puppeteer page must be provided.");
|
|
263
312
|
}
|
|
313
|
+
const contextOrPage = page;
|
|
314
|
+
|
|
315
|
+
// Check if we already have a route config for this context/page
|
|
316
|
+
let config = routeConfigs.get(contextOrPage);
|
|
317
|
+
if (config) {
|
|
318
|
+
// Update the existing configuration with new options!
|
|
319
|
+
Object.assign(config, options);
|
|
320
|
+
updateDerivedConfig(config);
|
|
321
|
+
|
|
322
|
+
// Handle ad blocking initialization if newly enabled
|
|
323
|
+
if (config.blockAds && !AdBlockEngine) {
|
|
324
|
+
AdBlockEngine = await FiltersEngine.fromPrebuiltAdsAndTracking(fetch);
|
|
325
|
+
if (!AdBlockEngine) {
|
|
326
|
+
throw new Error("Failed to initialize AdBlockEngine.");
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
264
331
|
|
|
265
|
-
//
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const defaultSkipPatterns = [];
|
|
273
|
-
// Normalize: if entry contains "://", extract hostname. Otherwise keep as-is (assumed to be a hostname).
|
|
274
|
-
const finalSkipHosts = new Set(
|
|
275
|
-
[...defaultSkipPatterns, ...skipGotPatterns].map((entry) => {
|
|
276
|
-
try {
|
|
277
|
-
if (entry.includes("://")) return new URL(entry).hostname;
|
|
278
|
-
} catch {}
|
|
279
|
-
return entry;
|
|
280
|
-
})
|
|
281
|
-
);
|
|
332
|
+
// First time registration:
|
|
333
|
+
config = {
|
|
334
|
+
...DEFAULTS,
|
|
335
|
+
...options,
|
|
336
|
+
};
|
|
337
|
+
routeConfigs.set(contextOrPage, config);
|
|
338
|
+
updateDerivedConfig(config);
|
|
282
339
|
|
|
283
340
|
// Initialize ad blocking AdBlockEngine if enabled and not already loaded
|
|
284
|
-
if (blockAds && !AdBlockEngine) {
|
|
285
|
-
// console.log("Initializing AdBlockEngine..............................");
|
|
341
|
+
if (config.blockAds && !AdBlockEngine) {
|
|
286
342
|
AdBlockEngine = await FiltersEngine.fromPrebuiltAdsAndTracking(fetch);
|
|
287
343
|
if (!AdBlockEngine) {
|
|
288
344
|
throw new Error("Failed to initialize AdBlockEngine.");
|
|
289
345
|
}
|
|
290
346
|
}
|
|
291
347
|
|
|
292
|
-
// Define resource types to intercept for custom fetching (useGot)
|
|
293
|
-
const interceptedResourceTypes = ["stylesheet", "script", "font"];
|
|
294
|
-
|
|
295
|
-
// Create proxy agent once (reused for all requests in this route)
|
|
296
|
-
const proxyUrl = formatProxyUrl(proxy);
|
|
297
|
-
const proxyAgent = createProxyAgent(proxyUrl);
|
|
298
|
-
|
|
299
|
-
// If images are NOT blocked, we generally want to intercept/cache them too.
|
|
300
|
-
if (!blockImage) {
|
|
301
|
-
interceptedResourceTypes.push("image");
|
|
302
|
-
}
|
|
303
|
-
|
|
304
348
|
// Enable request interception in Puppeteer
|
|
305
349
|
await page.setRequestInterception(true);
|
|
306
350
|
|
|
@@ -309,6 +353,12 @@ export async function ppRoute({
|
|
|
309
353
|
// Puppeteer best practice: do not handle request if already handled
|
|
310
354
|
if (request.isInterceptResolutionHandled()) return;
|
|
311
355
|
|
|
356
|
+
const currentConfig = routeConfigs.get(contextOrPage);
|
|
357
|
+
if (!currentConfig) {
|
|
358
|
+
await request.continue();
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
|
|
312
362
|
const url = request.url();
|
|
313
363
|
const method = request.method();
|
|
314
364
|
const resourceType = request.resourceType();
|
|
@@ -316,9 +366,9 @@ export async function ppRoute({
|
|
|
316
366
|
// ============================================================
|
|
317
367
|
// Group 1: Image Blocking
|
|
318
368
|
// ============================================================
|
|
319
|
-
if (blockImage && resourceType === "image") {
|
|
369
|
+
if (currentConfig.blockImage && resourceType === "image") {
|
|
320
370
|
// Check against the merged list (defaults + user input)
|
|
321
|
-
const isAllowed = finalImagePatterns.some((pattern) => url.includes(pattern));
|
|
371
|
+
const isAllowed = currentConfig.finalImagePatterns.some((pattern) => url.includes(pattern));
|
|
322
372
|
|
|
323
373
|
if (!isAllowed) {
|
|
324
374
|
await request.abort();
|
|
@@ -329,7 +379,7 @@ export async function ppRoute({
|
|
|
329
379
|
// ============================================================
|
|
330
380
|
// Group 2: Ad Blocking
|
|
331
381
|
// ============================================================
|
|
332
|
-
if (blockAds && AdBlockEngine) {
|
|
382
|
+
if (currentConfig.blockAds && AdBlockEngine) {
|
|
333
383
|
const adsBlockResult = AdBlockEngine.match(
|
|
334
384
|
Request.fromRawDetails({
|
|
335
385
|
url: url,
|
|
@@ -353,23 +403,23 @@ export async function ppRoute({
|
|
|
353
403
|
// ============================================================
|
|
354
404
|
// Group 4: m4w_send_on_message Handling (Doublelist API)
|
|
355
405
|
// ============================================================
|
|
356
|
-
if (m4w_send_on_message && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
406
|
+
if (currentConfig.m4w_send_on_message && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
357
407
|
const headers = request.headers();
|
|
358
408
|
const postData = request.postData();
|
|
359
409
|
const urlParams = new URLSearchParams(postData);
|
|
360
410
|
const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
|
|
361
411
|
const parsedMessageJSON = JSON.parse(messageJSON);
|
|
362
412
|
|
|
363
|
-
m4w_send_on_message.sender_id = parsedMessageJSON.sender_id;
|
|
413
|
+
currentConfig.m4w_send_on_message.sender_id = parsedMessageJSON.sender_id;
|
|
364
414
|
const authorizationHeader = headers["authorization"];
|
|
365
|
-
m4w_send_on_message.token_value = authorizationHeader.replace("Bearer ", "");
|
|
415
|
+
currentConfig.m4w_send_on_message.token_value = authorizationHeader.replace("Bearer ", "");
|
|
366
416
|
|
|
367
417
|
console.log("posts_send_message Blocked (Data Extracted)");
|
|
368
418
|
await request.abort();
|
|
369
419
|
return;
|
|
370
420
|
}
|
|
371
421
|
|
|
372
|
-
if (m4w_send_on_message && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
422
|
+
if (currentConfig.m4w_send_on_message && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
373
423
|
console.log("posts_send_message Blocked");
|
|
374
424
|
await request.abort();
|
|
375
425
|
return;
|
|
@@ -378,23 +428,23 @@ export async function ppRoute({
|
|
|
378
428
|
// ============================================================
|
|
379
429
|
// Group 5: m4w_send_on_post Handling (Doublelist API)
|
|
380
430
|
// ============================================================
|
|
381
|
-
if (m4w_send_on_post && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
431
|
+
if (currentConfig.m4w_send_on_post && url.includes("api.doublelist.com/api/messages") && method === "POST") {
|
|
382
432
|
if (
|
|
383
|
-
m4w_send_on_post.current_post_id &&
|
|
384
|
-
m4w_send_on_post.current_post_id !== m4w_send_on_post.last_post_id &&
|
|
385
|
-
m4w_send_on_post.current_send_to &&
|
|
386
|
-
m4w_send_on_post.current_send_to != m4w_send_on_post.last_send_to
|
|
433
|
+
currentConfig.m4w_send_on_post.current_post_id &&
|
|
434
|
+
currentConfig.m4w_send_on_post.current_post_id !== currentConfig.m4w_send_on_post.last_post_id &&
|
|
435
|
+
currentConfig.m4w_send_on_post.current_send_to &&
|
|
436
|
+
currentConfig.m4w_send_on_post.current_send_to != currentConfig.m4w_send_on_post.last_send_to
|
|
387
437
|
) {
|
|
388
|
-
m4w_send_on_post.last_send_to = m4w_send_on_post.current_send_to;
|
|
389
|
-
m4w_send_on_post.last_post_id = m4w_send_on_post.current_post_id;
|
|
438
|
+
currentConfig.m4w_send_on_post.last_send_to = currentConfig.m4w_send_on_post.current_send_to;
|
|
439
|
+
currentConfig.m4w_send_on_post.last_post_id = currentConfig.m4w_send_on_post.current_post_id;
|
|
390
440
|
|
|
391
441
|
let postData = request.postData();
|
|
392
442
|
const urlParams = new URLSearchParams(postData);
|
|
393
443
|
const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
|
|
394
444
|
let parsedMessageJSON = JSON.parse(messageJSON);
|
|
395
445
|
|
|
396
|
-
parsedMessageJSON.chat_user = m4w_send_on_post.current_send_to;
|
|
397
|
-
parsedMessageJSON.post_id = m4w_send_on_post.current_post_id;
|
|
446
|
+
parsedMessageJSON.chat_user = currentConfig.m4w_send_on_post.current_send_to;
|
|
447
|
+
parsedMessageJSON.post_id = currentConfig.m4w_send_on_post.current_post_id;
|
|
398
448
|
|
|
399
449
|
urlParams.set("messageJSON", JSON.stringify(parsedMessageJSON));
|
|
400
450
|
const final_postData = urlParams.toString();
|
|
@@ -410,13 +460,13 @@ export async function ppRoute({
|
|
|
410
460
|
}
|
|
411
461
|
}
|
|
412
462
|
|
|
413
|
-
if (m4w_send_on_post && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
463
|
+
if (currentConfig.m4w_send_on_post && url.includes("https://doublelist.com/posts_send_message/")) {
|
|
414
464
|
console.log("posts_send_message Blocked");
|
|
415
465
|
await request.abort();
|
|
416
466
|
return;
|
|
417
467
|
}
|
|
418
468
|
|
|
419
|
-
if (m4w_send_on_post && url.includes("doublelist.com/messages/")) {
|
|
469
|
+
if (currentConfig.m4w_send_on_post && url.includes("doublelist.com/messages/")) {
|
|
420
470
|
console.log("doublelist.com/messages/ Redirected");
|
|
421
471
|
await request.respond({
|
|
422
472
|
status: 302,
|
|
@@ -430,27 +480,32 @@ export async function ppRoute({
|
|
|
430
480
|
// ============================================================
|
|
431
481
|
// Group 6: Resource Interception (Custom Fetch/Cache)
|
|
432
482
|
// ============================================================
|
|
433
|
-
if (useGot && interceptedResourceTypes.includes(resourceType) && !url.startsWith("data:")) {
|
|
483
|
+
if (currentConfig.useGot && currentConfig.interceptedResourceTypes.includes(resourceType) && !url.startsWith("data:")) {
|
|
434
484
|
// Check against the normalized host list (defaults + user input)
|
|
435
485
|
let shouldSkipGot = false;
|
|
436
486
|
try {
|
|
437
|
-
shouldSkipGot = finalSkipHosts.has(new URL(url).hostname);
|
|
487
|
+
shouldSkipGot = currentConfig.finalSkipHosts.has(new URL(url).hostname);
|
|
438
488
|
} catch {}
|
|
439
489
|
|
|
440
490
|
if (!shouldSkipGot) {
|
|
441
491
|
const requestHeaders = request.headers();
|
|
442
492
|
const requestMethod = request.method();
|
|
443
493
|
|
|
494
|
+
const isXhrOrFetch = resourceType === "xhr" || resourceType === "fetch";
|
|
495
|
+
const agentToUse = (isXhrOrFetch && currentConfig.xhrProxyAgent)
|
|
496
|
+
? currentConfig.xhrProxyAgent
|
|
497
|
+
: currentConfig.proxyAgent;
|
|
498
|
+
|
|
444
499
|
const response = await fetchWithClient(
|
|
445
|
-
useCache,
|
|
500
|
+
currentConfig.useCache,
|
|
446
501
|
url,
|
|
447
502
|
requestHeaders,
|
|
448
503
|
requestMethod,
|
|
449
|
-
useFullUrl,
|
|
450
|
-
logger,
|
|
451
|
-
|
|
452
|
-
stripGotHeaders,
|
|
453
|
-
stripGotLogger
|
|
504
|
+
currentConfig.useFullUrl,
|
|
505
|
+
currentConfig.logger,
|
|
506
|
+
agentToUse,
|
|
507
|
+
currentConfig.stripGotHeaders,
|
|
508
|
+
currentConfig.stripGotLogger
|
|
454
509
|
);
|
|
455
510
|
|
|
456
511
|
if (response) {
|
|
@@ -461,7 +516,7 @@ export async function ppRoute({
|
|
|
461
516
|
});
|
|
462
517
|
return;
|
|
463
518
|
} else {
|
|
464
|
-
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
519
|
+
if (currentConfig.logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
465
520
|
await request.continue();
|
|
466
521
|
return;
|
|
467
522
|
}
|