arn-browser 0.1.6 → 0.1.8

package/src/utility/puppeteer/routes/ppRoute.d.ts

@@ -0,0 +1,64 @@
+import { Browser, Page } from "puppeteer-core";
+
+// ============================================================================
+// ROUTING & CACHE TYPES
+// ============================================================================
+
+/**
+ * Options for the ppRoute function.
+ */
+export interface PpRouteOptions {
+	/** Puppeteer Page */
+	page?: Page | null;
+
+	/** Log successful requests to console */
+	successLogs?: boolean;
+
+	/** Log failed requests to console */
+	errorLogs?: boolean;
+
+	/** Block Ad requests (Ghostery engine) */
+	blockAds?: boolean;
+
+	/** Block Image requests */
+	blockImage?: boolean;
+
+	/** Intercept requests using Superagent (bypassing browser stack for intercepted resources) */
+	useGot?: boolean;
+
+	/** Use full URL for cache key (otherwise origin+pathname) */
+	useFullUrl?: boolean;
+
+	/** Enable caching for requests */
+	useCache?: boolean;
+
+	/** Data object for Doublelist message interception (POST logic) */
+	m4w_send_on_post?: Record<string, any> | null;
+
+	/** Data object for Doublelist message interception (Message logic) */
+	m4w_send_on_message?: Record<string, any> | null;
+
+	/** * Array of URL substrings. If a URL contains any of these strings,
+	 * it will NOT be blocked, even if blockImage is true.
+	 * Default includes: ["cdn-cgi/challenge-platform"]
+	 */
+	allowImagePatterns?: string[];
+
+	/** * Array of URL substrings. If a URL contains any of these strings,
+	 * the custom Superagent/Got fetch will be skipped, and the default
+	 * Playwright network stack will be used instead.
+	 */
+	skipGotPatterns?: string[];
+}
+
+/**
+ * Sets up request interception, caching, and ad-blocking on a Puppeteer page.
+ */
+export function ppRoute(options: PpRouteOptions): Promise<void>;
+
+/**
+ * Starts logging cache statistics to the console at a set interval.
+ * @param log_cache Optional NodeCache instance (uses global by default)
+ * @param interval Interval in seconds (default: 10)
+ */
+export function ppCacheLogs(log_cache?: any, interval?: number): void;

package/src/utility/puppeteer/routes/ppRoute.js

@@ -0,0 +1,326 @@
+// Import required libraries
+import superagent from "superagent";
+import { FiltersEngine, Request } from "@ghostery/adblocker";
+import fetch from "node-fetch";
+import NodeCache from "node-cache";
+
+let AdBlockEngine;
+
+// Create a NodeCache instance for caching responses
+// This helps reduce bandwidth usage and speed up repeated requests
+const globalCache = new NodeCache({
+	stdTTL: 3600, // Standard Time To Live in seconds for every generated cache element
+	checkperiod: 600, // The period in seconds for the automatic delete check interval
+	useClones: false, // Don't use clones for performance reasons
+});
+
+/**
+ * Function to start cache logging
+ * This outputs cache statistics (hits, misses, memory usage) to the console at a set interval.
+ * @param {Object} log_cache - The cache instance to monitor
+ * @param {number} interval - The interval in seconds between log outputs
+ */
+export function ppCacheLogs(log_cache = globalCache, interval = 10) {
+	if (interval > 0) {
+		setInterval(() => {
+			const stats = log_cache.stats;
+			const ksizeMB = (stats.ksize / (1024 * 1024)).toFixed(2);
+			const vsizeMB = (stats.vsize / (1024 * 1024)).toFixed(2);
+			const totalRequests = stats.hits + stats.misses;
+			const hitPercentage = totalRequests > 0 ? ((stats.hits / totalRequests) * 100).toFixed(2) : "0.00";
+			const missPercentage = totalRequests > 0 ? ((stats.misses / totalRequests) * 100).toFixed(2) : "0.00";
+
+			console.warn(
+				`Cache Stats: Success: ${stats.hits} (${hitPercentage}%), Misses: ${stats.misses} (${missPercentage}%), Keys: ${stats.keys}, Key: ${ksizeMB} MB, Value: ${vsizeMB} MB`
+			);
+		}, interval * 1000);
+	}
+}
+
+/**
+ * Function to fetch resources using Superagent library with optional caching.
+ * This mimics the browser's request but handles it in Node.js to allow caching or header manipulation.
+ * @param {boolean} useCache - Whether to use caching
+ * @param {string} url - The URL to fetch
+ * @param {Object} requestHeaders - Request headers from the original request
+ * @param {string} method - HTTP method (GET, POST, etc.)
+ * @param {boolean} useFullUrl - Whether to use the full URL as cache key or just origin+path
+ * @param {boolean} successLogs - Whether to log successful requests
+ * @param {boolean} errorLogs - Whether to log error requests
+ * @returns {Promise<Object>} - The response object containing status, headers, and body
+ */
+async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl, successLogs, errorLogs) {
+	// Determine the cache key based on configuration
+	let mainUrl = new URL(url).origin + new URL(url).pathname;
+	if (useFullUrl) {
+		mainUrl = url;
+	}
+
+	// Check if the response is cached
+	if (useCache) {
+		const cachedResponse = globalCache.get(mainUrl);
+		if (cachedResponse) {
+			if (successLogs) console.log(`Serving from globalCache: ${mainUrl}`);
+			return cachedResponse;
+		}
+	}
+
+	try {
+		// Fetch the resource using superagent
+		// buffer(true) ensures we get the raw binary data (essential for images/fonts)
+		const response = await superagent(method, url).set(requestHeaders).buffer(true);
+
+		// Determine the correct body type (Buffer for binary, text for others)
+		const responseBody = response.body instanceof Buffer ? response.body : response.text;
+
+		// Save to cache
+		globalCache.set(mainUrl, {
+			status: response.status,
+			headers: response.headers,
+			body: responseBody,
+		});
+		if (successLogs) console.log(`Success (cached): ${mainUrl}`);
+
+		return {
+			status: response.status,
+			headers: response.headers,
+			body: responseBody,
+		};
+	} catch (error) {
+		if (errorLogs) console.error(`Failed to fetch: ${url}`, error);
+		// We return undefined on error, which signals the route handler to fall back to normal request
+	}
+}
+
+/**
+ * Main function to set up routing, ad blocking, and request interception in Playwright.
+ * @param {Object} options - Configuration options
+ * @param {Object} options.context - Playwright context (optional, one is required)
+ * @param {Object} options.page - Playwright page (optional, one is required)
+ * @param {boolean} options.successLogs - Enable logging for successful fetches
+ * @param {boolean} options.errorLogs - Enable logging for failed fetches
+ * @param {boolean} options.blockImage - Enable global image blocking
+ * @param {boolean} options.blockAds - Enable Ghostery ad blocking
+ * @param {boolean} options.useGot - Enable custom fetching via Superagent (bypassing browser network stack for intercepted types)
+ * @param {boolean} options.useFullUrl - Use full URL for cache keys
+ * @param {boolean} options.useCache - Enable caching
+ * @param {Object} options.m4w_send_on_post - Custom handler data for Doublelist posts
+ * @param {Object} options.m4w_send_on_message - Custom handler data for Doublelist messages
+ * @param {Array<string>} options.allowImagePatterns - Array of strings/patterns. If a URL contains any of these, it will NOT be blocked even if blockImage is true.
+ * @param {Array<string>} options.skipGotPatterns - Array of strings/patterns. If a URL contains any of these, it will skip the custom Superagent fetch.
+ */
+export async function ppRoute({
+	page = null,
+	successLogs = false,
+	errorLogs = false,
+	blockAds = true,
+	blockImage = true,
+	useGot = true,
+	useFullUrl = true,
+	useCache = true,
+	m4w_send_on_post = null,
+	m4w_send_on_message = null,
+	allowImagePatterns = [], // Default empty, merged inside
+	skipGotPatterns = [], // Default empty, merged inside
+}) {
+	// Validation: Ensure we have a page
+	if (!page) {
+		throw new Error("A Puppeteer page must be provided.");
+	}
+
+	// --- SETUP: Merge Defaults for allowImagePatterns ---
+	// Always allow Cloudflare challenge platform images
+	const defaultAllowedPatterns = ["cdn-cgi/challenge-platform"];
+	const finalImagePatterns = [...defaultAllowedPatterns, ...allowImagePatterns];
+
+	// --- SETUP: Merge Defaults for skipGotPatterns ---
+	// Always skip custom fetch for Cloudflare challenges (let browser handle it)
+	const defaultSkipPatterns = [];
+	const finalSkipPatterns = [...defaultSkipPatterns, ...skipGotPatterns];
+
+	// Initialize ad blocking AdBlockEngine if enabled and not already loaded
+	if (blockAds && !AdBlockEngine) {
+		// console.log("Initializing AdBlockEngine..............................");
+		AdBlockEngine = await FiltersEngine.fromPrebuiltAdsAndTracking(fetch);
+		if (!AdBlockEngine) {
+			throw new Error("Failed to initialize AdBlockEngine.");
+		}
+	}
+
+	// Define resource types to intercept for custom fetching (useGot)
+	const interceptedResourceTypes = ["stylesheet", "script", "font"];
+
+	// If images are NOT blocked, we generally want to intercept/cache them too.
+	if (!blockImage) {
+		interceptedResourceTypes.push("image");
+	}
+
+	// Enable request interception in Puppeteer
+	await page.setRequestInterception(true);
+
+	// Set up the global route interception
+	page.on("request", async (request) => {
+		// Puppeteer best practice: do not handle request if already handled
+		if (request.isInterceptResolutionHandled()) return;
+
+		const url = request.url();
+		const method = request.method();
+		const resourceType = request.resourceType();
+
+		// ============================================================
+		// Group 1: Image Blocking
+		// ============================================================
+		if (blockImage && resourceType === "image") {
+			// Check against the merged list (defaults + user input)
+			const isAllowed = finalImagePatterns.some((pattern) => url.includes(pattern));
+
+			if (!isAllowed) {
+				await request.abort();
+				return;
+			}
+		}
+
+		// ============================================================
+		// Group 2: Ad Blocking
+		// ============================================================
+		if (blockAds && AdBlockEngine) {
+			const adsBlockResult = AdBlockEngine.match(
+				Request.fromRawDetails({
+					url: url,
+					type: resourceType,
+				})
+			);
+			if (adsBlockResult.match) {
+				await request.abort();
+				return;
+			}
+		}
+
+		// ============================================================
+		// Group 3: Specific Script Blocking (Cookie Law)
+		// ============================================================
+		if (url.includes("cdn.cookielaw.org")) {
+			await request.abort();
+			return;
+		}
+
+		// ============================================================
+		// Group 4: m4w_send_on_message Handling (Doublelist API)
+		// ============================================================
+		if (m4w_send_on_message && url.includes("api.doublelist.com/api/messages") && method === "POST") {
+			const headers = request.headers();
+			const postData = request.postData();
+			const urlParams = new URLSearchParams(postData);
+			const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
+			const parsedMessageJSON = JSON.parse(messageJSON);
+
+			m4w_send_on_message.sender_id = parsedMessageJSON.sender_id;
+			const authorizationHeader = headers["authorization"];
+			m4w_send_on_message.token_value = authorizationHeader.replace("Bearer ", "");
+
+			console.log("posts_send_message Blocked (Data Extracted)");
+			await request.abort();
+			return;
+		}
+
+		if (m4w_send_on_message && url.includes("https://doublelist.com/posts_send_message/")) {
+			console.log("posts_send_message Blocked");
+			await request.abort();
+			return;
+		}
+
+		// ============================================================
+		// Group 5: m4w_send_on_post Handling (Doublelist API)
+		// ============================================================
+		if (m4w_send_on_post && url.includes("api.doublelist.com/api/messages") && method === "POST") {
+			if (
+				m4w_send_on_post.current_post_id &&
+				m4w_send_on_post.current_post_id !== m4w_send_on_post.last_post_id &&
+				m4w_send_on_post.current_send_to &&
+				m4w_send_on_post.current_send_to != m4w_send_on_post.last_send_to
+			) {
+				m4w_send_on_post.last_send_to = m4w_send_on_post.current_send_to;
+				m4w_send_on_post.last_post_id = m4w_send_on_post.current_post_id;
+
+				let postData = request.postData();
+				const urlParams = new URLSearchParams(postData);
+				const messageJSON = decodeURIComponent(urlParams.get("messageJSON"));
+				let parsedMessageJSON = JSON.parse(messageJSON);
+
+				parsedMessageJSON.chat_user = m4w_send_on_post.current_send_to;
+				parsedMessageJSON.post_id = m4w_send_on_post.current_post_id;
+
+				urlParams.set("messageJSON", JSON.stringify(parsedMessageJSON));
+				const final_postData = urlParams.toString();
+
+				await request.continue({
+					postData: final_postData,
+				});
+				return;
+			} else {
+				console.log("send Message route abort (Duplicate or missing data)");
+				await request.abort();
+				return;
+			}
+		}
+
+		if (m4w_send_on_post && url.includes("https://doublelist.com/posts_send_message/")) {
+			console.log("posts_send_message Blocked");
+			await request.abort();
+			return;
+		}
+
+		if (m4w_send_on_post && url.includes("doublelist.com/messages/")) {
+			console.log("doublelist.com/messages/ Redirected");
+			await request.respond({
+				status: 302,
+				headers: {
+					location: "https://httpbin.org/ip",
+				},
+			});
+			return;
+		}
+
+		// ============================================================
+		// Group 6: Resource Interception (Custom Fetch/Cache)
+		// ============================================================
+		if (useGot && interceptedResourceTypes.includes(resourceType)) {
+			// Check against the merged list (defaults + user input)
+			const shouldSkipGot = finalSkipPatterns.some((pattern) => url.includes(pattern));
+
+			if (!shouldSkipGot) {
+				const requestHeaders = request.headers();
+				const requestMethod = request.method();
+
+				const response = await fetchWithClient(
+					useCache,
+					url,
+					requestHeaders,
+					requestMethod,
+					useFullUrl,
+					successLogs,
+					errorLogs
+				);
+
+				if (response) {
+					await request.respond({
+						status: response.status,
+						headers: response.headers,
+						body: response.body,
+					});
+					return;
+				} else {
+					if (errorLogs) console.log("Continuing with normal request (fetchWithClient returned null):", url);
+					await request.continue();
+					return;
+				}
+			}
+		}
+
+		// ============================================================
+		// Default: Continue with normal request
+		// ============================================================
+		await request.continue();
+		return;
+	});
+}