arn-browser 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +51 -6
- package/src/utility/playwright/pwHelper.d.ts +62 -0
- package/src/utility/playwright/pwHelper.js +84 -55
- package/src/utility/playwright/routes/pwRoute.d.ts +9 -5
- package/src/utility/playwright/routes/pwRoute.js +70 -21
- package/src/utility/puppeteer/ppHelper.d.ts +62 -0
- package/src/utility/puppeteer/ppHelper.js +74 -41
- package/src/utility/puppeteer/routes/ppRoute.d.ts +9 -5
- package/src/utility/puppeteer/routes/ppRoute.js +72 -24
package/package.json
CHANGED
package/src/index.d.ts
CHANGED
|
@@ -4,8 +4,8 @@ import { ppLaunch } from "./utility/puppeteer/ppLaunch";
|
|
|
4
4
|
import { ppRoute, ppCacheLogs } from "./utility/puppeteer/routes/ppRoute";
|
|
5
5
|
import { pwLaunch } from "./utility/playwright/pwLaunch";
|
|
6
6
|
import { pwRoute, pwCacheLogs } from "./utility/playwright/routes/pwRoute";
|
|
7
|
-
import { retryNavigation, retryClick, checkPageConditions
|
|
8
|
-
import { retryNavigation as ppRetryNavigation, retryClick as ppRetryClick, checkPageConditions as ppCheckPageConditions
|
|
7
|
+
import { retryNavigation, retryClick, checkPageConditions } from "./utility/playwright/pwHelper";
|
|
8
|
+
import { retryNavigation as ppRetryNavigation, retryClick as ppRetryClick, checkPageConditions as ppCheckPageConditions } from "./utility/puppeteer/ppHelper";
|
|
9
9
|
import { startProxyServer, fetchPublicIP, fetchProxyDetails } from "./utility/proxy-utility/proxy-chain";
|
|
10
10
|
import {
|
|
11
11
|
fetchAwsProxy, getInstanceStatus, getPublicIpAddress,
|
|
@@ -17,6 +17,31 @@ import {
|
|
|
17
17
|
import { get_multilogin_proxy, get_packetstream_proxy, get_x_proxy, fetchNextProxy } from "./utility/proxy-utility/proxy-helper";
|
|
18
18
|
import { generateOTP } from "./others/totp-generator";
|
|
19
19
|
|
|
20
|
+
/** A single cookie object (CDP format) */
|
|
21
|
+
interface SessionCookie {
|
|
22
|
+
name: string;
|
|
23
|
+
value: string;
|
|
24
|
+
domain?: string;
|
|
25
|
+
path?: string;
|
|
26
|
+
expires?: number;
|
|
27
|
+
httpOnly?: boolean;
|
|
28
|
+
secure?: boolean;
|
|
29
|
+
sameSite?: string;
|
|
30
|
+
[key: string]: any;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A single localStorage entry for one origin */
|
|
34
|
+
interface LocalStorageEntry {
|
|
35
|
+
origin: string;
|
|
36
|
+
items: { name: string; value: string }[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Session data result from export */
|
|
40
|
+
interface SessionData {
|
|
41
|
+
cookies?: SessionCookie[];
|
|
42
|
+
localStorage?: LocalStorageEntry[];
|
|
43
|
+
}
|
|
44
|
+
|
|
20
45
|
export declare const ppBrowser: {
|
|
21
46
|
launch: typeof ppLaunch;
|
|
22
47
|
route: typeof ppRoute;
|
|
@@ -24,8 +49,18 @@ export declare const ppBrowser: {
|
|
|
24
49
|
Goto: typeof ppRetryNavigation;
|
|
25
50
|
Click: typeof ppRetryClick;
|
|
26
51
|
Conditions: typeof ppCheckPageConditions;
|
|
27
|
-
exportSession:
|
|
28
|
-
|
|
52
|
+
exportSession: (options: {
|
|
53
|
+
page: import('puppeteer-core').Page;
|
|
54
|
+
cookies?: boolean;
|
|
55
|
+
localStorage?: boolean;
|
|
56
|
+
logger?: boolean;
|
|
57
|
+
}) => Promise<SessionData>;
|
|
58
|
+
importSession: (options: {
|
|
59
|
+
page: import('puppeteer-core').Page;
|
|
60
|
+
cookies?: SessionCookie[];
|
|
61
|
+
localStorage?: LocalStorageEntry[];
|
|
62
|
+
logger?: boolean;
|
|
63
|
+
}) => Promise<void>;
|
|
29
64
|
};
|
|
30
65
|
|
|
31
66
|
export declare const pwBrowser: {
|
|
@@ -35,8 +70,18 @@ export declare const pwBrowser: {
|
|
|
35
70
|
Goto: typeof retryNavigation;
|
|
36
71
|
Click: typeof retryClick;
|
|
37
72
|
Conditions: typeof checkPageConditions;
|
|
38
|
-
exportSession:
|
|
39
|
-
|
|
73
|
+
exportSession: (options: {
|
|
74
|
+
page: import('playwright-core').Page;
|
|
75
|
+
cookies?: boolean;
|
|
76
|
+
localStorage?: boolean;
|
|
77
|
+
logger?: boolean;
|
|
78
|
+
}) => Promise<SessionData>;
|
|
79
|
+
importSession: (options: {
|
|
80
|
+
page: import('playwright-core').Page;
|
|
81
|
+
cookies?: SessionCookie[];
|
|
82
|
+
localStorage?: LocalStorageEntry[];
|
|
83
|
+
logger?: boolean;
|
|
84
|
+
}) => Promise<void>;
|
|
40
85
|
};
|
|
41
86
|
|
|
42
87
|
export declare const proxyUtil: {
|
|
@@ -30,6 +30,55 @@ export interface RetryClickOptions {
|
|
|
30
30
|
timeout?: number;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/** A single cookie object (CDP format) */
|
|
34
|
+
export interface SessionCookie {
|
|
35
|
+
name: string;
|
|
36
|
+
value: string;
|
|
37
|
+
domain?: string;
|
|
38
|
+
path?: string;
|
|
39
|
+
expires?: number;
|
|
40
|
+
httpOnly?: boolean;
|
|
41
|
+
secure?: boolean;
|
|
42
|
+
sameSite?: string;
|
|
43
|
+
[key: string]: any;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A single localStorage entry for one origin */
|
|
47
|
+
export interface LocalStorageEntry {
|
|
48
|
+
origin: string;
|
|
49
|
+
items: { name: string; value: string }[];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Options for exportSession */
|
|
53
|
+
export interface ExportSessionOptions {
|
|
54
|
+
/** Playwright Page */
|
|
55
|
+
page: Page;
|
|
56
|
+
/** Export cookies. Default: true */
|
|
57
|
+
cookies?: boolean;
|
|
58
|
+
/** Export localStorage. Default: false */
|
|
59
|
+
localStorage?: boolean;
|
|
60
|
+
/** Log export progress. Default: true */
|
|
61
|
+
logger?: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Options for importSession */
|
|
65
|
+
export interface ImportSessionOptions {
|
|
66
|
+
/** Playwright Page */
|
|
67
|
+
page: Page;
|
|
68
|
+
/** Cookies to import */
|
|
69
|
+
cookies?: SessionCookie[];
|
|
70
|
+
/** localStorage entries to import */
|
|
71
|
+
localStorage?: LocalStorageEntry[];
|
|
72
|
+
/** Log import progress. Default: true */
|
|
73
|
+
logger?: boolean;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Session data result */
|
|
77
|
+
export interface SessionData {
|
|
78
|
+
cookies?: SessionCookie[];
|
|
79
|
+
localStorage?: LocalStorageEntry[];
|
|
80
|
+
}
|
|
81
|
+
|
|
33
82
|
/**
|
|
34
83
|
* Navigates to a URL with retry logic and incremental timeouts.
|
|
35
84
|
* If navigation fails, it temporarily goes to "about:blank" before retrying.
|
|
@@ -59,3 +108,16 @@ export function checkPageConditions(
|
|
|
59
108
|
checksToPerform: Record<string, string | Locator | null>,
|
|
60
109
|
timeout: number
|
|
61
110
|
): Promise<string | null>;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
114
|
+
* Silent operation — no browser blinking.
|
|
115
|
+
* cookies default: true, localStorage default: false
|
|
116
|
+
*/
|
|
117
|
+
export function exportSession(options: ExportSessionOptions): Promise<SessionData>;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Imports browser session (cookies and/or localStorage) into a Playwright Browser.
|
|
121
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
122
|
+
*/
|
|
123
|
+
export function importSession(options: ImportSessionOptions): Promise<void>;
|
|
@@ -129,81 +129,110 @@ export const checkPageConditions = async (page, checksToPerform, timeout) => {
|
|
|
129
129
|
};
|
|
130
130
|
|
|
131
131
|
/**
|
|
132
|
-
* Exports browser session (cookies
|
|
132
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
133
133
|
* Silent operation — no browser blinking.
|
|
134
134
|
*
|
|
135
|
-
* @param {
|
|
136
|
-
* @
|
|
135
|
+
* @param {Object} options
|
|
136
|
+
* @param {import('playwright-core').Page} options.page - Playwright Page
|
|
137
|
+
* @param {boolean} [options.cookies=true] - Export cookies
|
|
138
|
+
* @param {boolean} [options.localStorage=false] - Export localStorage
|
|
139
|
+
* @param {boolean} [options.logger=true] - Log export progress
|
|
140
|
+
* @returns {Promise<{cookies?: Array, localStorage?: Array}>} Session data for JSONB storage
|
|
137
141
|
*/
|
|
138
|
-
export const exportSession = async (
|
|
139
|
-
const
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
142
|
+
export const exportSession = async ({ page, cookies = true, localStorage = false, logger = true }) => {
|
|
143
|
+
const context = page.context();
|
|
144
|
+
const result = {};
|
|
145
|
+
|
|
146
|
+
if (cookies) {
|
|
147
|
+
const client = await context.newCDPSession(page);
|
|
148
|
+
const { cookies: allCookies } = await client.send("Network.getAllCookies");
|
|
149
|
+
result.cookies = allCookies;
|
|
150
|
+
await client.detach();
|
|
151
|
+
if (logger) console.log(`░░░░░ exportSession: ${allCookies.length} cookies exported`);
|
|
152
|
+
}
|
|
146
153
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
|
|
154
|
+
if (localStorage) {
|
|
155
|
+
result.localStorage = [];
|
|
156
|
+
const pages = context.pages();
|
|
157
|
+
for (const p of pages) {
|
|
158
|
+
const url = p.url();
|
|
159
|
+
if (!url || url === "about:blank" || url.startsWith("chrome")) continue;
|
|
160
|
+
|
|
161
|
+
try {
|
|
162
|
+
const origin = new URL(url).origin;
|
|
163
|
+
const items = await p.evaluate(() => {
|
|
164
|
+
const data = [];
|
|
165
|
+
for (let i = 0; i < window.localStorage.length; i++) {
|
|
166
|
+
const key = window.localStorage.key(i);
|
|
167
|
+
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
168
|
+
}
|
|
169
|
+
return data;
|
|
170
|
+
});
|
|
151
171
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const items = await p.evaluate(() => {
|
|
155
|
-
const data = [];
|
|
156
|
-
for (let i = 0; i < window.localStorage.length; i++) {
|
|
157
|
-
const key = window.localStorage.key(i);
|
|
158
|
-
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
172
|
+
if (items.length > 0 && !result.localStorage.some((o) => o.origin === origin)) {
|
|
173
|
+
result.localStorage.push({ origin, items });
|
|
159
174
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if (items.length > 0 && !localStorage.some((o) => o.origin === origin)) {
|
|
164
|
-
localStorage.push({ origin, items });
|
|
175
|
+
} catch (e) {
|
|
176
|
+
// Skip pages that can't be evaluated
|
|
165
177
|
}
|
|
166
|
-
} catch (e) {
|
|
167
|
-
// Skip pages that can't be evaluated
|
|
168
178
|
}
|
|
179
|
+
if (logger) console.log(`░░░░░ exportSession: ${result.localStorage.length} localStorage origins exported`);
|
|
169
180
|
}
|
|
170
181
|
|
|
171
|
-
return
|
|
182
|
+
return result;
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
/**
|
|
175
|
-
* Imports browser session (cookies
|
|
176
|
-
*
|
|
186
|
+
* Imports browser session (cookies and/or localStorage) into a Playwright Browser.
|
|
187
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
177
188
|
*
|
|
178
|
-
* @param {
|
|
179
|
-
* @param {
|
|
189
|
+
* @param {Object} options
|
|
190
|
+
* @param {import('playwright-core').Page} options.page - Playwright Page
|
|
191
|
+
* @param {Array} [options.cookies] - Cookies to import
|
|
192
|
+
* @param {Array} [options.localStorage] - localStorage entries to import
|
|
193
|
+
* @param {boolean} [options.logger=true] - Log import progress
|
|
180
194
|
*/
|
|
181
|
-
export const importSession = async (
|
|
182
|
-
const
|
|
183
|
-
const page = pages[0];
|
|
184
|
-
if (!page) throw new Error("No page available for importSession");
|
|
185
|
-
|
|
186
|
-
const client = await context.newCDPSession(page);
|
|
187
|
-
|
|
188
|
-
if (data.cookies?.length) {
|
|
189
|
-
await client.send("Network.setCookies", { cookies: data.cookies });
|
|
190
|
-
}
|
|
195
|
+
export const importSession = async ({ page, cookies, localStorage, logger = true }) => {
|
|
196
|
+
const context = page.context();
|
|
191
197
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
for (const entry of
|
|
198
|
+
// localStorage first — navigate to each origin and set items
|
|
199
|
+
if (localStorage?.length) {
|
|
200
|
+
for (const entry of localStorage) {
|
|
195
201
|
if (!entry.items?.length) continue;
|
|
196
202
|
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
await
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
value
|
|
203
|
-
|
|
203
|
+
const p = await context.newPage();
|
|
204
|
+
try {
|
|
205
|
+
await p.goto(entry.origin, { waitUntil: "load", timeout: 15000 });
|
|
206
|
+
await p.waitForLoadState("domcontentloaded");
|
|
207
|
+
await p.evaluate((items) => {
|
|
208
|
+
for (const { name, value } of items) {
|
|
209
|
+
window.localStorage.setItem(name, value);
|
|
210
|
+
}
|
|
211
|
+
}, entry.items);
|
|
212
|
+
} catch (e) {
|
|
213
|
+
// Origin may redirect — try to set localStorage on the final page
|
|
214
|
+
try {
|
|
215
|
+
await p.waitForLoadState("load", { timeout: 10000 });
|
|
216
|
+
await p.evaluate((items) => {
|
|
217
|
+
for (const { name, value } of items) {
|
|
218
|
+
window.localStorage.setItem(name, value);
|
|
219
|
+
}
|
|
220
|
+
}, entry.items);
|
|
221
|
+
} catch (_) {
|
|
222
|
+
// Skip this origin if it can't be reached
|
|
223
|
+
}
|
|
224
|
+
} finally {
|
|
225
|
+
await p.close();
|
|
204
226
|
}
|
|
205
227
|
}
|
|
228
|
+
if (logger) console.log(`░░░░░ importSession: ${localStorage.length} localStorage origins imported`);
|
|
206
229
|
}
|
|
207
230
|
|
|
208
|
-
|
|
231
|
+
// Cookies via CDP — silent, no navigation needed
|
|
232
|
+
if (cookies?.length) {
|
|
233
|
+
const client = await context.newCDPSession(page);
|
|
234
|
+
await client.send("Network.setCookies", { cookies });
|
|
235
|
+
await client.detach();
|
|
236
|
+
if (logger) console.log(`░░░░░ importSession: ${cookies.length} cookies imported`);
|
|
237
|
+
}
|
|
209
238
|
};
|
|
@@ -14,11 +14,8 @@ export interface PwRouteOptions {
|
|
|
14
14
|
/** Playwright Page (provide either context or page) */
|
|
15
15
|
page?: Page | null;
|
|
16
16
|
|
|
17
|
-
/** Log
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
/** Log failed requests to console */
|
|
21
|
-
errorLogs?: boolean;
|
|
17
|
+
/** Log level: "info" (success+error), "error" (errors only), false (no logs). Default: "error" */
|
|
18
|
+
logger?: "info" | "error" | false;
|
|
22
19
|
|
|
23
20
|
/** Block Ad requests (Ghostery engine) */
|
|
24
21
|
blockAds?: boolean;
|
|
@@ -35,6 +32,13 @@ export interface PwRouteOptions {
|
|
|
35
32
|
/** Enable caching for requests */
|
|
36
33
|
useCache?: boolean;
|
|
37
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Proxy for custom fetch requests (only used when useGot is true).
|
|
37
|
+
* String: "http://host:port", "socks5://user:pass@host:port"
|
|
38
|
+
* Object: { type, host, port, user, pass }
|
|
39
|
+
*/
|
|
40
|
+
proxy?: string | { type?: string; host: string; port: number; user?: string; pass?: string } | null;
|
|
41
|
+
|
|
38
42
|
/** Data object for Doublelist message interception (POST logic) */
|
|
39
43
|
m4w_send_on_post?: Record<string, any> | null;
|
|
40
44
|
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
import superagent from "superagent";
|
|
3
3
|
import { FiltersEngine, Request } from "@ghostery/adblocker";
|
|
4
4
|
import fetch from "node-fetch";
|
|
5
|
+
import { HttpsProxyAgent } from "https-proxy-agent";
|
|
6
|
+
import { SocksProxyAgent } from "socks-proxy-agent";
|
|
5
7
|
import NodeCache from "node-cache";
|
|
6
8
|
|
|
7
9
|
let AdBlockEngine;
|
|
@@ -37,6 +39,38 @@ export function pwCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Normalizes proxy input (string or object) into a URL string.
|
|
44
|
+
* Accepts:
|
|
45
|
+
* - String: "http://host:port", "socks5://user:pass@host:port", etc.
|
|
46
|
+
* - Object: { type, host, port, user, pass }
|
|
47
|
+
* @param {string|Object|null} proxy
|
|
48
|
+
* @returns {string|null} - Proxy URL string or null
|
|
49
|
+
*/
|
|
50
|
+
function formatProxyUrl(proxy) {
|
|
51
|
+
if (!proxy) return null;
|
|
52
|
+
if (typeof proxy === "string") return proxy;
|
|
53
|
+
if (typeof proxy === "object") {
|
|
54
|
+
const { type = "http", host, port, user, pass } = proxy;
|
|
55
|
+
const auth = user && pass ? `${user}:${pass}@` : "";
|
|
56
|
+
return `${type}://${auth}${host}:${port}`;
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Creates an HTTP agent for the given proxy URL.
|
|
63
|
+
* @param {string|null} proxyUrl
|
|
64
|
+
* @returns {Object|null} - HttpsProxyAgent or SocksProxyAgent instance
|
|
65
|
+
*/
|
|
66
|
+
function createProxyAgent(proxyUrl) {
|
|
67
|
+
if (!proxyUrl) return null;
|
|
68
|
+
if (proxyUrl.startsWith("socks")) {
|
|
69
|
+
return new SocksProxyAgent(proxyUrl);
|
|
70
|
+
}
|
|
71
|
+
return new HttpsProxyAgent(proxyUrl);
|
|
72
|
+
}
|
|
73
|
+
|
|
40
74
|
/**
|
|
41
75
|
* Function to fetch resources using Superagent library with optional caching.
|
|
42
76
|
* This mimics the browser's request but handles it in Node.js to allow caching or header manipulation.
|
|
@@ -45,11 +79,11 @@ export function pwCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
45
79
|
* @param {Object} requestHeaders - Request headers from the original request
|
|
46
80
|
* @param {string} method - HTTP method (GET, POST, etc.)
|
|
47
81
|
* @param {boolean} useFullUrl - Whether to use the full URL as cache key or just origin+path
|
|
48
|
-
* @param {
|
|
49
|
-
* @param {
|
|
82
|
+
* @param {string|false} logger - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
83
|
+
* @param {Object|null} proxyAgent - Proxy agent to use for the request
|
|
50
84
|
* @returns {Promise<Object>} - The response object containing status, headers, and body
|
|
51
85
|
*/
|
|
52
|
-
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl,
|
|
86
|
+
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl, logger, proxyAgent) {
|
|
53
87
|
// Determine the cache key based on configuration
|
|
54
88
|
let mainUrl = new URL(url).origin + new URL(url).pathname;
|
|
55
89
|
if (useFullUrl) {
|
|
@@ -60,7 +94,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
60
94
|
if (useCache) {
|
|
61
95
|
const cachedResponse = globalCache.get(mainUrl);
|
|
62
96
|
if (cachedResponse) {
|
|
63
|
-
if (
|
|
97
|
+
if (logger === "info") console.log(`Serving from globalCache: ${mainUrl}`);
|
|
64
98
|
return cachedResponse;
|
|
65
99
|
}
|
|
66
100
|
}
|
|
@@ -68,18 +102,29 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
68
102
|
try {
|
|
69
103
|
// Fetch the resource using superagent
|
|
70
104
|
// buffer(true) ensures we get the raw binary data (essential for images/fonts)
|
|
71
|
-
|
|
105
|
+
let request = superagent(method, url).set(requestHeaders).buffer(true);
|
|
106
|
+
|
|
107
|
+
// Apply proxy agent if provided
|
|
108
|
+
if (proxyAgent) {
|
|
109
|
+
request = request.agent(proxyAgent);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const response = await request;
|
|
72
113
|
|
|
73
114
|
// Determine the correct body type (Buffer for binary, text for others)
|
|
74
115
|
const responseBody = response.body instanceof Buffer ? response.body : response.text;
|
|
75
116
|
|
|
76
|
-
// Save to cache
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
117
|
+
// Save to cache only when caching is enabled
|
|
118
|
+
if (useCache) {
|
|
119
|
+
globalCache.set(mainUrl, {
|
|
120
|
+
status: response.status,
|
|
121
|
+
headers: response.headers,
|
|
122
|
+
body: responseBody,
|
|
123
|
+
});
|
|
124
|
+
if (logger === "info") console.log(`Success (cached): ${mainUrl}`);
|
|
125
|
+
} else {
|
|
126
|
+
if (logger === "info") console.log(`Success (not cached): ${mainUrl}`);
|
|
127
|
+
}
|
|
83
128
|
|
|
84
129
|
return {
|
|
85
130
|
status: response.status,
|
|
@@ -87,7 +132,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
87
132
|
body: responseBody,
|
|
88
133
|
};
|
|
89
134
|
} catch (error) {
|
|
90
|
-
if (
|
|
135
|
+
if (logger) console.error(`Failed to fetch: ${url}`, error);
|
|
91
136
|
// We return undefined on error, which signals the route handler to fall back to normal request
|
|
92
137
|
}
|
|
93
138
|
}
|
|
@@ -97,13 +142,13 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
97
142
|
* @param {Object} options - Configuration options
|
|
98
143
|
* @param {Object} options.context - Playwright context (optional, one is required)
|
|
99
144
|
* @param {Object} options.page - Playwright page (optional, one is required)
|
|
100
|
-
* @param {boolean} options.successLogs - Enable logging for successful fetches
|
|
101
|
-
* @param {boolean} options.errorLogs - Enable logging for failed fetches
|
|
102
145
|
* @param {boolean} options.blockImage - Enable global image blocking
|
|
103
146
|
* @param {boolean} options.blockAds - Enable Ghostery ad blocking
|
|
147
|
+
* @param {string|false} [options.logger="error"] - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
104
148
|
* @param {boolean} options.useGot - Enable custom fetching via Superagent (bypassing browser network stack for intercepted types)
|
|
105
149
|
* @param {boolean} options.useFullUrl - Use full URL for cache keys
|
|
106
150
|
* @param {boolean} options.useCache - Enable caching
|
|
151
|
+
* @param {string|Object|null} options.proxy - Proxy for custom fetch. String: "http://host:port" or "socks5://user:pass@host:port". Object: { type, host, port, user, pass }
|
|
107
152
|
* @param {Object} options.m4w_send_on_post - Custom handler data for Doublelist posts
|
|
108
153
|
* @param {Object} options.m4w_send_on_message - Custom handler data for Doublelist messages
|
|
109
154
|
* @param {Array<string>} options.allowImagePatterns - Array of strings/patterns. If a URL contains any of these, it will NOT be blocked even if blockImage is true.
|
|
@@ -112,13 +157,13 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
112
157
|
export async function pwRoute({
|
|
113
158
|
context = null,
|
|
114
159
|
page = null,
|
|
115
|
-
|
|
116
|
-
errorLogs = false,
|
|
160
|
+
logger = false,
|
|
117
161
|
blockAds = true,
|
|
118
162
|
blockImage = true,
|
|
119
|
-
useGot =
|
|
163
|
+
useGot = false,
|
|
120
164
|
useFullUrl = true,
|
|
121
165
|
useCache = true,
|
|
166
|
+
proxy = null,
|
|
122
167
|
m4w_send_on_post = null,
|
|
123
168
|
m4w_send_on_message = null,
|
|
124
169
|
allowImagePatterns = [], // Default empty, merged inside
|
|
@@ -152,6 +197,10 @@ export async function pwRoute({
|
|
|
152
197
|
// Define resource types to intercept for custom fetching (useGot)
|
|
153
198
|
const interceptedResourceTypes = ["stylesheet", "script", "font"];
|
|
154
199
|
|
|
200
|
+
// Create proxy agent once (reused for all requests in this route)
|
|
201
|
+
const proxyUrl = formatProxyUrl(proxy);
|
|
202
|
+
const proxyAgent = createProxyAgent(proxyUrl);
|
|
203
|
+
|
|
155
204
|
// If images are NOT blocked, we generally want to intercept/cache them too.
|
|
156
205
|
if (!blockImage) {
|
|
157
206
|
interceptedResourceTypes.push("image");
|
|
@@ -294,8 +343,8 @@ export async function pwRoute({
|
|
|
294
343
|
requestHeaders,
|
|
295
344
|
requestMethod,
|
|
296
345
|
useFullUrl,
|
|
297
|
-
|
|
298
|
-
|
|
346
|
+
logger,
|
|
347
|
+
proxyAgent
|
|
299
348
|
);
|
|
300
349
|
|
|
301
350
|
if (response) {
|
|
@@ -306,7 +355,7 @@ export async function pwRoute({
|
|
|
306
355
|
});
|
|
307
356
|
return;
|
|
308
357
|
} else {
|
|
309
|
-
if (
|
|
358
|
+
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
310
359
|
await route.continue();
|
|
311
360
|
return;
|
|
312
361
|
}
|
|
@@ -34,6 +34,55 @@ export interface RetryClickOptions {
|
|
|
34
34
|
timeout?: number;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/** A single cookie object (CDP format) */
|
|
38
|
+
export interface SessionCookie {
|
|
39
|
+
name: string;
|
|
40
|
+
value: string;
|
|
41
|
+
domain?: string;
|
|
42
|
+
path?: string;
|
|
43
|
+
expires?: number;
|
|
44
|
+
httpOnly?: boolean;
|
|
45
|
+
secure?: boolean;
|
|
46
|
+
sameSite?: string;
|
|
47
|
+
[key: string]: any;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A single localStorage entry for one origin */
|
|
51
|
+
export interface LocalStorageEntry {
|
|
52
|
+
origin: string;
|
|
53
|
+
items: { name: string; value: string }[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Options for exportSession */
|
|
57
|
+
export interface ExportSessionOptions {
|
|
58
|
+
/** Any Puppeteer Page */
|
|
59
|
+
page: Page;
|
|
60
|
+
/** Export cookies. Default: true */
|
|
61
|
+
cookies?: boolean;
|
|
62
|
+
/** Export localStorage. Default: false */
|
|
63
|
+
localStorage?: boolean;
|
|
64
|
+
/** Log export progress. Default: true */
|
|
65
|
+
logger?: boolean;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Options for importSession */
|
|
69
|
+
export interface ImportSessionOptions {
|
|
70
|
+
/** Any Puppeteer Page */
|
|
71
|
+
page: Page;
|
|
72
|
+
/** Cookies to import */
|
|
73
|
+
cookies?: SessionCookie[];
|
|
74
|
+
/** localStorage entries to import */
|
|
75
|
+
localStorage?: LocalStorageEntry[];
|
|
76
|
+
/** Log import progress. Default: true */
|
|
77
|
+
logger?: boolean;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Session data result */
|
|
81
|
+
export interface SessionData {
|
|
82
|
+
cookies?: SessionCookie[];
|
|
83
|
+
localStorage?: LocalStorageEntry[];
|
|
84
|
+
}
|
|
85
|
+
|
|
37
86
|
/**
|
|
38
87
|
* Navigates to a URL with retry logic and incremental timeouts.
|
|
39
88
|
* If navigation fails, it temporarily goes to "about:blank" before retrying.
|
|
@@ -68,3 +117,16 @@ export function checkPageConditions(
|
|
|
68
117
|
checksToPerform: Record<string, string | Locator<Node> | null>,
|
|
69
118
|
timeout: number
|
|
70
119
|
): Promise<string | null>;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
123
|
+
* Silent operation — no browser blinking.
|
|
124
|
+
* cookies default: true, localStorage default: false
|
|
125
|
+
*/
|
|
126
|
+
export function exportSession(options: ExportSessionOptions): Promise<SessionData>;
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Imports browser session (cookies and/or localStorage) into a Puppeteer Browser.
|
|
130
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
131
|
+
*/
|
|
132
|
+
export function importSession(options: ImportSessionOptions): Promise<void>;
|
|
@@ -151,77 +151,110 @@ export const checkPageConditions = async (page, checksToPerform, timeout) => {
|
|
|
151
151
|
};
|
|
152
152
|
|
|
153
153
|
/**
|
|
154
|
-
* Exports browser session (cookies
|
|
154
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
155
155
|
* Silent operation — no browser blinking.
|
|
156
156
|
*
|
|
157
|
-
* @param {
|
|
158
|
-
* @
|
|
157
|
+
* @param {Object} options
|
|
158
|
+
* @param {import('puppeteer-core').Page} options.page - Any Puppeteer Page
|
|
159
|
+
* @param {boolean} [options.cookies=true] - Export cookies
|
|
160
|
+
* @param {boolean} [options.localStorage=false] - Export localStorage
|
|
161
|
+
* @param {boolean} [options.logger=true] - Log export progress
|
|
162
|
+
* @returns {Promise<{cookies?: Array, localStorage?: Array}>} Session data for JSONB storage
|
|
159
163
|
*/
|
|
160
|
-
export const exportSession = async (page) => {
|
|
161
|
-
const
|
|
162
|
-
const { cookies } = await client.send("Network.getAllCookies");
|
|
163
|
-
await client.detach();
|
|
164
|
+
export const exportSession = async ({ page, cookies = true, localStorage = false, logger = true }) => {
|
|
165
|
+
const result = {};
|
|
164
166
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
167
|
+
if (cookies) {
|
|
168
|
+
const client = await page.createCDPSession();
|
|
169
|
+
const { cookies: allCookies } = await client.send("Network.getAllCookies");
|
|
170
|
+
result.cookies = allCookies;
|
|
171
|
+
await client.detach();
|
|
172
|
+
if (logger) console.log(`░░░░░ exportSession: ${allCookies.length} cookies exported`);
|
|
173
|
+
}
|
|
168
174
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
175
|
+
if (localStorage) {
|
|
176
|
+
result.localStorage = [];
|
|
177
|
+
const browser = page.browser();
|
|
178
|
+
const pages = await browser.pages();
|
|
172
179
|
|
|
173
|
-
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
180
|
+
for (const p of pages) {
|
|
181
|
+
const url = p.url();
|
|
182
|
+
if (!url || url === "about:blank" || url.startsWith("chrome")) continue;
|
|
183
|
+
|
|
184
|
+
try {
|
|
185
|
+
const origin = new URL(url).origin;
|
|
186
|
+
const items = await p.evaluate(() => {
|
|
187
|
+
const data = [];
|
|
188
|
+
for (let i = 0; i < window.localStorage.length; i++) {
|
|
189
|
+
const key = window.localStorage.key(i);
|
|
190
|
+
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
191
|
+
}
|
|
192
|
+
return data;
|
|
193
|
+
});
|
|
183
194
|
|
|
184
|
-
|
|
185
|
-
|
|
195
|
+
if (items.length > 0 && !result.localStorage.some((o) => o.origin === origin)) {
|
|
196
|
+
result.localStorage.push({ origin, items });
|
|
197
|
+
}
|
|
198
|
+
} catch (e) {
|
|
199
|
+
// Skip pages that can't be evaluated
|
|
186
200
|
}
|
|
187
|
-
} catch (e) {
|
|
188
|
-
// Skip pages that can't be evaluated
|
|
189
201
|
}
|
|
202
|
+
if (logger) console.log(`░░░░░ exportSession: ${result.localStorage.length} localStorage origins exported`);
|
|
190
203
|
}
|
|
191
204
|
|
|
192
|
-
return
|
|
205
|
+
return result;
|
|
193
206
|
};
|
|
194
207
|
|
|
195
208
|
/**
|
|
196
|
-
* Imports browser session (cookies
|
|
209
|
+
* Imports browser session (cookies and/or localStorage) into a Puppeteer Browser.
|
|
210
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
197
211
|
*
|
|
198
|
-
* @param {
|
|
199
|
-
* @param {
|
|
212
|
+
* @param {Object} options
|
|
213
|
+
* @param {import('puppeteer-core').Page} options.page - Any Puppeteer Page
|
|
214
|
+
* @param {Array} [options.cookies] - Cookies to import
|
|
215
|
+
* @param {Array} [options.localStorage] - localStorage entries to import
|
|
216
|
+
* @param {boolean} [options.logger=true] - Log import progress
|
|
200
217
|
*/
|
|
201
|
-
export const importSession = async (page,
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
await client.send("Network.setCookies", { cookies: data.cookies });
|
|
205
|
-
await client.detach();
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
if (data.localStorage?.length) {
|
|
218
|
+
export const importSession = async ({ page, cookies, localStorage, logger = true }) => {
|
|
219
|
+
// localStorage first — navigate to each origin and set items
|
|
220
|
+
if (localStorage?.length) {
|
|
209
221
|
const browser = page.browser();
|
|
210
222
|
|
|
211
|
-
for (const entry of
|
|
223
|
+
for (const entry of localStorage) {
|
|
212
224
|
if (!entry.items?.length) continue;
|
|
213
225
|
|
|
214
226
|
const p = await browser.newPage();
|
|
215
227
|
try {
|
|
216
|
-
await p.goto(entry.origin, { waitUntil: "
|
|
228
|
+
await p.goto(entry.origin, { waitUntil: "load", timeout: 15000 });
|
|
217
229
|
await p.evaluate((items) => {
|
|
218
230
|
for (const { name, value } of items) {
|
|
219
231
|
window.localStorage.setItem(name, value);
|
|
220
232
|
}
|
|
221
233
|
}, entry.items);
|
|
234
|
+
} catch (e) {
|
|
235
|
+
// Origin may redirect — retry after load
|
|
236
|
+
try {
|
|
237
|
+
await p.waitForNavigation({ waitUntil: "load", timeout: 10000 }).catch(() => {});
|
|
238
|
+
await p.evaluate((items) => {
|
|
239
|
+
for (const { name, value } of items) {
|
|
240
|
+
window.localStorage.setItem(name, value);
|
|
241
|
+
}
|
|
242
|
+
}, entry.items);
|
|
243
|
+
} catch (_) {
|
|
244
|
+
// Skip this origin
|
|
245
|
+
}
|
|
222
246
|
} finally {
|
|
223
247
|
await p.close();
|
|
224
248
|
}
|
|
225
249
|
}
|
|
250
|
+
if (logger) console.log(`░░░░░ importSession: ${localStorage.length} localStorage origins imported`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Cookies via CDP — silent, no navigation needed
|
|
254
|
+
if (cookies?.length) {
|
|
255
|
+
const client = await page.createCDPSession();
|
|
256
|
+
await client.send("Network.setCookies", { cookies });
|
|
257
|
+
await client.detach();
|
|
258
|
+
if (logger) console.log(`░░░░░ importSession: ${cookies.length} cookies imported`);
|
|
226
259
|
}
|
|
227
260
|
};
|
|
@@ -11,11 +11,8 @@ export interface PpRouteOptions {
|
|
|
11
11
|
/** Puppeteer Page */
|
|
12
12
|
page?: Page | null;
|
|
13
13
|
|
|
14
|
-
/** Log
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
/** Log failed requests to console */
|
|
18
|
-
errorLogs?: boolean;
|
|
14
|
+
/** Log level: "info" (success+error), "error" (errors only), false (no logs). Default: "error" */
|
|
15
|
+
logger?: "info" | "error" | false;
|
|
19
16
|
|
|
20
17
|
/** Block Ad requests (Ghostery engine) */
|
|
21
18
|
blockAds?: boolean;
|
|
@@ -32,6 +29,13 @@ export interface PpRouteOptions {
|
|
|
32
29
|
/** Enable caching for requests */
|
|
33
30
|
useCache?: boolean;
|
|
34
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Proxy for custom fetch requests (only used when useGot is true).
|
|
34
|
+
* String: "http://host:port", "socks5://user:pass@host:port"
|
|
35
|
+
* Object: { type, host, port, user, pass }
|
|
36
|
+
*/
|
|
37
|
+
proxy?: string | { type?: string; host: string; port: number; user?: string; pass?: string } | null;
|
|
38
|
+
|
|
35
39
|
/** Data object for Doublelist message interception (POST logic) */
|
|
36
40
|
m4w_send_on_post?: Record<string, any> | null;
|
|
37
41
|
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
import superagent from "superagent";
|
|
3
3
|
import { FiltersEngine, Request } from "@ghostery/adblocker";
|
|
4
4
|
import fetch from "node-fetch";
|
|
5
|
+
import { HttpsProxyAgent } from "https-proxy-agent";
|
|
6
|
+
import { SocksProxyAgent } from "socks-proxy-agent";
|
|
5
7
|
import NodeCache from "node-cache";
|
|
6
8
|
|
|
7
9
|
let AdBlockEngine;
|
|
@@ -37,6 +39,38 @@ export function ppCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Normalizes proxy input (string or object) into a URL string.
|
|
44
|
+
* Accepts:
|
|
45
|
+
* - String: "http://host:port", "socks5://user:pass@host:port", etc.
|
|
46
|
+
* - Object: { type, host, port, user, pass }
|
|
47
|
+
* @param {string|Object|null} proxy
|
|
48
|
+
* @returns {string|null} - Proxy URL string or null
|
|
49
|
+
*/
|
|
50
|
+
function formatProxyUrl(proxy) {
|
|
51
|
+
if (!proxy) return null;
|
|
52
|
+
if (typeof proxy === "string") return proxy;
|
|
53
|
+
if (typeof proxy === "object") {
|
|
54
|
+
const { type = "http", host, port, user, pass } = proxy;
|
|
55
|
+
const auth = user && pass ? `${user}:${pass}@` : "";
|
|
56
|
+
return `${type}://${auth}${host}:${port}`;
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Creates an HTTP agent for the given proxy URL.
|
|
63
|
+
* @param {string|null} proxyUrl
|
|
64
|
+
* @returns {Object|null} - HttpsProxyAgent or SocksProxyAgent instance
|
|
65
|
+
*/
|
|
66
|
+
function createProxyAgent(proxyUrl) {
|
|
67
|
+
if (!proxyUrl) return null;
|
|
68
|
+
if (proxyUrl.startsWith("socks")) {
|
|
69
|
+
return new SocksProxyAgent(proxyUrl);
|
|
70
|
+
}
|
|
71
|
+
return new HttpsProxyAgent(proxyUrl);
|
|
72
|
+
}
|
|
73
|
+
|
|
40
74
|
/**
|
|
41
75
|
* Function to fetch resources using Superagent library with optional caching.
|
|
42
76
|
* This mimics the browser's request but handles it in Node.js to allow caching or header manipulation.
|
|
@@ -45,11 +79,11 @@ export function ppCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
45
79
|
* @param {Object} requestHeaders - Request headers from the original request
|
|
46
80
|
* @param {string} method - HTTP method (GET, POST, etc.)
|
|
47
81
|
* @param {boolean} useFullUrl - Whether to use the full URL as cache key or just origin+path
|
|
48
|
-
* @param {
|
|
49
|
-
* @param {
|
|
82
|
+
* @param {string|false} logger - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
83
|
+
* @param {Object|null} proxyAgent - Proxy agent to use for the request
|
|
50
84
|
* @returns {Promise<Object>} - The response object containing status, headers, and body
|
|
51
85
|
*/
|
|
52
|
-
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl,
|
|
86
|
+
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl, logger, proxyAgent) {
|
|
53
87
|
// Determine the cache key based on configuration
|
|
54
88
|
let mainUrl = new URL(url).origin + new URL(url).pathname;
|
|
55
89
|
if (useFullUrl) {
|
|
@@ -60,7 +94,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
60
94
|
if (useCache) {
|
|
61
95
|
const cachedResponse = globalCache.get(mainUrl);
|
|
62
96
|
if (cachedResponse) {
|
|
63
|
-
if (
|
|
97
|
+
if (logger === "info") console.log(`Serving from globalCache: ${mainUrl}`);
|
|
64
98
|
return cachedResponse;
|
|
65
99
|
}
|
|
66
100
|
}
|
|
@@ -68,18 +102,29 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
68
102
|
try {
|
|
69
103
|
// Fetch the resource using superagent
|
|
70
104
|
// buffer(true) ensures we get the raw binary data (essential for images/fonts)
|
|
71
|
-
|
|
105
|
+
let request = superagent(method, url).set(requestHeaders).buffer(true);
|
|
106
|
+
|
|
107
|
+
// Apply proxy agent if provided
|
|
108
|
+
if (proxyAgent) {
|
|
109
|
+
request = request.agent(proxyAgent);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const response = await request;
|
|
72
113
|
|
|
73
114
|
// Determine the correct body type (Buffer for binary, text for others)
|
|
74
115
|
const responseBody = response.body instanceof Buffer ? response.body : response.text;
|
|
75
116
|
|
|
76
|
-
// Save to cache
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
117
|
+
// Save to cache only when caching is enabled
|
|
118
|
+
if (useCache) {
|
|
119
|
+
globalCache.set(mainUrl, {
|
|
120
|
+
status: response.status,
|
|
121
|
+
headers: response.headers,
|
|
122
|
+
body: responseBody,
|
|
123
|
+
});
|
|
124
|
+
if (logger === "info") console.log(`Success (cached): ${mainUrl}`);
|
|
125
|
+
} else {
|
|
126
|
+
if (logger === "info") console.log(`Success (not cached): ${mainUrl}`);
|
|
127
|
+
}
|
|
83
128
|
|
|
84
129
|
return {
|
|
85
130
|
status: response.status,
|
|
@@ -87,23 +132,22 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
87
132
|
body: responseBody,
|
|
88
133
|
};
|
|
89
134
|
} catch (error) {
|
|
90
|
-
if (
|
|
135
|
+
if (logger) console.error(`Failed to fetch: ${url}`, error);
|
|
91
136
|
// We return undefined on error, which signals the route handler to fall back to normal request
|
|
92
137
|
}
|
|
93
138
|
}
|
|
94
139
|
|
|
95
140
|
/**
|
|
96
|
-
* Main function to set up routing, ad blocking, and request interception in
|
|
141
|
+
* Main function to set up routing, ad blocking, and request interception in Puppeteer.
|
|
97
142
|
* @param {Object} options - Configuration options
|
|
98
|
-
* @param {Object} options.
|
|
99
|
-
* @param {Object} options.page - Playwright page (optional, one is required)
|
|
100
|
-
* @param {boolean} options.successLogs - Enable logging for successful fetches
|
|
101
|
-
* @param {boolean} options.errorLogs - Enable logging for failed fetches
|
|
143
|
+
* @param {Object} options.page - Puppeteer page (required)
|
|
102
144
|
* @param {boolean} options.blockImage - Enable global image blocking
|
|
103
145
|
* @param {boolean} options.blockAds - Enable Ghostery ad blocking
|
|
146
|
+
* @param {string|false} [options.logger="error"] - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
104
147
|
* @param {boolean} options.useGot - Enable custom fetching via Superagent (bypassing browser network stack for intercepted types)
|
|
105
148
|
* @param {boolean} options.useFullUrl - Use full URL for cache keys
|
|
106
149
|
* @param {boolean} options.useCache - Enable caching
|
|
150
|
+
* @param {string|Object|null} options.proxy - Proxy for custom fetch. String: "http://host:port" or "socks5://user:pass@host:port". Object: { type, host, port, user, pass }
|
|
107
151
|
* @param {Object} options.m4w_send_on_post - Custom handler data for Doublelist posts
|
|
108
152
|
* @param {Object} options.m4w_send_on_message - Custom handler data for Doublelist messages
|
|
109
153
|
* @param {Array<string>} options.allowImagePatterns - Array of strings/patterns. If a URL contains any of these, it will NOT be blocked even if blockImage is true.
|
|
@@ -111,13 +155,13 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
111
155
|
*/
|
|
112
156
|
export async function ppRoute({
|
|
113
157
|
page = null,
|
|
114
|
-
|
|
115
|
-
errorLogs = false,
|
|
158
|
+
logger = false,
|
|
116
159
|
blockAds = true,
|
|
117
160
|
blockImage = true,
|
|
118
|
-
useGot =
|
|
161
|
+
useGot = false,
|
|
119
162
|
useFullUrl = true,
|
|
120
163
|
useCache = true,
|
|
164
|
+
proxy = null,
|
|
121
165
|
m4w_send_on_post = null,
|
|
122
166
|
m4w_send_on_message = null,
|
|
123
167
|
allowImagePatterns = [], // Default empty, merged inside
|
|
@@ -150,6 +194,10 @@ export async function ppRoute({
|
|
|
150
194
|
// Define resource types to intercept for custom fetching (useGot)
|
|
151
195
|
const interceptedResourceTypes = ["stylesheet", "script", "font"];
|
|
152
196
|
|
|
197
|
+
// Create proxy agent once (reused for all requests in this route)
|
|
198
|
+
const proxyUrl = formatProxyUrl(proxy);
|
|
199
|
+
const proxyAgent = createProxyAgent(proxyUrl);
|
|
200
|
+
|
|
153
201
|
// If images are NOT blocked, we generally want to intercept/cache them too.
|
|
154
202
|
if (!blockImage) {
|
|
155
203
|
interceptedResourceTypes.push("image");
|
|
@@ -298,8 +346,8 @@ export async function ppRoute({
|
|
|
298
346
|
requestHeaders,
|
|
299
347
|
requestMethod,
|
|
300
348
|
useFullUrl,
|
|
301
|
-
|
|
302
|
-
|
|
349
|
+
logger,
|
|
350
|
+
proxyAgent
|
|
303
351
|
);
|
|
304
352
|
|
|
305
353
|
if (response) {
|
|
@@ -310,7 +358,7 @@ export async function ppRoute({
|
|
|
310
358
|
});
|
|
311
359
|
return;
|
|
312
360
|
} else {
|
|
313
|
-
if (
|
|
361
|
+
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
314
362
|
await request.continue();
|
|
315
363
|
return;
|
|
316
364
|
}
|