arn-browser 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +51 -6
- package/src/utility/playwright/pwHelper.d.ts +62 -0
- package/src/utility/playwright/pwHelper.js +84 -55
- package/src/utility/playwright/routes/pwRoute.d.ts +2 -5
- package/src/utility/playwright/routes/pwRoute.js +9 -13
- package/src/utility/puppeteer/ppHelper.d.ts +62 -0
- package/src/utility/puppeteer/ppHelper.js +74 -41
- package/src/utility/puppeteer/routes/ppRoute.d.ts +2 -5
- package/src/utility/puppeteer/routes/ppRoute.js +9 -13
package/package.json
CHANGED
package/src/index.d.ts
CHANGED
|
@@ -4,8 +4,8 @@ import { ppLaunch } from "./utility/puppeteer/ppLaunch";
|
|
|
4
4
|
import { ppRoute, ppCacheLogs } from "./utility/puppeteer/routes/ppRoute";
|
|
5
5
|
import { pwLaunch } from "./utility/playwright/pwLaunch";
|
|
6
6
|
import { pwRoute, pwCacheLogs } from "./utility/playwright/routes/pwRoute";
|
|
7
|
-
import { retryNavigation, retryClick, checkPageConditions
|
|
8
|
-
import { retryNavigation as ppRetryNavigation, retryClick as ppRetryClick, checkPageConditions as ppCheckPageConditions
|
|
7
|
+
import { retryNavigation, retryClick, checkPageConditions } from "./utility/playwright/pwHelper";
|
|
8
|
+
import { retryNavigation as ppRetryNavigation, retryClick as ppRetryClick, checkPageConditions as ppCheckPageConditions } from "./utility/puppeteer/ppHelper";
|
|
9
9
|
import { startProxyServer, fetchPublicIP, fetchProxyDetails } from "./utility/proxy-utility/proxy-chain";
|
|
10
10
|
import {
|
|
11
11
|
fetchAwsProxy, getInstanceStatus, getPublicIpAddress,
|
|
@@ -17,6 +17,31 @@ import {
|
|
|
17
17
|
import { get_multilogin_proxy, get_packetstream_proxy, get_x_proxy, fetchNextProxy } from "./utility/proxy-utility/proxy-helper";
|
|
18
18
|
import { generateOTP } from "./others/totp-generator";
|
|
19
19
|
|
|
20
|
+
/** A single cookie object (CDP format) */
|
|
21
|
+
interface SessionCookie {
|
|
22
|
+
name: string;
|
|
23
|
+
value: string;
|
|
24
|
+
domain?: string;
|
|
25
|
+
path?: string;
|
|
26
|
+
expires?: number;
|
|
27
|
+
httpOnly?: boolean;
|
|
28
|
+
secure?: boolean;
|
|
29
|
+
sameSite?: string;
|
|
30
|
+
[key: string]: any;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A single localStorage entry for one origin */
|
|
34
|
+
interface LocalStorageEntry {
|
|
35
|
+
origin: string;
|
|
36
|
+
items: { name: string; value: string }[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Session data result from export */
|
|
40
|
+
interface SessionData {
|
|
41
|
+
cookies?: SessionCookie[];
|
|
42
|
+
localStorage?: LocalStorageEntry[];
|
|
43
|
+
}
|
|
44
|
+
|
|
20
45
|
export declare const ppBrowser: {
|
|
21
46
|
launch: typeof ppLaunch;
|
|
22
47
|
route: typeof ppRoute;
|
|
@@ -24,8 +49,18 @@ export declare const ppBrowser: {
|
|
|
24
49
|
Goto: typeof ppRetryNavigation;
|
|
25
50
|
Click: typeof ppRetryClick;
|
|
26
51
|
Conditions: typeof ppCheckPageConditions;
|
|
27
|
-
exportSession:
|
|
28
|
-
|
|
52
|
+
exportSession: (options: {
|
|
53
|
+
page: import('puppeteer-core').Page;
|
|
54
|
+
cookies?: boolean;
|
|
55
|
+
localStorage?: boolean;
|
|
56
|
+
logger?: boolean;
|
|
57
|
+
}) => Promise<SessionData>;
|
|
58
|
+
importSession: (options: {
|
|
59
|
+
page: import('puppeteer-core').Page;
|
|
60
|
+
cookies?: SessionCookie[];
|
|
61
|
+
localStorage?: LocalStorageEntry[];
|
|
62
|
+
logger?: boolean;
|
|
63
|
+
}) => Promise<void>;
|
|
29
64
|
};
|
|
30
65
|
|
|
31
66
|
export declare const pwBrowser: {
|
|
@@ -35,8 +70,18 @@ export declare const pwBrowser: {
|
|
|
35
70
|
Goto: typeof retryNavigation;
|
|
36
71
|
Click: typeof retryClick;
|
|
37
72
|
Conditions: typeof checkPageConditions;
|
|
38
|
-
exportSession:
|
|
39
|
-
|
|
73
|
+
exportSession: (options: {
|
|
74
|
+
page: import('playwright-core').Page;
|
|
75
|
+
cookies?: boolean;
|
|
76
|
+
localStorage?: boolean;
|
|
77
|
+
logger?: boolean;
|
|
78
|
+
}) => Promise<SessionData>;
|
|
79
|
+
importSession: (options: {
|
|
80
|
+
page: import('playwright-core').Page;
|
|
81
|
+
cookies?: SessionCookie[];
|
|
82
|
+
localStorage?: LocalStorageEntry[];
|
|
83
|
+
logger?: boolean;
|
|
84
|
+
}) => Promise<void>;
|
|
40
85
|
};
|
|
41
86
|
|
|
42
87
|
export declare const proxyUtil: {
|
|
@@ -30,6 +30,55 @@ export interface RetryClickOptions {
|
|
|
30
30
|
timeout?: number;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/** A single cookie object (CDP format) */
|
|
34
|
+
export interface SessionCookie {
|
|
35
|
+
name: string;
|
|
36
|
+
value: string;
|
|
37
|
+
domain?: string;
|
|
38
|
+
path?: string;
|
|
39
|
+
expires?: number;
|
|
40
|
+
httpOnly?: boolean;
|
|
41
|
+
secure?: boolean;
|
|
42
|
+
sameSite?: string;
|
|
43
|
+
[key: string]: any;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A single localStorage entry for one origin */
|
|
47
|
+
export interface LocalStorageEntry {
|
|
48
|
+
origin: string;
|
|
49
|
+
items: { name: string; value: string }[];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Options for exportSession */
|
|
53
|
+
export interface ExportSessionOptions {
|
|
54
|
+
/** Playwright Page */
|
|
55
|
+
page: Page;
|
|
56
|
+
/** Export cookies. Default: true */
|
|
57
|
+
cookies?: boolean;
|
|
58
|
+
/** Export localStorage. Default: false */
|
|
59
|
+
localStorage?: boolean;
|
|
60
|
+
/** Log export progress. Default: true */
|
|
61
|
+
logger?: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Options for importSession */
|
|
65
|
+
export interface ImportSessionOptions {
|
|
66
|
+
/** Playwright Page */
|
|
67
|
+
page: Page;
|
|
68
|
+
/** Cookies to import */
|
|
69
|
+
cookies?: SessionCookie[];
|
|
70
|
+
/** localStorage entries to import */
|
|
71
|
+
localStorage?: LocalStorageEntry[];
|
|
72
|
+
/** Log import progress. Default: true */
|
|
73
|
+
logger?: boolean;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Session data result */
|
|
77
|
+
export interface SessionData {
|
|
78
|
+
cookies?: SessionCookie[];
|
|
79
|
+
localStorage?: LocalStorageEntry[];
|
|
80
|
+
}
|
|
81
|
+
|
|
33
82
|
/**
|
|
34
83
|
* Navigates to a URL with retry logic and incremental timeouts.
|
|
35
84
|
* If navigation fails, it temporarily goes to "about:blank" before retrying.
|
|
@@ -59,3 +108,16 @@ export function checkPageConditions(
|
|
|
59
108
|
checksToPerform: Record<string, string | Locator | null>,
|
|
60
109
|
timeout: number
|
|
61
110
|
): Promise<string | null>;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
114
|
+
* Silent operation — no browser blinking.
|
|
115
|
+
* cookies default: true, localStorage default: false
|
|
116
|
+
*/
|
|
117
|
+
export function exportSession(options: ExportSessionOptions): Promise<SessionData>;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Imports browser session (cookies and/or localStorage) into a Playwright Browser.
|
|
121
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
122
|
+
*/
|
|
123
|
+
export function importSession(options: ImportSessionOptions): Promise<void>;
|
|
@@ -129,81 +129,110 @@ export const checkPageConditions = async (page, checksToPerform, timeout) => {
|
|
|
129
129
|
};
|
|
130
130
|
|
|
131
131
|
/**
|
|
132
|
-
* Exports browser session (cookies
|
|
132
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
133
133
|
* Silent operation — no browser blinking.
|
|
134
134
|
*
|
|
135
|
-
* @param {
|
|
136
|
-
* @
|
|
135
|
+
* @param {Object} options
|
|
136
|
+
* @param {import('playwright-core').Page} options.page - Playwright Page
|
|
137
|
+
* @param {boolean} [options.cookies=true] - Export cookies
|
|
138
|
+
* @param {boolean} [options.localStorage=false] - Export localStorage
|
|
139
|
+
* @param {boolean} [options.logger=true] - Log export progress
|
|
140
|
+
* @returns {Promise<{cookies?: Array, localStorage?: Array}>} Session data for JSONB storage
|
|
137
141
|
*/
|
|
138
|
-
export const exportSession = async (
|
|
139
|
-
const
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
142
|
+
export const exportSession = async ({ page, cookies = true, localStorage = false, logger = true }) => {
|
|
143
|
+
const context = page.context();
|
|
144
|
+
const result = {};
|
|
145
|
+
|
|
146
|
+
if (cookies) {
|
|
147
|
+
const client = await context.newCDPSession(page);
|
|
148
|
+
const { cookies: allCookies } = await client.send("Network.getAllCookies");
|
|
149
|
+
result.cookies = allCookies;
|
|
150
|
+
await client.detach();
|
|
151
|
+
if (logger) console.log(`░░░░░ exportSession: ${allCookies.length} cookies exported`);
|
|
152
|
+
}
|
|
146
153
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
|
|
154
|
+
if (localStorage) {
|
|
155
|
+
result.localStorage = [];
|
|
156
|
+
const pages = context.pages();
|
|
157
|
+
for (const p of pages) {
|
|
158
|
+
const url = p.url();
|
|
159
|
+
if (!url || url === "about:blank" || url.startsWith("chrome")) continue;
|
|
160
|
+
|
|
161
|
+
try {
|
|
162
|
+
const origin = new URL(url).origin;
|
|
163
|
+
const items = await p.evaluate(() => {
|
|
164
|
+
const data = [];
|
|
165
|
+
for (let i = 0; i < window.localStorage.length; i++) {
|
|
166
|
+
const key = window.localStorage.key(i);
|
|
167
|
+
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
168
|
+
}
|
|
169
|
+
return data;
|
|
170
|
+
});
|
|
151
171
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const items = await p.evaluate(() => {
|
|
155
|
-
const data = [];
|
|
156
|
-
for (let i = 0; i < window.localStorage.length; i++) {
|
|
157
|
-
const key = window.localStorage.key(i);
|
|
158
|
-
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
172
|
+
if (items.length > 0 && !result.localStorage.some((o) => o.origin === origin)) {
|
|
173
|
+
result.localStorage.push({ origin, items });
|
|
159
174
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if (items.length > 0 && !localStorage.some((o) => o.origin === origin)) {
|
|
164
|
-
localStorage.push({ origin, items });
|
|
175
|
+
} catch (e) {
|
|
176
|
+
// Skip pages that can't be evaluated
|
|
165
177
|
}
|
|
166
|
-
} catch (e) {
|
|
167
|
-
// Skip pages that can't be evaluated
|
|
168
178
|
}
|
|
179
|
+
if (logger) console.log(`░░░░░ exportSession: ${result.localStorage.length} localStorage origins exported`);
|
|
169
180
|
}
|
|
170
181
|
|
|
171
|
-
return
|
|
182
|
+
return result;
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
/**
|
|
175
|
-
* Imports browser session (cookies
|
|
176
|
-
*
|
|
186
|
+
* Imports browser session (cookies and/or localStorage) into a Playwright Browser.
|
|
187
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
177
188
|
*
|
|
178
|
-
* @param {
|
|
179
|
-
* @param {
|
|
189
|
+
* @param {Object} options
|
|
190
|
+
* @param {import('playwright-core').Page} options.page - Playwright Page
|
|
191
|
+
* @param {Array} [options.cookies] - Cookies to import
|
|
192
|
+
* @param {Array} [options.localStorage] - localStorage entries to import
|
|
193
|
+
* @param {boolean} [options.logger=true] - Log import progress
|
|
180
194
|
*/
|
|
181
|
-
export const importSession = async (
|
|
182
|
-
const
|
|
183
|
-
const page = pages[0];
|
|
184
|
-
if (!page) throw new Error("No page available for importSession");
|
|
185
|
-
|
|
186
|
-
const client = await context.newCDPSession(page);
|
|
187
|
-
|
|
188
|
-
if (data.cookies?.length) {
|
|
189
|
-
await client.send("Network.setCookies", { cookies: data.cookies });
|
|
190
|
-
}
|
|
195
|
+
export const importSession = async ({ page, cookies, localStorage, logger = true }) => {
|
|
196
|
+
const context = page.context();
|
|
191
197
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
for (const entry of
|
|
198
|
+
// localStorage first — navigate to each origin and set items
|
|
199
|
+
if (localStorage?.length) {
|
|
200
|
+
for (const entry of localStorage) {
|
|
195
201
|
if (!entry.items?.length) continue;
|
|
196
202
|
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
await
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
value
|
|
203
|
-
|
|
203
|
+
const p = await context.newPage();
|
|
204
|
+
try {
|
|
205
|
+
await p.goto(entry.origin, { waitUntil: "load", timeout: 15000 });
|
|
206
|
+
await p.waitForLoadState("domcontentloaded");
|
|
207
|
+
await p.evaluate((items) => {
|
|
208
|
+
for (const { name, value } of items) {
|
|
209
|
+
window.localStorage.setItem(name, value);
|
|
210
|
+
}
|
|
211
|
+
}, entry.items);
|
|
212
|
+
} catch (e) {
|
|
213
|
+
// Origin may redirect — try to set localStorage on the final page
|
|
214
|
+
try {
|
|
215
|
+
await p.waitForLoadState("load", { timeout: 10000 });
|
|
216
|
+
await p.evaluate((items) => {
|
|
217
|
+
for (const { name, value } of items) {
|
|
218
|
+
window.localStorage.setItem(name, value);
|
|
219
|
+
}
|
|
220
|
+
}, entry.items);
|
|
221
|
+
} catch (_) {
|
|
222
|
+
// Skip this origin if it can't be reached
|
|
223
|
+
}
|
|
224
|
+
} finally {
|
|
225
|
+
await p.close();
|
|
204
226
|
}
|
|
205
227
|
}
|
|
228
|
+
if (logger) console.log(`░░░░░ importSession: ${localStorage.length} localStorage origins imported`);
|
|
206
229
|
}
|
|
207
230
|
|
|
208
|
-
|
|
231
|
+
// Cookies via CDP — silent, no navigation needed
|
|
232
|
+
if (cookies?.length) {
|
|
233
|
+
const client = await context.newCDPSession(page);
|
|
234
|
+
await client.send("Network.setCookies", { cookies });
|
|
235
|
+
await client.detach();
|
|
236
|
+
if (logger) console.log(`░░░░░ importSession: ${cookies.length} cookies imported`);
|
|
237
|
+
}
|
|
209
238
|
};
|
|
@@ -14,11 +14,8 @@ export interface PwRouteOptions {
|
|
|
14
14
|
/** Playwright Page (provide either context or page) */
|
|
15
15
|
page?: Page | null;
|
|
16
16
|
|
|
17
|
-
/** Log
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
/** Log failed requests to console */
|
|
21
|
-
errorLogs?: boolean;
|
|
17
|
+
/** Log level: "info" (success+error), "error" (errors only), false (no logs). Default: "error" */
|
|
18
|
+
logger?: "info" | "error" | false;
|
|
22
19
|
|
|
23
20
|
/** Block Ad requests (Ghostery engine) */
|
|
24
21
|
blockAds?: boolean;
|
|
@@ -45,11 +45,10 @@ export function pwCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
45
45
|
* @param {Object} requestHeaders - Request headers from the original request
|
|
46
46
|
* @param {string} method - HTTP method (GET, POST, etc.)
|
|
47
47
|
* @param {boolean} useFullUrl - Whether to use the full URL as cache key or just origin+path
|
|
48
|
-
* @param {
|
|
49
|
-
* @param {boolean} errorLogs - Whether to log error requests
|
|
48
|
+
* @param {string|false} logger - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
50
49
|
* @returns {Promise<Object>} - The response object containing status, headers, and body
|
|
51
50
|
*/
|
|
52
|
-
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl,
|
|
51
|
+
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl, logger) {
|
|
53
52
|
// Determine the cache key based on configuration
|
|
54
53
|
let mainUrl = new URL(url).origin + new URL(url).pathname;
|
|
55
54
|
if (useFullUrl) {
|
|
@@ -60,7 +59,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
60
59
|
if (useCache) {
|
|
61
60
|
const cachedResponse = globalCache.get(mainUrl);
|
|
62
61
|
if (cachedResponse) {
|
|
63
|
-
if (
|
|
62
|
+
if (logger === "info") console.log(`Serving from globalCache: ${mainUrl}`);
|
|
64
63
|
return cachedResponse;
|
|
65
64
|
}
|
|
66
65
|
}
|
|
@@ -79,7 +78,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
79
78
|
headers: response.headers,
|
|
80
79
|
body: responseBody,
|
|
81
80
|
});
|
|
82
|
-
if (
|
|
81
|
+
if (logger === "info") console.log(`Success (cached): ${mainUrl}`);
|
|
83
82
|
|
|
84
83
|
return {
|
|
85
84
|
status: response.status,
|
|
@@ -87,7 +86,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
87
86
|
body: responseBody,
|
|
88
87
|
};
|
|
89
88
|
} catch (error) {
|
|
90
|
-
if (
|
|
89
|
+
if (logger) console.error(`Failed to fetch: ${url}`, error);
|
|
91
90
|
// We return undefined on error, which signals the route handler to fall back to normal request
|
|
92
91
|
}
|
|
93
92
|
}
|
|
@@ -97,10 +96,9 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
97
96
|
* @param {Object} options - Configuration options
|
|
98
97
|
* @param {Object} options.context - Playwright context (optional, one is required)
|
|
99
98
|
* @param {Object} options.page - Playwright page (optional, one is required)
|
|
100
|
-
* @param {boolean} options.successLogs - Enable logging for successful fetches
|
|
101
|
-
* @param {boolean} options.errorLogs - Enable logging for failed fetches
|
|
102
99
|
* @param {boolean} options.blockImage - Enable global image blocking
|
|
103
100
|
* @param {boolean} options.blockAds - Enable Ghostery ad blocking
|
|
101
|
+
* @param {string|false} [options.logger="error"] - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
104
102
|
* @param {boolean} options.useGot - Enable custom fetching via Superagent (bypassing browser network stack for intercepted types)
|
|
105
103
|
* @param {boolean} options.useFullUrl - Use full URL for cache keys
|
|
106
104
|
* @param {boolean} options.useCache - Enable caching
|
|
@@ -112,8 +110,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
112
110
|
export async function pwRoute({
|
|
113
111
|
context = null,
|
|
114
112
|
page = null,
|
|
115
|
-
|
|
116
|
-
errorLogs = false,
|
|
113
|
+
logger = false,
|
|
117
114
|
blockAds = true,
|
|
118
115
|
blockImage = true,
|
|
119
116
|
useGot = true,
|
|
@@ -294,8 +291,7 @@ export async function pwRoute({
|
|
|
294
291
|
requestHeaders,
|
|
295
292
|
requestMethod,
|
|
296
293
|
useFullUrl,
|
|
297
|
-
|
|
298
|
-
errorLogs
|
|
294
|
+
logger
|
|
299
295
|
);
|
|
300
296
|
|
|
301
297
|
if (response) {
|
|
@@ -306,7 +302,7 @@ export async function pwRoute({
|
|
|
306
302
|
});
|
|
307
303
|
return;
|
|
308
304
|
} else {
|
|
309
|
-
if (
|
|
305
|
+
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
310
306
|
await route.continue();
|
|
311
307
|
return;
|
|
312
308
|
}
|
|
@@ -34,6 +34,55 @@ export interface RetryClickOptions {
|
|
|
34
34
|
timeout?: number;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/** A single cookie object (CDP format) */
|
|
38
|
+
export interface SessionCookie {
|
|
39
|
+
name: string;
|
|
40
|
+
value: string;
|
|
41
|
+
domain?: string;
|
|
42
|
+
path?: string;
|
|
43
|
+
expires?: number;
|
|
44
|
+
httpOnly?: boolean;
|
|
45
|
+
secure?: boolean;
|
|
46
|
+
sameSite?: string;
|
|
47
|
+
[key: string]: any;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A single localStorage entry for one origin */
|
|
51
|
+
export interface LocalStorageEntry {
|
|
52
|
+
origin: string;
|
|
53
|
+
items: { name: string; value: string }[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Options for exportSession */
|
|
57
|
+
export interface ExportSessionOptions {
|
|
58
|
+
/** Any Puppeteer Page */
|
|
59
|
+
page: Page;
|
|
60
|
+
/** Export cookies. Default: true */
|
|
61
|
+
cookies?: boolean;
|
|
62
|
+
/** Export localStorage. Default: false */
|
|
63
|
+
localStorage?: boolean;
|
|
64
|
+
/** Log export progress. Default: true */
|
|
65
|
+
logger?: boolean;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Options for importSession */
|
|
69
|
+
export interface ImportSessionOptions {
|
|
70
|
+
/** Any Puppeteer Page */
|
|
71
|
+
page: Page;
|
|
72
|
+
/** Cookies to import */
|
|
73
|
+
cookies?: SessionCookie[];
|
|
74
|
+
/** localStorage entries to import */
|
|
75
|
+
localStorage?: LocalStorageEntry[];
|
|
76
|
+
/** Log import progress. Default: true */
|
|
77
|
+
logger?: boolean;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Session data result */
|
|
81
|
+
export interface SessionData {
|
|
82
|
+
cookies?: SessionCookie[];
|
|
83
|
+
localStorage?: LocalStorageEntry[];
|
|
84
|
+
}
|
|
85
|
+
|
|
37
86
|
/**
|
|
38
87
|
* Navigates to a URL with retry logic and incremental timeouts.
|
|
39
88
|
* If navigation fails, it temporarily goes to "about:blank" before retrying.
|
|
@@ -68,3 +117,16 @@ export function checkPageConditions(
|
|
|
68
117
|
checksToPerform: Record<string, string | Locator<Node> | null>,
|
|
69
118
|
timeout: number
|
|
70
119
|
): Promise<string | null>;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
123
|
+
* Silent operation — no browser blinking.
|
|
124
|
+
* cookies default: true, localStorage default: false
|
|
125
|
+
*/
|
|
126
|
+
export function exportSession(options: ExportSessionOptions): Promise<SessionData>;
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Imports browser session (cookies and/or localStorage) into a Puppeteer Browser.
|
|
130
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
131
|
+
*/
|
|
132
|
+
export function importSession(options: ImportSessionOptions): Promise<void>;
|
|
@@ -151,77 +151,110 @@ export const checkPageConditions = async (page, checksToPerform, timeout) => {
|
|
|
151
151
|
};
|
|
152
152
|
|
|
153
153
|
/**
|
|
154
|
-
* Exports browser session (cookies
|
|
154
|
+
* Exports browser session (cookies and/or localStorage) via CDP.
|
|
155
155
|
* Silent operation — no browser blinking.
|
|
156
156
|
*
|
|
157
|
-
* @param {
|
|
158
|
-
* @
|
|
157
|
+
* @param {Object} options
|
|
158
|
+
* @param {import('puppeteer-core').Page} options.page - Any Puppeteer Page
|
|
159
|
+
* @param {boolean} [options.cookies=true] - Export cookies
|
|
160
|
+
* @param {boolean} [options.localStorage=false] - Export localStorage
|
|
161
|
+
* @param {boolean} [options.logger=true] - Log export progress
|
|
162
|
+
* @returns {Promise<{cookies?: Array, localStorage?: Array}>} Session data for JSONB storage
|
|
159
163
|
*/
|
|
160
|
-
export const exportSession = async (page) => {
|
|
161
|
-
const
|
|
162
|
-
const { cookies } = await client.send("Network.getAllCookies");
|
|
163
|
-
await client.detach();
|
|
164
|
+
export const exportSession = async ({ page, cookies = true, localStorage = false, logger = true }) => {
|
|
165
|
+
const result = {};
|
|
164
166
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
167
|
+
if (cookies) {
|
|
168
|
+
const client = await page.createCDPSession();
|
|
169
|
+
const { cookies: allCookies } = await client.send("Network.getAllCookies");
|
|
170
|
+
result.cookies = allCookies;
|
|
171
|
+
await client.detach();
|
|
172
|
+
if (logger) console.log(`░░░░░ exportSession: ${allCookies.length} cookies exported`);
|
|
173
|
+
}
|
|
168
174
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
175
|
+
if (localStorage) {
|
|
176
|
+
result.localStorage = [];
|
|
177
|
+
const browser = page.browser();
|
|
178
|
+
const pages = await browser.pages();
|
|
172
179
|
|
|
173
|
-
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
180
|
+
for (const p of pages) {
|
|
181
|
+
const url = p.url();
|
|
182
|
+
if (!url || url === "about:blank" || url.startsWith("chrome")) continue;
|
|
183
|
+
|
|
184
|
+
try {
|
|
185
|
+
const origin = new URL(url).origin;
|
|
186
|
+
const items = await p.evaluate(() => {
|
|
187
|
+
const data = [];
|
|
188
|
+
for (let i = 0; i < window.localStorage.length; i++) {
|
|
189
|
+
const key = window.localStorage.key(i);
|
|
190
|
+
data.push({ name: key, value: window.localStorage.getItem(key) });
|
|
191
|
+
}
|
|
192
|
+
return data;
|
|
193
|
+
});
|
|
183
194
|
|
|
184
|
-
|
|
185
|
-
|
|
195
|
+
if (items.length > 0 && !result.localStorage.some((o) => o.origin === origin)) {
|
|
196
|
+
result.localStorage.push({ origin, items });
|
|
197
|
+
}
|
|
198
|
+
} catch (e) {
|
|
199
|
+
// Skip pages that can't be evaluated
|
|
186
200
|
}
|
|
187
|
-
} catch (e) {
|
|
188
|
-
// Skip pages that can't be evaluated
|
|
189
201
|
}
|
|
202
|
+
if (logger) console.log(`░░░░░ exportSession: ${result.localStorage.length} localStorage origins exported`);
|
|
190
203
|
}
|
|
191
204
|
|
|
192
|
-
return
|
|
205
|
+
return result;
|
|
193
206
|
};
|
|
194
207
|
|
|
195
208
|
/**
|
|
196
|
-
* Imports browser session (cookies
|
|
209
|
+
* Imports browser session (cookies and/or localStorage) into a Puppeteer Browser.
|
|
210
|
+
* Imports whatever is provided. Order: localStorage first, then cookies.
|
|
197
211
|
*
|
|
198
|
-
* @param {
|
|
199
|
-
* @param {
|
|
212
|
+
* @param {Object} options
|
|
213
|
+
* @param {import('puppeteer-core').Page} options.page - Any Puppeteer Page
|
|
214
|
+
* @param {Array} [options.cookies] - Cookies to import
|
|
215
|
+
* @param {Array} [options.localStorage] - localStorage entries to import
|
|
216
|
+
* @param {boolean} [options.logger=true] - Log import progress
|
|
200
217
|
*/
|
|
201
|
-
export const importSession = async (page,
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
await client.send("Network.setCookies", { cookies: data.cookies });
|
|
205
|
-
await client.detach();
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
if (data.localStorage?.length) {
|
|
218
|
+
export const importSession = async ({ page, cookies, localStorage, logger = true }) => {
|
|
219
|
+
// localStorage first — navigate to each origin and set items
|
|
220
|
+
if (localStorage?.length) {
|
|
209
221
|
const browser = page.browser();
|
|
210
222
|
|
|
211
|
-
for (const entry of
|
|
223
|
+
for (const entry of localStorage) {
|
|
212
224
|
if (!entry.items?.length) continue;
|
|
213
225
|
|
|
214
226
|
const p = await browser.newPage();
|
|
215
227
|
try {
|
|
216
|
-
await p.goto(entry.origin, { waitUntil: "
|
|
228
|
+
await p.goto(entry.origin, { waitUntil: "load", timeout: 15000 });
|
|
217
229
|
await p.evaluate((items) => {
|
|
218
230
|
for (const { name, value } of items) {
|
|
219
231
|
window.localStorage.setItem(name, value);
|
|
220
232
|
}
|
|
221
233
|
}, entry.items);
|
|
234
|
+
} catch (e) {
|
|
235
|
+
// Origin may redirect — retry after load
|
|
236
|
+
try {
|
|
237
|
+
await p.waitForNavigation({ waitUntil: "load", timeout: 10000 }).catch(() => {});
|
|
238
|
+
await p.evaluate((items) => {
|
|
239
|
+
for (const { name, value } of items) {
|
|
240
|
+
window.localStorage.setItem(name, value);
|
|
241
|
+
}
|
|
242
|
+
}, entry.items);
|
|
243
|
+
} catch (_) {
|
|
244
|
+
// Skip this origin
|
|
245
|
+
}
|
|
222
246
|
} finally {
|
|
223
247
|
await p.close();
|
|
224
248
|
}
|
|
225
249
|
}
|
|
250
|
+
if (logger) console.log(`░░░░░ importSession: ${localStorage.length} localStorage origins imported`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Cookies via CDP — silent, no navigation needed
|
|
254
|
+
if (cookies?.length) {
|
|
255
|
+
const client = await page.createCDPSession();
|
|
256
|
+
await client.send("Network.setCookies", { cookies });
|
|
257
|
+
await client.detach();
|
|
258
|
+
if (logger) console.log(`░░░░░ importSession: ${cookies.length} cookies imported`);
|
|
226
259
|
}
|
|
227
260
|
};
|
|
@@ -11,11 +11,8 @@ export interface PpRouteOptions {
|
|
|
11
11
|
/** Puppeteer Page */
|
|
12
12
|
page?: Page | null;
|
|
13
13
|
|
|
14
|
-
/** Log
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
/** Log failed requests to console */
|
|
18
|
-
errorLogs?: boolean;
|
|
14
|
+
/** Log level: "info" (success+error), "error" (errors only), false (no logs). Default: "error" */
|
|
15
|
+
logger?: "info" | "error" | false;
|
|
19
16
|
|
|
20
17
|
/** Block Ad requests (Ghostery engine) */
|
|
21
18
|
blockAds?: boolean;
|
|
@@ -45,11 +45,10 @@ export function ppCacheLogs(log_cache = globalCache, interval = 10) {
|
|
|
45
45
|
* @param {Object} requestHeaders - Request headers from the original request
|
|
46
46
|
* @param {string} method - HTTP method (GET, POST, etc.)
|
|
47
47
|
* @param {boolean} useFullUrl - Whether to use the full URL as cache key or just origin+path
|
|
48
|
-
* @param {
|
|
49
|
-
* @param {boolean} errorLogs - Whether to log error requests
|
|
48
|
+
* @param {string|false} logger - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
50
49
|
* @returns {Promise<Object>} - The response object containing status, headers, and body
|
|
51
50
|
*/
|
|
52
|
-
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl,
|
|
51
|
+
async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl, logger) {
|
|
53
52
|
// Determine the cache key based on configuration
|
|
54
53
|
let mainUrl = new URL(url).origin + new URL(url).pathname;
|
|
55
54
|
if (useFullUrl) {
|
|
@@ -60,7 +59,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
60
59
|
if (useCache) {
|
|
61
60
|
const cachedResponse = globalCache.get(mainUrl);
|
|
62
61
|
if (cachedResponse) {
|
|
63
|
-
if (
|
|
62
|
+
if (logger === "info") console.log(`Serving from globalCache: ${mainUrl}`);
|
|
64
63
|
return cachedResponse;
|
|
65
64
|
}
|
|
66
65
|
}
|
|
@@ -79,7 +78,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
79
78
|
headers: response.headers,
|
|
80
79
|
body: responseBody,
|
|
81
80
|
});
|
|
82
|
-
if (
|
|
81
|
+
if (logger === "info") console.log(`Success (cached): ${mainUrl}`);
|
|
83
82
|
|
|
84
83
|
return {
|
|
85
84
|
status: response.status,
|
|
@@ -87,7 +86,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
87
86
|
body: responseBody,
|
|
88
87
|
};
|
|
89
88
|
} catch (error) {
|
|
90
|
-
if (
|
|
89
|
+
if (logger) console.error(`Failed to fetch: ${url}`, error);
|
|
91
90
|
// We return undefined on error, which signals the route handler to fall back to normal request
|
|
92
91
|
}
|
|
93
92
|
}
|
|
@@ -97,10 +96,9 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
97
96
|
* @param {Object} options - Configuration options
|
|
98
97
|
* @param {Object} options.context - Playwright context (optional, one is required)
|
|
99
98
|
* @param {Object} options.page - Playwright page (optional, one is required)
|
|
100
|
-
* @param {boolean} options.successLogs - Enable logging for successful fetches
|
|
101
|
-
* @param {boolean} options.errorLogs - Enable logging for failed fetches
|
|
102
99
|
* @param {boolean} options.blockImage - Enable global image blocking
|
|
103
100
|
* @param {boolean} options.blockAds - Enable Ghostery ad blocking
|
|
101
|
+
* @param {string|false} [options.logger="error"] - Log level: "info" (success+error), "error" (errors only), false (no logs)
|
|
104
102
|
* @param {boolean} options.useGot - Enable custom fetching via Superagent (bypassing browser network stack for intercepted types)
|
|
105
103
|
* @param {boolean} options.useFullUrl - Use full URL for cache keys
|
|
106
104
|
* @param {boolean} options.useCache - Enable caching
|
|
@@ -111,8 +109,7 @@ async function fetchWithClient(useCache, url, requestHeaders, method, useFullUrl
|
|
|
111
109
|
*/
|
|
112
110
|
export async function ppRoute({
|
|
113
111
|
page = null,
|
|
114
|
-
|
|
115
|
-
errorLogs = false,
|
|
112
|
+
logger = false,
|
|
116
113
|
blockAds = true,
|
|
117
114
|
blockImage = true,
|
|
118
115
|
useGot = true,
|
|
@@ -298,8 +295,7 @@ export async function ppRoute({
|
|
|
298
295
|
requestHeaders,
|
|
299
296
|
requestMethod,
|
|
300
297
|
useFullUrl,
|
|
301
|
-
|
|
302
|
-
errorLogs
|
|
298
|
+
logger
|
|
303
299
|
);
|
|
304
300
|
|
|
305
301
|
if (response) {
|
|
@@ -310,7 +306,7 @@ export async function ppRoute({
|
|
|
310
306
|
});
|
|
311
307
|
return;
|
|
312
308
|
} else {
|
|
313
|
-
if (
|
|
309
|
+
if (logger) console.log("Continuing with normal request (fetchWithClient returned null):", url);
|
|
314
310
|
await request.continue();
|
|
315
311
|
return;
|
|
316
312
|
}
|