@letsscrapedata/controller 0.0.49 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +141 -118
- package/dist/index.d.cts +9 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.js +93 -71
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -40,23 +40,46 @@ __export(src_exports, {
|
|
|
40
40
|
PuppeteerBrowserContext: () => PuppeteerBrowserContext,
|
|
41
41
|
PuppeteerElement: () => PuppeteerElement,
|
|
42
42
|
PuppeteerPage: () => PuppeteerPage,
|
|
43
|
-
controller: () => controller
|
|
43
|
+
controller: () => controller,
|
|
44
|
+
setControllerLogFun: () => setControllerLogFun
|
|
44
45
|
});
|
|
45
46
|
module.exports = __toCommonJS(src_exports);
|
|
46
47
|
|
|
48
|
+
// src/utils/log.ts
|
|
49
|
+
var import_utils = require("@letsscrapedata/utils");
|
|
50
|
+
var pkgLog = import_utils.log;
|
|
51
|
+
function setControllerLogFun(logFun) {
|
|
52
|
+
if (typeof logFun === "function") {
|
|
53
|
+
pkgLog = logFun;
|
|
54
|
+
return true;
|
|
55
|
+
} else {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async function loginfo(...args) {
|
|
60
|
+
await pkgLog(import_utils.LogLevel.INF, ...args);
|
|
61
|
+
}
|
|
62
|
+
async function logwarn(...args) {
|
|
63
|
+
await pkgLog(import_utils.LogLevel.WRN, ...args);
|
|
64
|
+
}
|
|
65
|
+
async function logerr(...args) {
|
|
66
|
+
await pkgLog(import_utils.LogLevel.ERR, ...args);
|
|
67
|
+
}
|
|
68
|
+
|
|
47
69
|
// src/playwright/browser.ts
|
|
48
70
|
var import_node_events3 = __toESM(require("events"), 1);
|
|
71
|
+
var import_utils5 = require("@letsscrapedata/utils");
|
|
49
72
|
|
|
50
73
|
// src/playwright/context.ts
|
|
51
74
|
var import_node_events2 = __toESM(require("events"), 1);
|
|
52
|
-
var
|
|
75
|
+
var import_utils4 = require("@letsscrapedata/utils");
|
|
53
76
|
|
|
54
77
|
// src/playwright/page.ts
|
|
55
78
|
var import_node_events = __toESM(require("events"), 1);
|
|
56
|
-
var
|
|
79
|
+
var import_utils3 = require("@letsscrapedata/utils");
|
|
57
80
|
|
|
58
81
|
// src/playwright/element.ts
|
|
59
|
-
var
|
|
82
|
+
var import_utils2 = require("@letsscrapedata/utils");
|
|
60
83
|
var PlaywrightElement = class _PlaywrightElement {
|
|
61
84
|
#frame;
|
|
62
85
|
#locator;
|
|
@@ -161,7 +184,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
161
184
|
retObj.locators = locators;
|
|
162
185
|
return retObj;
|
|
163
186
|
} catch (err) {
|
|
164
|
-
|
|
187
|
+
loginfo(err);
|
|
165
188
|
return retObj;
|
|
166
189
|
}
|
|
167
190
|
}
|
|
@@ -236,7 +259,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
236
259
|
} else if (clickType === "evaluate") {
|
|
237
260
|
await this.#locator.evaluate(async (ev) => await ev.click());
|
|
238
261
|
} else {
|
|
239
|
-
(0,
|
|
262
|
+
(0, import_utils2.unreachable)(clickType);
|
|
240
263
|
}
|
|
241
264
|
return true;
|
|
242
265
|
}
|
|
@@ -309,7 +332,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
309
332
|
}
|
|
310
333
|
break;
|
|
311
334
|
default:
|
|
312
|
-
(0,
|
|
335
|
+
(0, import_utils2.unreachable)(type);
|
|
313
336
|
}
|
|
314
337
|
return true;
|
|
315
338
|
}
|
|
@@ -349,7 +372,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
349
372
|
const cookieItems = await this.#getCookies(page);
|
|
350
373
|
const domainSet = new Set(cookieItems.map((c) => c.domain));
|
|
351
374
|
if (domainSet.size !== 1) {
|
|
352
|
-
|
|
375
|
+
logwarn(`Domains in clearCookies: ${Array.from(domainSet.values())}`);
|
|
353
376
|
}
|
|
354
377
|
for (const domain of domainSet.values()) {
|
|
355
378
|
await browserContext.clearCookies({ domain });
|
|
@@ -514,7 +537,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
514
537
|
retObj.locators = locators;
|
|
515
538
|
return retObj;
|
|
516
539
|
} catch (err) {
|
|
517
|
-
|
|
540
|
+
loginfo(err);
|
|
518
541
|
return retObj;
|
|
519
542
|
}
|
|
520
543
|
}
|
|
@@ -525,9 +548,9 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
525
548
|
const page = this.#page;
|
|
526
549
|
const pageId = this.#pageId;
|
|
527
550
|
page.on("close", async () => {
|
|
528
|
-
|
|
551
|
+
loginfo(`##browser ${pageId} closed`);
|
|
529
552
|
if (!page.pageInfo) {
|
|
530
|
-
|
|
553
|
+
logerr(`Logic error in page.on("close")`);
|
|
531
554
|
}
|
|
532
555
|
this.emit("pageClose");
|
|
533
556
|
this.#lsdBrowserContext.emit("pageClose", this);
|
|
@@ -546,12 +569,12 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
546
569
|
pageInfo.relatedId = page.pageInfo.taskId;
|
|
547
570
|
}
|
|
548
571
|
} else {
|
|
549
|
-
|
|
572
|
+
logerr(`##browser ${pageId} has popup without page.pageInfo`);
|
|
550
573
|
}
|
|
551
|
-
|
|
574
|
+
loginfo(`##browser ${pageId} has popup ${popupPageId}`);
|
|
552
575
|
this.emit("pagePopup", evtData);
|
|
553
576
|
} else {
|
|
554
|
-
|
|
577
|
+
logerr(`##browser ${pageId} has popup page with null page`);
|
|
555
578
|
}
|
|
556
579
|
});
|
|
557
580
|
}
|
|
@@ -563,7 +586,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
563
586
|
this.#lsdBrowserContext = browserContext;
|
|
564
587
|
this.#page = page;
|
|
565
588
|
this.#status = "free";
|
|
566
|
-
const currentTime = (0,
|
|
589
|
+
const currentTime = (0, import_utils3.getCurrentUnixTime)();
|
|
567
590
|
const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
|
|
568
591
|
this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
|
|
569
592
|
this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
|
|
@@ -618,7 +641,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
618
641
|
}
|
|
619
642
|
return true;
|
|
620
643
|
} catch (err) {
|
|
621
|
-
|
|
644
|
+
logerr(err);
|
|
622
645
|
return false;
|
|
623
646
|
}
|
|
624
647
|
}
|
|
@@ -921,7 +944,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
921
944
|
}
|
|
922
945
|
return true;
|
|
923
946
|
} catch (err) {
|
|
924
|
-
|
|
947
|
+
logerr(err);
|
|
925
948
|
return false;
|
|
926
949
|
}
|
|
927
950
|
}
|
|
@@ -931,7 +954,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
931
954
|
}
|
|
932
955
|
const actOptions = Array.isArray(options) ? options : [options];
|
|
933
956
|
if (actOptions.length <= 0) {
|
|
934
|
-
|
|
957
|
+
logwarn("Invalid paras in setRequestInterception");
|
|
935
958
|
return false;
|
|
936
959
|
}
|
|
937
960
|
const firstRequestInterception = this.#resquestInterceptionOptions.length <= 0;
|
|
@@ -942,7 +965,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
942
965
|
this.#resquestInterceptionOptions.push(option);
|
|
943
966
|
break;
|
|
944
967
|
default:
|
|
945
|
-
(0,
|
|
968
|
+
(0, import_utils3.unreachable)(option.action);
|
|
946
969
|
}
|
|
947
970
|
}
|
|
948
971
|
if (firstRequestInterception && this.#resquestInterceptionOptions.length > 0) {
|
|
@@ -966,7 +989,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
966
989
|
});
|
|
967
990
|
break;
|
|
968
991
|
default:
|
|
969
|
-
(0,
|
|
992
|
+
(0, import_utils3.unreachable)(action);
|
|
970
993
|
}
|
|
971
994
|
return true;
|
|
972
995
|
} else {
|
|
@@ -975,7 +998,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
975
998
|
await route.continue();
|
|
976
999
|
return true;
|
|
977
1000
|
} catch (err) {
|
|
978
|
-
|
|
1001
|
+
logerr(err);
|
|
979
1002
|
return false;
|
|
980
1003
|
}
|
|
981
1004
|
});
|
|
@@ -1019,7 +1042,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
1019
1042
|
requestData,
|
|
1020
1043
|
responseData
|
|
1021
1044
|
});
|
|
1022
|
-
|
|
1045
|
+
loginfo(`##browser cache matched response: ${requestUrl}`);
|
|
1023
1046
|
}
|
|
1024
1047
|
if (typeof handler === "function") {
|
|
1025
1048
|
const pageData = { pageUrl, cookies: "" };
|
|
@@ -1028,7 +1051,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
1028
1051
|
}
|
|
1029
1052
|
return;
|
|
1030
1053
|
} catch (err) {
|
|
1031
|
-
|
|
1054
|
+
logerr(err);
|
|
1032
1055
|
return;
|
|
1033
1056
|
}
|
|
1034
1057
|
}
|
|
@@ -1233,7 +1256,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1233
1256
|
}
|
|
1234
1257
|
const pages = this.#browserContext.pages();
|
|
1235
1258
|
const openType = this.#lsdBrowser.browserCreationMethod();
|
|
1236
|
-
const lastStatusUpdateTime = (0,
|
|
1259
|
+
const lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1237
1260
|
for (const page of pages) {
|
|
1238
1261
|
const pageInfo = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType, openTime: this.#createTime, lastStatusUpdateTime, taskId: 0, relatedId: 0, misc: {} };
|
|
1239
1262
|
const lsdPage = new PlaywrightPage(this, page, pageInfo);
|
|
@@ -1241,7 +1264,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1241
1264
|
await lsdPage.maximizeViewport();
|
|
1242
1265
|
}
|
|
1243
1266
|
this.#lsdPages.push(lsdPage);
|
|
1244
|
-
|
|
1267
|
+
loginfo(`##browser ${lsdPage.id()} ${openType}ed`);
|
|
1245
1268
|
}
|
|
1246
1269
|
}
|
|
1247
1270
|
constructor(lsdBrowser, browserContext, browserContextCreationMethod, incognito = false, proxy = null, browserIdx = 0, browserContextIdx = 0, maxPagesPerBrowserContext = 20, maxPageFreeSeconds = 0, maxViewportOfNewPage = true) {
|
|
@@ -1259,7 +1282,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1259
1282
|
this.#browserContextCreationMethod = browserContextCreationMethod;
|
|
1260
1283
|
const apiRequestContext = browserContext.request;
|
|
1261
1284
|
this.#apiContext = new PlaywrightApiContext(apiRequestContext);
|
|
1262
|
-
const currentTime = (0,
|
|
1285
|
+
const currentTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1263
1286
|
this.#createTime = currentTime;
|
|
1264
1287
|
this.#lastStatusUpdateTime = currentTime;
|
|
1265
1288
|
this.#status = "free";
|
|
@@ -1276,32 +1299,32 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1276
1299
|
const pageInfo = page.pageInfo;
|
|
1277
1300
|
if (pageInfo) {
|
|
1278
1301
|
const { browserIdx: browserIdx2, browserContextIdx: browserContextIdx2, pageIdx } = pageInfo;
|
|
1279
|
-
|
|
1302
|
+
logwarn(`##browser page-${browserIdx2}-${browserContextIdx2}-${pageIdx} has been already created`);
|
|
1280
1303
|
} else {
|
|
1281
|
-
const currentTime2 = (0,
|
|
1304
|
+
const currentTime2 = (0, import_utils4.getCurrentUnixTime)();
|
|
1282
1305
|
const pageInfo2 = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType: "other", openTime: currentTime2, lastStatusUpdateTime: currentTime2, taskId: 0, relatedId: 0, misc: {} };
|
|
1283
1306
|
const lsdPage = new PlaywrightPage(this, page, pageInfo2);
|
|
1284
1307
|
if (this.#maxViewportOfNewPage) {
|
|
1285
1308
|
await lsdPage.maximizeViewport();
|
|
1286
1309
|
}
|
|
1287
1310
|
this.#lsdPages.push(lsdPage);
|
|
1288
|
-
|
|
1311
|
+
loginfo(`##page ${lsdPage.id()} created`);
|
|
1289
1312
|
}
|
|
1290
1313
|
});
|
|
1291
1314
|
browserContext.on("close", (bc) => {
|
|
1292
1315
|
if (browserContext !== bc) {
|
|
1293
|
-
|
|
1316
|
+
logerr(`##browser different browserContext in browserContext.on("close")`);
|
|
1294
1317
|
}
|
|
1295
1318
|
this.#lsdBrowser.emit("browserContextClose", this);
|
|
1296
1319
|
});
|
|
1297
1320
|
this.on("pageClose", (lsdPage) => {
|
|
1298
1321
|
if (!(lsdPage instanceof PlaywrightPage)) {
|
|
1299
|
-
|
|
1322
|
+
logerr(`Invalid data in LsdBrowserContext.on("pageClose)`);
|
|
1300
1323
|
return;
|
|
1301
1324
|
}
|
|
1302
1325
|
const idx = this.#lsdPages.findIndex((p) => p === lsdPage);
|
|
1303
1326
|
if (idx < 0) {
|
|
1304
|
-
|
|
1327
|
+
logerr(`Invalid lsdPage in LsdBrowserContext.on("pageClose)`);
|
|
1305
1328
|
return;
|
|
1306
1329
|
}
|
|
1307
1330
|
this.#lsdPages.splice(idx, 1);
|
|
@@ -1317,8 +1340,8 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1317
1340
|
async close() {
|
|
1318
1341
|
if (this.#browserContext) {
|
|
1319
1342
|
this.#status = "closed";
|
|
1320
|
-
this.#lastStatusUpdateTime = (0,
|
|
1321
|
-
|
|
1343
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1344
|
+
loginfo(`browserContext ${this.id()} closed at ${this.#lastStatusUpdateTime}`);
|
|
1322
1345
|
await this.#browserContext.close();
|
|
1323
1346
|
}
|
|
1324
1347
|
return true;
|
|
@@ -1330,15 +1353,15 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1330
1353
|
this.#gettingPage = true;
|
|
1331
1354
|
return true;
|
|
1332
1355
|
} else {
|
|
1333
|
-
await (0,
|
|
1356
|
+
await (0, import_utils4.sleep)(200);
|
|
1334
1357
|
}
|
|
1335
1358
|
}
|
|
1336
|
-
|
|
1359
|
+
logwarn(`Cannot get the gettingLock.`);
|
|
1337
1360
|
return false;
|
|
1338
1361
|
}
|
|
1339
1362
|
#freeGettingLock() {
|
|
1340
1363
|
if (!this.#gettingPage) {
|
|
1341
|
-
|
|
1364
|
+
logwarn(`Getting lock is already free now.`);
|
|
1342
1365
|
}
|
|
1343
1366
|
this.#gettingPage = false;
|
|
1344
1367
|
}
|
|
@@ -1347,7 +1370,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1347
1370
|
maxPageFreeSeconds = this.#maxPageFreeSeconds;
|
|
1348
1371
|
}
|
|
1349
1372
|
if (maxPageFreeSeconds <= 0) {
|
|
1350
|
-
|
|
1373
|
+
logwarn(`Please set valid maxPageFreeSeconds to close free pages`);
|
|
1351
1374
|
return false;
|
|
1352
1375
|
}
|
|
1353
1376
|
const gotLock = await this.#tryToGetGettingLock();
|
|
@@ -1355,7 +1378,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1355
1378
|
return false;
|
|
1356
1379
|
}
|
|
1357
1380
|
try {
|
|
1358
|
-
const maxUpdateTime = (0,
|
|
1381
|
+
const maxUpdateTime = (0, import_utils4.getCurrentUnixTime)() - this.#maxPageFreeSeconds;
|
|
1359
1382
|
let freePages = this.#lsdPages.filter((p) => p.isFree() && p.pageInfo().lastStatusUpdateTime < maxUpdateTime);
|
|
1360
1383
|
if (freePages.length === this.#lsdPages.length) {
|
|
1361
1384
|
freePages = freePages.slice(1);
|
|
@@ -1366,7 +1389,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1366
1389
|
this.#freeGettingLock();
|
|
1367
1390
|
return true;
|
|
1368
1391
|
} catch (err) {
|
|
1369
|
-
|
|
1392
|
+
logerr(err);
|
|
1370
1393
|
this.#freeGettingLock();
|
|
1371
1394
|
return false;
|
|
1372
1395
|
}
|
|
@@ -1401,7 +1424,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1401
1424
|
return null;
|
|
1402
1425
|
}
|
|
1403
1426
|
const page = await this.#browserContext.newPage();
|
|
1404
|
-
await (0,
|
|
1427
|
+
await (0, import_utils4.sleep)(2e3);
|
|
1405
1428
|
const pageInfo = page.pageInfo;
|
|
1406
1429
|
if (!pageInfo) {
|
|
1407
1430
|
throw new Error(`Logic error in getPage`);
|
|
@@ -1418,7 +1441,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1418
1441
|
return null;
|
|
1419
1442
|
}
|
|
1420
1443
|
} catch (err) {
|
|
1421
|
-
|
|
1444
|
+
logerr(err);
|
|
1422
1445
|
this.#freeGettingLock();
|
|
1423
1446
|
return null;
|
|
1424
1447
|
}
|
|
@@ -1426,7 +1449,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1426
1449
|
free(clearStateData = false) {
|
|
1427
1450
|
if (this.#status === "busy") {
|
|
1428
1451
|
this.#status = "free";
|
|
1429
|
-
this.#lastStatusUpdateTime = (0,
|
|
1452
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1430
1453
|
if (clearStateData) {
|
|
1431
1454
|
}
|
|
1432
1455
|
return true;
|
|
@@ -1495,12 +1518,12 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1495
1518
|
}, localStorage);
|
|
1496
1519
|
}
|
|
1497
1520
|
}
|
|
1498
|
-
await (0,
|
|
1521
|
+
await (0, import_utils4.sleep)(2e3);
|
|
1499
1522
|
await origPage.unrouteAll();
|
|
1500
1523
|
await page.free();
|
|
1501
1524
|
return true;
|
|
1502
1525
|
} catch (err) {
|
|
1503
|
-
|
|
1526
|
+
logerr(err);
|
|
1504
1527
|
return false;
|
|
1505
1528
|
}
|
|
1506
1529
|
}
|
|
@@ -1510,7 +1533,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1510
1533
|
use() {
|
|
1511
1534
|
if (this.#status === "free") {
|
|
1512
1535
|
this.#status = "busy";
|
|
1513
|
-
this.#lastStatusUpdateTime = (0,
|
|
1536
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1514
1537
|
return true;
|
|
1515
1538
|
} else {
|
|
1516
1539
|
return false;
|
|
@@ -1522,7 +1545,6 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1522
1545
|
};
|
|
1523
1546
|
|
|
1524
1547
|
// src/playwright/browser.ts
|
|
1525
|
-
var import_utils4 = require("@letsscrapedata/utils");
|
|
1526
1548
|
var PlaywrightBrowser = class extends import_node_events3.default {
|
|
1527
1549
|
#browser;
|
|
1528
1550
|
#browserIdx;
|
|
@@ -1561,7 +1583,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1561
1583
|
this.#browser = browser;
|
|
1562
1584
|
this.#browserIdx = browserIdx;
|
|
1563
1585
|
this.#pid = pid;
|
|
1564
|
-
this.#createTime = (0,
|
|
1586
|
+
this.#createTime = (0, import_utils5.getCurrentUnixTime)();
|
|
1565
1587
|
this.#lsdBrowserContexts = [];
|
|
1566
1588
|
this.#browserControllerType = "playwright";
|
|
1567
1589
|
this.#browserType = browerType;
|
|
@@ -1572,38 +1594,38 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1572
1594
|
this.#executablePath = executablePath;
|
|
1573
1595
|
this.#nextBrowserContextIdx = 1;
|
|
1574
1596
|
this.#closeFreePagesIntervalId = null;
|
|
1575
|
-
|
|
1597
|
+
loginfo(`##browser ${this.id()} ${this.#browserCreationMethod}ed by ${this.#browserControllerType}`);
|
|
1576
1598
|
const browserContexts = browser.contexts();
|
|
1577
1599
|
if (browserContexts.length > 0) {
|
|
1578
|
-
|
|
1600
|
+
logwarn(`There are ${browserContexts.length} new browserContexts when playwright launches new browser`);
|
|
1579
1601
|
}
|
|
1580
1602
|
const incognito = typeof options?.incognito === "boolean" ? options.incognito : true;
|
|
1581
1603
|
for (const browserContext of browserContexts) {
|
|
1582
1604
|
const lsdBrowserContext = new PlaywrightBrowserContext(this, browserContext, "launch", incognito, this.#proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), maxViewportOfNewPage);
|
|
1583
1605
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
1584
|
-
|
|
1606
|
+
loginfo(`##browserContext ${lsdBrowserContext.id()} ${this.#browserCreationMethod}ed`);
|
|
1585
1607
|
}
|
|
1586
1608
|
browser.on("disconnected", () => {
|
|
1587
|
-
|
|
1609
|
+
loginfo(`##browser ${this.id()} disconnected`);
|
|
1588
1610
|
if (this.#lsdBrowserContexts.length > 0) {
|
|
1589
|
-
|
|
1611
|
+
logerr(`${this.id()} has browserContexts when disconnected`);
|
|
1590
1612
|
}
|
|
1591
1613
|
});
|
|
1592
1614
|
this.on("browserContextClose", (lsdBrowserContext) => {
|
|
1593
1615
|
if (!(lsdBrowserContext instanceof PlaywrightBrowserContext)) {
|
|
1594
|
-
|
|
1616
|
+
logerr(`Invalid data in LsdBrowser.on("browserContextClose)`);
|
|
1595
1617
|
return;
|
|
1596
1618
|
}
|
|
1597
1619
|
const idx = this.#lsdBrowserContexts.findIndex((bc) => bc === lsdBrowserContext);
|
|
1598
1620
|
if (idx < 0) {
|
|
1599
|
-
|
|
1621
|
+
logerr(`Invalid lsdBrowserContext in LsdBrowser.on("browserContextClose)`);
|
|
1600
1622
|
return;
|
|
1601
1623
|
}
|
|
1602
|
-
|
|
1624
|
+
loginfo(`##browserContext ${lsdBrowserContext.id()} closed
|
|
1603
1625
|
`);
|
|
1604
1626
|
this.#lsdBrowserContexts.splice(idx, 1);
|
|
1605
1627
|
if (this.#lsdBrowserContexts.length === 0) {
|
|
1606
|
-
|
|
1628
|
+
loginfo(`##browser ${this.id()} has no browserContexts now`);
|
|
1607
1629
|
}
|
|
1608
1630
|
return;
|
|
1609
1631
|
});
|
|
@@ -1620,7 +1642,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1620
1642
|
}
|
|
1621
1643
|
async newBrowserContext(options) {
|
|
1622
1644
|
if (this.#lsdBrowserContexts.length >= this.#maxBrowserContextsPerBrowser()) {
|
|
1623
|
-
|
|
1645
|
+
logwarn(`##browser ${this.id()} can not create more new browserContext`);
|
|
1624
1646
|
return null;
|
|
1625
1647
|
}
|
|
1626
1648
|
const browserContextOptions = {};
|
|
@@ -1639,7 +1661,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1639
1661
|
const { maxViewportOfNewPage = this.#options.maxViewportOfNewPage } = options ? options : {};
|
|
1640
1662
|
const lsdBrowserContext = new PlaywrightBrowserContext(this, browserContext, "new", true, proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), maxViewportOfNewPage);
|
|
1641
1663
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
1642
|
-
|
|
1664
|
+
loginfo(`##browser ${lsdBrowserContext.id()} created`);
|
|
1643
1665
|
return lsdBrowserContext;
|
|
1644
1666
|
}
|
|
1645
1667
|
async close() {
|
|
@@ -1691,7 +1713,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1691
1713
|
}
|
|
1692
1714
|
async pidUsage() {
|
|
1693
1715
|
if (this.#pid > 0) {
|
|
1694
|
-
const usage = await (0,
|
|
1716
|
+
const usage = await (0, import_utils5.getPerformanceOfPidTree)(this.#pid, "MB");
|
|
1695
1717
|
return usage;
|
|
1696
1718
|
} else {
|
|
1697
1719
|
return { cpu: 0, memory: 0 };
|
|
@@ -1711,15 +1733,18 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1711
1733
|
|
|
1712
1734
|
// src/puppeteer/browser.ts
|
|
1713
1735
|
var import_node_events6 = __toESM(require("events"), 1);
|
|
1736
|
+
var import_utils9 = require("@letsscrapedata/utils");
|
|
1714
1737
|
|
|
1715
1738
|
// src/puppeteer/context.ts
|
|
1716
1739
|
var import_node_events5 = __toESM(require("events"), 1);
|
|
1740
|
+
var import_utils8 = require("@letsscrapedata/utils");
|
|
1717
1741
|
|
|
1718
1742
|
// src/puppeteer/page.ts
|
|
1719
1743
|
var import_node_events4 = __toESM(require("events"), 1);
|
|
1744
|
+
var import_utils7 = require("@letsscrapedata/utils");
|
|
1720
1745
|
|
|
1721
1746
|
// src/puppeteer/element.ts
|
|
1722
|
-
var
|
|
1747
|
+
var import_utils6 = require("@letsscrapedata/utils");
|
|
1723
1748
|
var PuppeteerElement = class _PuppeteerElement {
|
|
1724
1749
|
#frame;
|
|
1725
1750
|
#$ele;
|
|
@@ -1808,7 +1833,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1808
1833
|
}
|
|
1809
1834
|
return retObj;
|
|
1810
1835
|
} catch (err) {
|
|
1811
|
-
|
|
1836
|
+
logerr(err);
|
|
1812
1837
|
return retObj;
|
|
1813
1838
|
}
|
|
1814
1839
|
}
|
|
@@ -1884,7 +1909,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1884
1909
|
await this.#frame.evaluate(async (ev) => await ev.click(), this.#$ele);
|
|
1885
1910
|
await this.#$ele.click(actOptions);
|
|
1886
1911
|
} else {
|
|
1887
|
-
(0,
|
|
1912
|
+
(0, import_utils6.unreachable)(clickType);
|
|
1888
1913
|
}
|
|
1889
1914
|
return true;
|
|
1890
1915
|
}
|
|
@@ -1974,7 +1999,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1974
1999
|
}
|
|
1975
2000
|
break;
|
|
1976
2001
|
default:
|
|
1977
|
-
(0,
|
|
2002
|
+
(0, import_utils6.unreachable)(type);
|
|
1978
2003
|
}
|
|
1979
2004
|
return true;
|
|
1980
2005
|
}
|
|
@@ -1990,7 +2015,6 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1990
2015
|
};
|
|
1991
2016
|
|
|
1992
2017
|
// src/puppeteer/page.ts
|
|
1993
|
-
var import_utils6 = require("@letsscrapedata/utils");
|
|
1994
2018
|
var PuppeteerPage = class extends import_node_events4.default {
|
|
1995
2019
|
#lsdBrowserContext;
|
|
1996
2020
|
#page;
|
|
@@ -2150,9 +2174,9 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2150
2174
|
const page = this.#page;
|
|
2151
2175
|
const pageId = this.#pageId;
|
|
2152
2176
|
page.on("close", async () => {
|
|
2153
|
-
|
|
2177
|
+
loginfo(`##browser ${pageId} closed`);
|
|
2154
2178
|
if (!page.pageInfo) {
|
|
2155
|
-
|
|
2179
|
+
logerr(`Logic error in page.on("close")`);
|
|
2156
2180
|
}
|
|
2157
2181
|
this.emit("pageClose");
|
|
2158
2182
|
this.#lsdBrowserContext.emit("pageClose", this);
|
|
@@ -2171,12 +2195,12 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2171
2195
|
pageInfo.relatedId = page.pageInfo.taskId;
|
|
2172
2196
|
}
|
|
2173
2197
|
} else {
|
|
2174
|
-
|
|
2198
|
+
logerr(`##browser ${pageId} has popup without page.pageInfo`);
|
|
2175
2199
|
}
|
|
2176
|
-
|
|
2200
|
+
loginfo(`##browser ${pageId} has popup ${popupPageId}`);
|
|
2177
2201
|
this.emit("pagePopup", evtData);
|
|
2178
2202
|
} else {
|
|
2179
|
-
|
|
2203
|
+
logerr(`##browser ${pageId} has popup page with null page`);
|
|
2180
2204
|
}
|
|
2181
2205
|
});
|
|
2182
2206
|
}
|
|
@@ -2188,7 +2212,7 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2188
2212
|
this.#lsdBrowserContext = browserContext;
|
|
2189
2213
|
this.#page = page;
|
|
2190
2214
|
this.#status = "free";
|
|
2191
|
-
const currentTime = (0,
|
|
2215
|
+
const currentTime = (0, import_utils7.getCurrentUnixTime)();
|
|
2192
2216
|
const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
|
|
2193
2217
|
this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
|
|
2194
2218
|
this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
|
|
@@ -2582,7 +2606,7 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2582
2606
|
});
|
|
2583
2607
|
break;
|
|
2584
2608
|
default:
|
|
2585
|
-
(0,
|
|
2609
|
+
(0, import_utils7.unreachable)(action);
|
|
2586
2610
|
}
|
|
2587
2611
|
return true;
|
|
2588
2612
|
}
|
|
@@ -2786,7 +2810,6 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2786
2810
|
};
|
|
2787
2811
|
|
|
2788
2812
|
// src/puppeteer/context.ts
|
|
2789
|
-
var import_utils7 = require("@letsscrapedata/utils");
|
|
2790
2813
|
var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
2791
2814
|
#lsdBrowser;
|
|
2792
2815
|
#browserIdx;
|
|
@@ -2815,7 +2838,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2815
2838
|
}
|
|
2816
2839
|
const pages = await this.#browserContext.pages();
|
|
2817
2840
|
const openType = this.#lsdBrowser.browserCreationMethod();
|
|
2818
|
-
const lastStatusUpdateTime = (0,
|
|
2841
|
+
const lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2819
2842
|
for (const page of pages) {
|
|
2820
2843
|
const pageInfo = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType, openTime: this.#createTime, lastStatusUpdateTime, taskId: 0, relatedId: 0, misc: {} };
|
|
2821
2844
|
const lsdPage = new PuppeteerPage(this, page, pageInfo);
|
|
@@ -2826,7 +2849,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2826
2849
|
await lsdPage.setUserAgent(this.#userAgent);
|
|
2827
2850
|
}
|
|
2828
2851
|
this.#lsdPages.push(lsdPage);
|
|
2829
|
-
|
|
2852
|
+
loginfo(`##browser ${lsdPage.id()} ${openType}ed`);
|
|
2830
2853
|
}
|
|
2831
2854
|
}
|
|
2832
2855
|
constructor(lsdBrowser, browserContext, browserContextCreationMethod, incognito = false, proxy = null, browserIdx = 0, browserContextIdx = 0, maxPagesPerBrowserContext = 20, maxPageFreeSeconds = 0, userAgent = "", maxViewportOfNewPage = true) {
|
|
@@ -2843,7 +2866,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2843
2866
|
this.#browserContext = browserContext;
|
|
2844
2867
|
this.#browserContextCreationMethod = browserContextCreationMethod;
|
|
2845
2868
|
this.#userAgent = userAgent;
|
|
2846
|
-
const currentTime = (0,
|
|
2869
|
+
const currentTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2847
2870
|
this.#createTime = currentTime;
|
|
2848
2871
|
this.#lastStatusUpdateTime = currentTime;
|
|
2849
2872
|
this.#status = "free";
|
|
@@ -2865,9 +2888,9 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2865
2888
|
const pageInfo = page.pageInfo;
|
|
2866
2889
|
if (pageInfo) {
|
|
2867
2890
|
const { browserIdx: browserIdx2, browserContextIdx: browserContextIdx2, pageIdx } = pageInfo;
|
|
2868
|
-
|
|
2891
|
+
logwarn(`##browser page-${browserIdx2}-${browserContextIdx2}-${pageIdx} has been already created`);
|
|
2869
2892
|
} else {
|
|
2870
|
-
const currentTime2 = (0,
|
|
2893
|
+
const currentTime2 = (0, import_utils8.getCurrentUnixTime)();
|
|
2871
2894
|
const pageInfo2 = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType: "other", openTime: currentTime2, lastStatusUpdateTime: currentTime2, taskId: 0, relatedId: 0, misc: {} };
|
|
2872
2895
|
const lsdPage = new PuppeteerPage(this, page, pageInfo2);
|
|
2873
2896
|
if (this.#maxViewportOfNewPage) {
|
|
@@ -2877,18 +2900,18 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2877
2900
|
await lsdPage.setUserAgent(this.#userAgent);
|
|
2878
2901
|
}
|
|
2879
2902
|
this.#lsdPages.push(lsdPage);
|
|
2880
|
-
|
|
2903
|
+
loginfo(`##page ${lsdPage.id()} created`);
|
|
2881
2904
|
}
|
|
2882
2905
|
}
|
|
2883
2906
|
});
|
|
2884
2907
|
this.on("pageClose", (lsdPage) => {
|
|
2885
2908
|
if (!(lsdPage instanceof PuppeteerPage)) {
|
|
2886
|
-
|
|
2909
|
+
logerr(`Invalid data in LsdBrowserContext.on("pageClose)`);
|
|
2887
2910
|
return;
|
|
2888
2911
|
}
|
|
2889
2912
|
const idx = this.#lsdPages.findIndex((p) => p === lsdPage);
|
|
2890
2913
|
if (idx < 0) {
|
|
2891
|
-
|
|
2914
|
+
logerr(`Invalid lsdPage in LsdBrowserContext.on("pageClose)`);
|
|
2892
2915
|
return;
|
|
2893
2916
|
}
|
|
2894
2917
|
this.#lsdPages.splice(idx, 1);
|
|
@@ -2907,8 +2930,8 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2907
2930
|
async close() {
|
|
2908
2931
|
if (this.#browserContext) {
|
|
2909
2932
|
this.#status = "closed";
|
|
2910
|
-
this.#lastStatusUpdateTime = (0,
|
|
2911
|
-
|
|
2933
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2934
|
+
loginfo(`browserContext ${this.id()} closed at ${this.#lastStatusUpdateTime}`);
|
|
2912
2935
|
if (this.#browserContextCreationMethod !== "launch") {
|
|
2913
2936
|
await this.#browserContext.close();
|
|
2914
2937
|
}
|
|
@@ -2922,15 +2945,15 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2922
2945
|
this.#gettingPage = true;
|
|
2923
2946
|
return true;
|
|
2924
2947
|
} else {
|
|
2925
|
-
await (0,
|
|
2948
|
+
await (0, import_utils8.sleep)(200);
|
|
2926
2949
|
}
|
|
2927
2950
|
}
|
|
2928
|
-
|
|
2951
|
+
logwarn(`Cannot get the gettingLock.`);
|
|
2929
2952
|
return false;
|
|
2930
2953
|
}
|
|
2931
2954
|
#freeGettingLock() {
|
|
2932
2955
|
if (!this.#gettingPage) {
|
|
2933
|
-
|
|
2956
|
+
logwarn(`Getting lock is already free now.`);
|
|
2934
2957
|
}
|
|
2935
2958
|
this.#gettingPage = false;
|
|
2936
2959
|
}
|
|
@@ -2939,7 +2962,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2939
2962
|
maxPageFreeSeconds = this.#maxPageFreeSeconds;
|
|
2940
2963
|
}
|
|
2941
2964
|
if (maxPageFreeSeconds <= 0) {
|
|
2942
|
-
|
|
2965
|
+
logwarn(`Please set valid maxPageFreeSeconds to close free pages`);
|
|
2943
2966
|
return false;
|
|
2944
2967
|
}
|
|
2945
2968
|
const gotLock = await this.#tryToGetGettingLock();
|
|
@@ -2947,7 +2970,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2947
2970
|
return false;
|
|
2948
2971
|
}
|
|
2949
2972
|
try {
|
|
2950
|
-
const maxUpdateTime = (0,
|
|
2973
|
+
const maxUpdateTime = (0, import_utils8.getCurrentUnixTime)() - this.#maxPageFreeSeconds;
|
|
2951
2974
|
let freePages = this.#lsdPages.filter((p) => p.isFree() && p.pageInfo().lastStatusUpdateTime < maxUpdateTime);
|
|
2952
2975
|
if (freePages.length === this.#lsdPages.length) {
|
|
2953
2976
|
freePages = freePages.slice(1);
|
|
@@ -2958,7 +2981,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2958
2981
|
this.#freeGettingLock();
|
|
2959
2982
|
return true;
|
|
2960
2983
|
} catch (err) {
|
|
2961
|
-
|
|
2984
|
+
logerr(err);
|
|
2962
2985
|
this.#freeGettingLock();
|
|
2963
2986
|
return false;
|
|
2964
2987
|
}
|
|
@@ -2983,7 +3006,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2983
3006
|
}
|
|
2984
3007
|
try {
|
|
2985
3008
|
if (this.#lsdPages.length === 0) {
|
|
2986
|
-
await (0,
|
|
3009
|
+
await (0, import_utils8.sleep)(1e3);
|
|
2987
3010
|
}
|
|
2988
3011
|
let lsdPage = this.#lsdPages.find((p) => p.isFree());
|
|
2989
3012
|
if (lsdPage) {
|
|
@@ -2995,7 +3018,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2995
3018
|
return null;
|
|
2996
3019
|
}
|
|
2997
3020
|
const page = await this.#browserContext.newPage();
|
|
2998
|
-
await (0,
|
|
3021
|
+
await (0, import_utils8.sleep)(2e3);
|
|
2999
3022
|
const pageInfo = page.pageInfo;
|
|
3000
3023
|
if (!pageInfo) {
|
|
3001
3024
|
throw new Error(`Logic error in getPage`);
|
|
@@ -3012,7 +3035,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3012
3035
|
return null;
|
|
3013
3036
|
}
|
|
3014
3037
|
} catch (err) {
|
|
3015
|
-
|
|
3038
|
+
logerr(err);
|
|
3016
3039
|
this.#freeGettingLock();
|
|
3017
3040
|
return null;
|
|
3018
3041
|
}
|
|
@@ -3020,7 +3043,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3020
3043
|
free(clearStateData = false) {
|
|
3021
3044
|
if (this.#status === "busy") {
|
|
3022
3045
|
this.#status = "free";
|
|
3023
|
-
this.#lastStatusUpdateTime = (0,
|
|
3046
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
3024
3047
|
if (clearStateData) {
|
|
3025
3048
|
}
|
|
3026
3049
|
return true;
|
|
@@ -3089,12 +3112,12 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3089
3112
|
}, localStorage);
|
|
3090
3113
|
}
|
|
3091
3114
|
}
|
|
3092
|
-
await (0,
|
|
3115
|
+
await (0, import_utils8.sleep)(2e3);
|
|
3093
3116
|
await origPage.setRequestInterception(false);
|
|
3094
3117
|
await page.free();
|
|
3095
3118
|
return true;
|
|
3096
3119
|
} catch (err) {
|
|
3097
|
-
|
|
3120
|
+
logerr(err);
|
|
3098
3121
|
return false;
|
|
3099
3122
|
}
|
|
3100
3123
|
}
|
|
@@ -3104,7 +3127,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3104
3127
|
use() {
|
|
3105
3128
|
if (this.#status === "free") {
|
|
3106
3129
|
this.#status = "busy";
|
|
3107
|
-
this.#lastStatusUpdateTime = (0,
|
|
3130
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
3108
3131
|
return true;
|
|
3109
3132
|
} else {
|
|
3110
3133
|
return false;
|
|
@@ -3116,7 +3139,6 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3116
3139
|
};
|
|
3117
3140
|
|
|
3118
3141
|
// src/puppeteer/browser.ts
|
|
3119
|
-
var import_utils8 = require("@letsscrapedata/utils");
|
|
3120
3142
|
var PuppeteerBrowser = class extends import_node_events6.default {
|
|
3121
3143
|
#browser;
|
|
3122
3144
|
#browserIdx;
|
|
@@ -3158,7 +3180,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3158
3180
|
this.#browser = browser;
|
|
3159
3181
|
this.#browserIdx = browserIdx;
|
|
3160
3182
|
this.#pid = pid;
|
|
3161
|
-
this.#createTime = (0,
|
|
3183
|
+
this.#createTime = (0, import_utils9.getCurrentUnixTime)();
|
|
3162
3184
|
this.#lsdBrowserContexts = [];
|
|
3163
3185
|
this.#browserControllerType = "puppeteer";
|
|
3164
3186
|
this.#browserType = browerType;
|
|
@@ -3169,35 +3191,35 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3169
3191
|
this.#executablePath = executablePath;
|
|
3170
3192
|
this.#nextBrowserContextIdx = 1;
|
|
3171
3193
|
this.#closeFreePagesIntervalId = null;
|
|
3172
|
-
|
|
3194
|
+
loginfo(`##browser ${this.#browserType} ${this.id()} ${this.#browserCreationMethod}ed by ${this.#browserControllerType}`);
|
|
3173
3195
|
const browserContexts = browser.browserContexts();
|
|
3174
3196
|
const incognito = typeof options?.incognito === "boolean" ? options.incognito : false;
|
|
3175
3197
|
for (const browserContext of browserContexts) {
|
|
3176
3198
|
const lsdBrowserContext = new PuppeteerBrowserContext(this, browserContext, "launch", incognito, this.#proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), this.#userAgent(), maxViewportOfNewPage);
|
|
3177
3199
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
3178
|
-
|
|
3200
|
+
loginfo(`##browser ${lsdBrowserContext.id()} ${this.#browserCreationMethod}ed`);
|
|
3179
3201
|
}
|
|
3180
3202
|
browser.on("disconnected", () => {
|
|
3181
|
-
|
|
3203
|
+
loginfo(`##browser ${this.id()} disconnected`);
|
|
3182
3204
|
if (this.#lsdBrowserContexts.length > 0) {
|
|
3183
|
-
|
|
3205
|
+
logerr(`${this.id()} has browserContexts when disconnected`);
|
|
3184
3206
|
}
|
|
3185
3207
|
});
|
|
3186
3208
|
this.on("browserContextClose", (lsdBrowserContext) => {
|
|
3187
3209
|
if (!(lsdBrowserContext instanceof PuppeteerBrowserContext)) {
|
|
3188
|
-
|
|
3210
|
+
logerr(`Invalid data in LsdBrowser.on("browserContextClose)`);
|
|
3189
3211
|
return;
|
|
3190
3212
|
}
|
|
3191
3213
|
const idx = this.#lsdBrowserContexts.findIndex((bc) => bc === lsdBrowserContext);
|
|
3192
3214
|
if (idx < 0) {
|
|
3193
|
-
|
|
3215
|
+
logerr(`Invalid lsdBrowserContext in LsdBrowser.on("browserContextClose)`);
|
|
3194
3216
|
return;
|
|
3195
3217
|
}
|
|
3196
|
-
|
|
3218
|
+
loginfo(`##browser ${lsdBrowserContext.id()} closed
|
|
3197
3219
|
`);
|
|
3198
3220
|
this.#lsdBrowserContexts.splice(idx, 1);
|
|
3199
3221
|
if (this.#lsdBrowserContexts.length === 0) {
|
|
3200
|
-
|
|
3222
|
+
loginfo(`##browser ${this.id()} has no browserContexts now`);
|
|
3201
3223
|
}
|
|
3202
3224
|
return;
|
|
3203
3225
|
});
|
|
@@ -3214,7 +3236,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3214
3236
|
}
|
|
3215
3237
|
async newBrowserContext(options) {
|
|
3216
3238
|
if (this.#lsdBrowserContexts.length >= this.#maxBrowserContextsPerBrowser()) {
|
|
3217
|
-
|
|
3239
|
+
logwarn(`##browser ${this.id()} can not create more new browserContext`);
|
|
3218
3240
|
return null;
|
|
3219
3241
|
}
|
|
3220
3242
|
const browserContextOptions = {};
|
|
@@ -3227,7 +3249,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3227
3249
|
const { maxViewportOfNewPage = this.#options.maxViewportOfNewPage } = options ? options : {};
|
|
3228
3250
|
const lsdBrowserContext = new PuppeteerBrowserContext(this, browserContext, "new", true, proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), userAgent, maxViewportOfNewPage);
|
|
3229
3251
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
3230
|
-
|
|
3252
|
+
loginfo(`##browser ${lsdBrowserContext.id()} created`);
|
|
3231
3253
|
return lsdBrowserContext;
|
|
3232
3254
|
}
|
|
3233
3255
|
async close() {
|
|
@@ -3279,7 +3301,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3279
3301
|
}
|
|
3280
3302
|
async pidUsage() {
|
|
3281
3303
|
if (this.#pid > 0) {
|
|
3282
|
-
const usage = await (0,
|
|
3304
|
+
const usage = await (0, import_utils9.getPerformanceOfPidTree)(this.#pid, "MB");
|
|
3283
3305
|
return usage;
|
|
3284
3306
|
} else {
|
|
3285
3307
|
return { cpu: 0, memory: 0 };
|
|
@@ -3626,10 +3648,10 @@ var CheerioPage = class extends import_node_events7.default {
|
|
|
3626
3648
|
var import_os = __toESM(require("os"), 1);
|
|
3627
3649
|
var import_puppeteer = __toESM(require("puppeteer"), 1);
|
|
3628
3650
|
var import_playwright = __toESM(require("playwright"), 1);
|
|
3629
|
-
var import_utils9 = require("@letsscrapedata/utils");
|
|
3630
3651
|
var import_puppeteer_extra = __toESM(require("puppeteer-extra"), 1);
|
|
3631
3652
|
var playwrightExtra = __toESM(require("playwright-extra"), 1);
|
|
3632
3653
|
var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
|
|
3654
|
+
var import_utils10 = require("@letsscrapedata/utils");
|
|
3633
3655
|
var LsdBrowserController = class _LsdBrowserController {
|
|
3634
3656
|
static #forbidConstructor = false;
|
|
3635
3657
|
#puppeteer;
|
|
@@ -3702,7 +3724,7 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3702
3724
|
}
|
|
3703
3725
|
break;
|
|
3704
3726
|
default:
|
|
3705
|
-
(0,
|
|
3727
|
+
(0, import_utils10.unreachable)(browserType);
|
|
3706
3728
|
}
|
|
3707
3729
|
return true;
|
|
3708
3730
|
}
|
|
@@ -3730,22 +3752,22 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3730
3752
|
const actOptions = { closeFreePagesIntervalSeconds, maxBrowserContextsPerBrowser, maxPagesPerBrowserContext, maxPageFreeSeconds, maxViewportOfNewPage, proxy, timeout, args, executablePath, maxWindowSize, headless, minBrowserContexts, incognito, proxyPerBrowserContext, userDataDir, userAgent };
|
|
3731
3753
|
let idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--incoginto"));
|
|
3732
3754
|
if (idx >= 0) {
|
|
3733
|
-
|
|
3755
|
+
logwarn(`Please use options.incognito instead when launching new browser.`);
|
|
3734
3756
|
args.splice(idx, 1);
|
|
3735
3757
|
}
|
|
3736
3758
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--proxy-server"));
|
|
3737
3759
|
if (idx >= 0) {
|
|
3738
|
-
|
|
3760
|
+
logwarn(`Please use options.proxy instead when launching new browser.`);
|
|
3739
3761
|
args.splice(idx, 1);
|
|
3740
3762
|
}
|
|
3741
3763
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--user-data-dir"));
|
|
3742
3764
|
if (idx >= 0) {
|
|
3743
|
-
|
|
3765
|
+
logwarn(`Please use options.userDataDir instead when launching new browser.`);
|
|
3744
3766
|
args.splice(idx, 1);
|
|
3745
3767
|
}
|
|
3746
3768
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--start-maximized"));
|
|
3747
3769
|
if (idx >= 0) {
|
|
3748
|
-
|
|
3770
|
+
logwarn(`Please use options.maxWindowSize instead when launching new browser.`);
|
|
3749
3771
|
args.splice(idx, 1);
|
|
3750
3772
|
}
|
|
3751
3773
|
let lsdBrowser;
|
|
@@ -3855,10 +3877,10 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3855
3877
|
}
|
|
3856
3878
|
const u = new URL(browserUrl);
|
|
3857
3879
|
const port = u.port ? parseInt(u.port) : 80;
|
|
3858
|
-
const pids = await (0,
|
|
3880
|
+
const pids = await (0, import_utils10.getPidsListeningOnPort)(port);
|
|
3859
3881
|
let browserPid = 0;
|
|
3860
3882
|
if (pids.length !== 1) {
|
|
3861
|
-
|
|
3883
|
+
logerr(`##browser pids.length ${pids.length} is not 1 when trying to connect to browserUrl ${browserUrl}`);
|
|
3862
3884
|
} else {
|
|
3863
3885
|
browserPid = pids[0];
|
|
3864
3886
|
}
|
|
@@ -3906,5 +3928,6 @@ var controller = new LsdBrowserController();
|
|
|
3906
3928
|
PuppeteerBrowserContext,
|
|
3907
3929
|
PuppeteerElement,
|
|
3908
3930
|
PuppeteerPage,
|
|
3909
|
-
controller
|
|
3931
|
+
controller,
|
|
3932
|
+
setControllerLogFun
|
|
3910
3933
|
});
|