@letsscrapedata/controller 0.0.49 → 0.0.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +129 -116
- package/dist/index.d.cts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +83 -70
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -46,17 +46,31 @@ module.exports = __toCommonJS(src_exports);
|
|
|
46
46
|
|
|
47
47
|
// src/playwright/browser.ts
|
|
48
48
|
var import_node_events3 = __toESM(require("events"), 1);
|
|
49
|
+
var import_utils5 = require("@letsscrapedata/utils");
|
|
49
50
|
|
|
50
51
|
// src/playwright/context.ts
|
|
51
52
|
var import_node_events2 = __toESM(require("events"), 1);
|
|
52
|
-
var
|
|
53
|
+
var import_utils4 = require("@letsscrapedata/utils");
|
|
54
|
+
|
|
55
|
+
// src/utils/log.ts
|
|
56
|
+
var import_utils = require("@letsscrapedata/utils");
|
|
57
|
+
var pkgLog = import_utils.log;
|
|
58
|
+
async function loginfo(...args) {
|
|
59
|
+
await pkgLog(import_utils.LogLevel.INF, ...args);
|
|
60
|
+
}
|
|
61
|
+
async function logwarn(...args) {
|
|
62
|
+
await pkgLog(import_utils.LogLevel.WRN, ...args);
|
|
63
|
+
}
|
|
64
|
+
async function logerr(...args) {
|
|
65
|
+
await pkgLog(import_utils.LogLevel.ERR, ...args);
|
|
66
|
+
}
|
|
53
67
|
|
|
54
68
|
// src/playwright/page.ts
|
|
55
69
|
var import_node_events = __toESM(require("events"), 1);
|
|
56
|
-
var
|
|
70
|
+
var import_utils3 = require("@letsscrapedata/utils");
|
|
57
71
|
|
|
58
72
|
// src/playwright/element.ts
|
|
59
|
-
var
|
|
73
|
+
var import_utils2 = require("@letsscrapedata/utils");
|
|
60
74
|
var PlaywrightElement = class _PlaywrightElement {
|
|
61
75
|
#frame;
|
|
62
76
|
#locator;
|
|
@@ -161,7 +175,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
161
175
|
retObj.locators = locators;
|
|
162
176
|
return retObj;
|
|
163
177
|
} catch (err) {
|
|
164
|
-
|
|
178
|
+
loginfo(err);
|
|
165
179
|
return retObj;
|
|
166
180
|
}
|
|
167
181
|
}
|
|
@@ -236,7 +250,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
236
250
|
} else if (clickType === "evaluate") {
|
|
237
251
|
await this.#locator.evaluate(async (ev) => await ev.click());
|
|
238
252
|
} else {
|
|
239
|
-
(0,
|
|
253
|
+
(0, import_utils2.unreachable)(clickType);
|
|
240
254
|
}
|
|
241
255
|
return true;
|
|
242
256
|
}
|
|
@@ -309,7 +323,7 @@ var PlaywrightElement = class _PlaywrightElement {
|
|
|
309
323
|
}
|
|
310
324
|
break;
|
|
311
325
|
default:
|
|
312
|
-
(0,
|
|
326
|
+
(0, import_utils2.unreachable)(type);
|
|
313
327
|
}
|
|
314
328
|
return true;
|
|
315
329
|
}
|
|
@@ -349,7 +363,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
349
363
|
const cookieItems = await this.#getCookies(page);
|
|
350
364
|
const domainSet = new Set(cookieItems.map((c) => c.domain));
|
|
351
365
|
if (domainSet.size !== 1) {
|
|
352
|
-
|
|
366
|
+
logwarn(`Domains in clearCookies: ${Array.from(domainSet.values())}`);
|
|
353
367
|
}
|
|
354
368
|
for (const domain of domainSet.values()) {
|
|
355
369
|
await browserContext.clearCookies({ domain });
|
|
@@ -514,7 +528,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
514
528
|
retObj.locators = locators;
|
|
515
529
|
return retObj;
|
|
516
530
|
} catch (err) {
|
|
517
|
-
|
|
531
|
+
loginfo(err);
|
|
518
532
|
return retObj;
|
|
519
533
|
}
|
|
520
534
|
}
|
|
@@ -525,9 +539,9 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
525
539
|
const page = this.#page;
|
|
526
540
|
const pageId = this.#pageId;
|
|
527
541
|
page.on("close", async () => {
|
|
528
|
-
|
|
542
|
+
loginfo(`##browser ${pageId} closed`);
|
|
529
543
|
if (!page.pageInfo) {
|
|
530
|
-
|
|
544
|
+
logerr(`Logic error in page.on("close")`);
|
|
531
545
|
}
|
|
532
546
|
this.emit("pageClose");
|
|
533
547
|
this.#lsdBrowserContext.emit("pageClose", this);
|
|
@@ -546,12 +560,12 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
546
560
|
pageInfo.relatedId = page.pageInfo.taskId;
|
|
547
561
|
}
|
|
548
562
|
} else {
|
|
549
|
-
|
|
563
|
+
logerr(`##browser ${pageId} has popup without page.pageInfo`);
|
|
550
564
|
}
|
|
551
|
-
|
|
565
|
+
loginfo(`##browser ${pageId} has popup ${popupPageId}`);
|
|
552
566
|
this.emit("pagePopup", evtData);
|
|
553
567
|
} else {
|
|
554
|
-
|
|
568
|
+
logerr(`##browser ${pageId} has popup page with null page`);
|
|
555
569
|
}
|
|
556
570
|
});
|
|
557
571
|
}
|
|
@@ -563,7 +577,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
563
577
|
this.#lsdBrowserContext = browserContext;
|
|
564
578
|
this.#page = page;
|
|
565
579
|
this.#status = "free";
|
|
566
|
-
const currentTime = (0,
|
|
580
|
+
const currentTime = (0, import_utils3.getCurrentUnixTime)();
|
|
567
581
|
const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
|
|
568
582
|
this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
|
|
569
583
|
this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
|
|
@@ -618,7 +632,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
618
632
|
}
|
|
619
633
|
return true;
|
|
620
634
|
} catch (err) {
|
|
621
|
-
|
|
635
|
+
logerr(err);
|
|
622
636
|
return false;
|
|
623
637
|
}
|
|
624
638
|
}
|
|
@@ -921,7 +935,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
921
935
|
}
|
|
922
936
|
return true;
|
|
923
937
|
} catch (err) {
|
|
924
|
-
|
|
938
|
+
logerr(err);
|
|
925
939
|
return false;
|
|
926
940
|
}
|
|
927
941
|
}
|
|
@@ -931,7 +945,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
931
945
|
}
|
|
932
946
|
const actOptions = Array.isArray(options) ? options : [options];
|
|
933
947
|
if (actOptions.length <= 0) {
|
|
934
|
-
|
|
948
|
+
logwarn("Invalid paras in setRequestInterception");
|
|
935
949
|
return false;
|
|
936
950
|
}
|
|
937
951
|
const firstRequestInterception = this.#resquestInterceptionOptions.length <= 0;
|
|
@@ -942,7 +956,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
942
956
|
this.#resquestInterceptionOptions.push(option);
|
|
943
957
|
break;
|
|
944
958
|
default:
|
|
945
|
-
(0,
|
|
959
|
+
(0, import_utils3.unreachable)(option.action);
|
|
946
960
|
}
|
|
947
961
|
}
|
|
948
962
|
if (firstRequestInterception && this.#resquestInterceptionOptions.length > 0) {
|
|
@@ -966,7 +980,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
966
980
|
});
|
|
967
981
|
break;
|
|
968
982
|
default:
|
|
969
|
-
(0,
|
|
983
|
+
(0, import_utils3.unreachable)(action);
|
|
970
984
|
}
|
|
971
985
|
return true;
|
|
972
986
|
} else {
|
|
@@ -975,7 +989,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
975
989
|
await route.continue();
|
|
976
990
|
return true;
|
|
977
991
|
} catch (err) {
|
|
978
|
-
|
|
992
|
+
logerr(err);
|
|
979
993
|
return false;
|
|
980
994
|
}
|
|
981
995
|
});
|
|
@@ -1019,7 +1033,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
1019
1033
|
requestData,
|
|
1020
1034
|
responseData
|
|
1021
1035
|
});
|
|
1022
|
-
|
|
1036
|
+
loginfo(`##browser cache matched response: ${requestUrl}`);
|
|
1023
1037
|
}
|
|
1024
1038
|
if (typeof handler === "function") {
|
|
1025
1039
|
const pageData = { pageUrl, cookies: "" };
|
|
@@ -1028,7 +1042,7 @@ var PlaywrightPage = class extends import_node_events.default {
|
|
|
1028
1042
|
}
|
|
1029
1043
|
return;
|
|
1030
1044
|
} catch (err) {
|
|
1031
|
-
|
|
1045
|
+
logerr(err);
|
|
1032
1046
|
return;
|
|
1033
1047
|
}
|
|
1034
1048
|
}
|
|
@@ -1233,7 +1247,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1233
1247
|
}
|
|
1234
1248
|
const pages = this.#browserContext.pages();
|
|
1235
1249
|
const openType = this.#lsdBrowser.browserCreationMethod();
|
|
1236
|
-
const lastStatusUpdateTime = (0,
|
|
1250
|
+
const lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1237
1251
|
for (const page of pages) {
|
|
1238
1252
|
const pageInfo = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType, openTime: this.#createTime, lastStatusUpdateTime, taskId: 0, relatedId: 0, misc: {} };
|
|
1239
1253
|
const lsdPage = new PlaywrightPage(this, page, pageInfo);
|
|
@@ -1241,7 +1255,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1241
1255
|
await lsdPage.maximizeViewport();
|
|
1242
1256
|
}
|
|
1243
1257
|
this.#lsdPages.push(lsdPage);
|
|
1244
|
-
|
|
1258
|
+
loginfo(`##browser ${lsdPage.id()} ${openType}ed`);
|
|
1245
1259
|
}
|
|
1246
1260
|
}
|
|
1247
1261
|
constructor(lsdBrowser, browserContext, browserContextCreationMethod, incognito = false, proxy = null, browserIdx = 0, browserContextIdx = 0, maxPagesPerBrowserContext = 20, maxPageFreeSeconds = 0, maxViewportOfNewPage = true) {
|
|
@@ -1259,7 +1273,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1259
1273
|
this.#browserContextCreationMethod = browserContextCreationMethod;
|
|
1260
1274
|
const apiRequestContext = browserContext.request;
|
|
1261
1275
|
this.#apiContext = new PlaywrightApiContext(apiRequestContext);
|
|
1262
|
-
const currentTime = (0,
|
|
1276
|
+
const currentTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1263
1277
|
this.#createTime = currentTime;
|
|
1264
1278
|
this.#lastStatusUpdateTime = currentTime;
|
|
1265
1279
|
this.#status = "free";
|
|
@@ -1276,32 +1290,32 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1276
1290
|
const pageInfo = page.pageInfo;
|
|
1277
1291
|
if (pageInfo) {
|
|
1278
1292
|
const { browserIdx: browserIdx2, browserContextIdx: browserContextIdx2, pageIdx } = pageInfo;
|
|
1279
|
-
|
|
1293
|
+
logwarn(`##browser page-${browserIdx2}-${browserContextIdx2}-${pageIdx} has been already created`);
|
|
1280
1294
|
} else {
|
|
1281
|
-
const currentTime2 = (0,
|
|
1295
|
+
const currentTime2 = (0, import_utils4.getCurrentUnixTime)();
|
|
1282
1296
|
const pageInfo2 = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType: "other", openTime: currentTime2, lastStatusUpdateTime: currentTime2, taskId: 0, relatedId: 0, misc: {} };
|
|
1283
1297
|
const lsdPage = new PlaywrightPage(this, page, pageInfo2);
|
|
1284
1298
|
if (this.#maxViewportOfNewPage) {
|
|
1285
1299
|
await lsdPage.maximizeViewport();
|
|
1286
1300
|
}
|
|
1287
1301
|
this.#lsdPages.push(lsdPage);
|
|
1288
|
-
|
|
1302
|
+
loginfo(`##page ${lsdPage.id()} created`);
|
|
1289
1303
|
}
|
|
1290
1304
|
});
|
|
1291
1305
|
browserContext.on("close", (bc) => {
|
|
1292
1306
|
if (browserContext !== bc) {
|
|
1293
|
-
|
|
1307
|
+
logerr(`##browser different browserContext in browserContext.on("close")`);
|
|
1294
1308
|
}
|
|
1295
1309
|
this.#lsdBrowser.emit("browserContextClose", this);
|
|
1296
1310
|
});
|
|
1297
1311
|
this.on("pageClose", (lsdPage) => {
|
|
1298
1312
|
if (!(lsdPage instanceof PlaywrightPage)) {
|
|
1299
|
-
|
|
1313
|
+
logerr(`Invalid data in LsdBrowserContext.on("pageClose)`);
|
|
1300
1314
|
return;
|
|
1301
1315
|
}
|
|
1302
1316
|
const idx = this.#lsdPages.findIndex((p) => p === lsdPage);
|
|
1303
1317
|
if (idx < 0) {
|
|
1304
|
-
|
|
1318
|
+
logerr(`Invalid lsdPage in LsdBrowserContext.on("pageClose)`);
|
|
1305
1319
|
return;
|
|
1306
1320
|
}
|
|
1307
1321
|
this.#lsdPages.splice(idx, 1);
|
|
@@ -1317,8 +1331,8 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1317
1331
|
async close() {
|
|
1318
1332
|
if (this.#browserContext) {
|
|
1319
1333
|
this.#status = "closed";
|
|
1320
|
-
this.#lastStatusUpdateTime = (0,
|
|
1321
|
-
|
|
1334
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1335
|
+
loginfo(`browserContext ${this.id()} closed at ${this.#lastStatusUpdateTime}`);
|
|
1322
1336
|
await this.#browserContext.close();
|
|
1323
1337
|
}
|
|
1324
1338
|
return true;
|
|
@@ -1330,15 +1344,15 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1330
1344
|
this.#gettingPage = true;
|
|
1331
1345
|
return true;
|
|
1332
1346
|
} else {
|
|
1333
|
-
await (0,
|
|
1347
|
+
await (0, import_utils4.sleep)(200);
|
|
1334
1348
|
}
|
|
1335
1349
|
}
|
|
1336
|
-
|
|
1350
|
+
logwarn(`Cannot get the gettingLock.`);
|
|
1337
1351
|
return false;
|
|
1338
1352
|
}
|
|
1339
1353
|
#freeGettingLock() {
|
|
1340
1354
|
if (!this.#gettingPage) {
|
|
1341
|
-
|
|
1355
|
+
logwarn(`Getting lock is already free now.`);
|
|
1342
1356
|
}
|
|
1343
1357
|
this.#gettingPage = false;
|
|
1344
1358
|
}
|
|
@@ -1347,7 +1361,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1347
1361
|
maxPageFreeSeconds = this.#maxPageFreeSeconds;
|
|
1348
1362
|
}
|
|
1349
1363
|
if (maxPageFreeSeconds <= 0) {
|
|
1350
|
-
|
|
1364
|
+
logwarn(`Please set valid maxPageFreeSeconds to close free pages`);
|
|
1351
1365
|
return false;
|
|
1352
1366
|
}
|
|
1353
1367
|
const gotLock = await this.#tryToGetGettingLock();
|
|
@@ -1355,7 +1369,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1355
1369
|
return false;
|
|
1356
1370
|
}
|
|
1357
1371
|
try {
|
|
1358
|
-
const maxUpdateTime = (0,
|
|
1372
|
+
const maxUpdateTime = (0, import_utils4.getCurrentUnixTime)() - this.#maxPageFreeSeconds;
|
|
1359
1373
|
let freePages = this.#lsdPages.filter((p) => p.isFree() && p.pageInfo().lastStatusUpdateTime < maxUpdateTime);
|
|
1360
1374
|
if (freePages.length === this.#lsdPages.length) {
|
|
1361
1375
|
freePages = freePages.slice(1);
|
|
@@ -1366,7 +1380,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1366
1380
|
this.#freeGettingLock();
|
|
1367
1381
|
return true;
|
|
1368
1382
|
} catch (err) {
|
|
1369
|
-
|
|
1383
|
+
logerr(err);
|
|
1370
1384
|
this.#freeGettingLock();
|
|
1371
1385
|
return false;
|
|
1372
1386
|
}
|
|
@@ -1401,7 +1415,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1401
1415
|
return null;
|
|
1402
1416
|
}
|
|
1403
1417
|
const page = await this.#browserContext.newPage();
|
|
1404
|
-
await (0,
|
|
1418
|
+
await (0, import_utils4.sleep)(2e3);
|
|
1405
1419
|
const pageInfo = page.pageInfo;
|
|
1406
1420
|
if (!pageInfo) {
|
|
1407
1421
|
throw new Error(`Logic error in getPage`);
|
|
@@ -1418,7 +1432,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1418
1432
|
return null;
|
|
1419
1433
|
}
|
|
1420
1434
|
} catch (err) {
|
|
1421
|
-
|
|
1435
|
+
logerr(err);
|
|
1422
1436
|
this.#freeGettingLock();
|
|
1423
1437
|
return null;
|
|
1424
1438
|
}
|
|
@@ -1426,7 +1440,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1426
1440
|
free(clearStateData = false) {
|
|
1427
1441
|
if (this.#status === "busy") {
|
|
1428
1442
|
this.#status = "free";
|
|
1429
|
-
this.#lastStatusUpdateTime = (0,
|
|
1443
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1430
1444
|
if (clearStateData) {
|
|
1431
1445
|
}
|
|
1432
1446
|
return true;
|
|
@@ -1495,12 +1509,12 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1495
1509
|
}, localStorage);
|
|
1496
1510
|
}
|
|
1497
1511
|
}
|
|
1498
|
-
await (0,
|
|
1512
|
+
await (0, import_utils4.sleep)(2e3);
|
|
1499
1513
|
await origPage.unrouteAll();
|
|
1500
1514
|
await page.free();
|
|
1501
1515
|
return true;
|
|
1502
1516
|
} catch (err) {
|
|
1503
|
-
|
|
1517
|
+
logerr(err);
|
|
1504
1518
|
return false;
|
|
1505
1519
|
}
|
|
1506
1520
|
}
|
|
@@ -1510,7 +1524,7 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1510
1524
|
use() {
|
|
1511
1525
|
if (this.#status === "free") {
|
|
1512
1526
|
this.#status = "busy";
|
|
1513
|
-
this.#lastStatusUpdateTime = (0,
|
|
1527
|
+
this.#lastStatusUpdateTime = (0, import_utils4.getCurrentUnixTime)();
|
|
1514
1528
|
return true;
|
|
1515
1529
|
} else {
|
|
1516
1530
|
return false;
|
|
@@ -1522,7 +1536,6 @@ var PlaywrightBrowserContext = class extends import_node_events2.default {
|
|
|
1522
1536
|
};
|
|
1523
1537
|
|
|
1524
1538
|
// src/playwright/browser.ts
|
|
1525
|
-
var import_utils4 = require("@letsscrapedata/utils");
|
|
1526
1539
|
var PlaywrightBrowser = class extends import_node_events3.default {
|
|
1527
1540
|
#browser;
|
|
1528
1541
|
#browserIdx;
|
|
@@ -1561,7 +1574,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1561
1574
|
this.#browser = browser;
|
|
1562
1575
|
this.#browserIdx = browserIdx;
|
|
1563
1576
|
this.#pid = pid;
|
|
1564
|
-
this.#createTime = (0,
|
|
1577
|
+
this.#createTime = (0, import_utils5.getCurrentUnixTime)();
|
|
1565
1578
|
this.#lsdBrowserContexts = [];
|
|
1566
1579
|
this.#browserControllerType = "playwright";
|
|
1567
1580
|
this.#browserType = browerType;
|
|
@@ -1572,38 +1585,38 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1572
1585
|
this.#executablePath = executablePath;
|
|
1573
1586
|
this.#nextBrowserContextIdx = 1;
|
|
1574
1587
|
this.#closeFreePagesIntervalId = null;
|
|
1575
|
-
|
|
1588
|
+
loginfo(`##browser ${this.id()} ${this.#browserCreationMethod}ed by ${this.#browserControllerType}`);
|
|
1576
1589
|
const browserContexts = browser.contexts();
|
|
1577
1590
|
if (browserContexts.length > 0) {
|
|
1578
|
-
|
|
1591
|
+
logwarn(`There are ${browserContexts.length} new browserContexts when playwright launches new browser`);
|
|
1579
1592
|
}
|
|
1580
1593
|
const incognito = typeof options?.incognito === "boolean" ? options.incognito : true;
|
|
1581
1594
|
for (const browserContext of browserContexts) {
|
|
1582
1595
|
const lsdBrowserContext = new PlaywrightBrowserContext(this, browserContext, "launch", incognito, this.#proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), maxViewportOfNewPage);
|
|
1583
1596
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
1584
|
-
|
|
1597
|
+
loginfo(`##browserContext ${lsdBrowserContext.id()} ${this.#browserCreationMethod}ed`);
|
|
1585
1598
|
}
|
|
1586
1599
|
browser.on("disconnected", () => {
|
|
1587
|
-
|
|
1600
|
+
loginfo(`##browser ${this.id()} disconnected`);
|
|
1588
1601
|
if (this.#lsdBrowserContexts.length > 0) {
|
|
1589
|
-
|
|
1602
|
+
logerr(`${this.id()} has browserContexts when disconnected`);
|
|
1590
1603
|
}
|
|
1591
1604
|
});
|
|
1592
1605
|
this.on("browserContextClose", (lsdBrowserContext) => {
|
|
1593
1606
|
if (!(lsdBrowserContext instanceof PlaywrightBrowserContext)) {
|
|
1594
|
-
|
|
1607
|
+
logerr(`Invalid data in LsdBrowser.on("browserContextClose)`);
|
|
1595
1608
|
return;
|
|
1596
1609
|
}
|
|
1597
1610
|
const idx = this.#lsdBrowserContexts.findIndex((bc) => bc === lsdBrowserContext);
|
|
1598
1611
|
if (idx < 0) {
|
|
1599
|
-
|
|
1612
|
+
logerr(`Invalid lsdBrowserContext in LsdBrowser.on("browserContextClose)`);
|
|
1600
1613
|
return;
|
|
1601
1614
|
}
|
|
1602
|
-
|
|
1615
|
+
loginfo(`##browserContext ${lsdBrowserContext.id()} closed
|
|
1603
1616
|
`);
|
|
1604
1617
|
this.#lsdBrowserContexts.splice(idx, 1);
|
|
1605
1618
|
if (this.#lsdBrowserContexts.length === 0) {
|
|
1606
|
-
|
|
1619
|
+
loginfo(`##browser ${this.id()} has no browserContexts now`);
|
|
1607
1620
|
}
|
|
1608
1621
|
return;
|
|
1609
1622
|
});
|
|
@@ -1620,7 +1633,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1620
1633
|
}
|
|
1621
1634
|
async newBrowserContext(options) {
|
|
1622
1635
|
if (this.#lsdBrowserContexts.length >= this.#maxBrowserContextsPerBrowser()) {
|
|
1623
|
-
|
|
1636
|
+
logwarn(`##browser ${this.id()} can not create more new browserContext`);
|
|
1624
1637
|
return null;
|
|
1625
1638
|
}
|
|
1626
1639
|
const browserContextOptions = {};
|
|
@@ -1639,7 +1652,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1639
1652
|
const { maxViewportOfNewPage = this.#options.maxViewportOfNewPage } = options ? options : {};
|
|
1640
1653
|
const lsdBrowserContext = new PlaywrightBrowserContext(this, browserContext, "new", true, proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), maxViewportOfNewPage);
|
|
1641
1654
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
1642
|
-
|
|
1655
|
+
loginfo(`##browser ${lsdBrowserContext.id()} created`);
|
|
1643
1656
|
return lsdBrowserContext;
|
|
1644
1657
|
}
|
|
1645
1658
|
async close() {
|
|
@@ -1691,7 +1704,7 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1691
1704
|
}
|
|
1692
1705
|
async pidUsage() {
|
|
1693
1706
|
if (this.#pid > 0) {
|
|
1694
|
-
const usage = await (0,
|
|
1707
|
+
const usage = await (0, import_utils5.getPerformanceOfPidTree)(this.#pid, "MB");
|
|
1695
1708
|
return usage;
|
|
1696
1709
|
} else {
|
|
1697
1710
|
return { cpu: 0, memory: 0 };
|
|
@@ -1711,15 +1724,18 @@ var PlaywrightBrowser = class extends import_node_events3.default {
|
|
|
1711
1724
|
|
|
1712
1725
|
// src/puppeteer/browser.ts
|
|
1713
1726
|
var import_node_events6 = __toESM(require("events"), 1);
|
|
1727
|
+
var import_utils9 = require("@letsscrapedata/utils");
|
|
1714
1728
|
|
|
1715
1729
|
// src/puppeteer/context.ts
|
|
1716
1730
|
var import_node_events5 = __toESM(require("events"), 1);
|
|
1731
|
+
var import_utils8 = require("@letsscrapedata/utils");
|
|
1717
1732
|
|
|
1718
1733
|
// src/puppeteer/page.ts
|
|
1719
1734
|
var import_node_events4 = __toESM(require("events"), 1);
|
|
1735
|
+
var import_utils7 = require("@letsscrapedata/utils");
|
|
1720
1736
|
|
|
1721
1737
|
// src/puppeteer/element.ts
|
|
1722
|
-
var
|
|
1738
|
+
var import_utils6 = require("@letsscrapedata/utils");
|
|
1723
1739
|
var PuppeteerElement = class _PuppeteerElement {
|
|
1724
1740
|
#frame;
|
|
1725
1741
|
#$ele;
|
|
@@ -1808,7 +1824,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1808
1824
|
}
|
|
1809
1825
|
return retObj;
|
|
1810
1826
|
} catch (err) {
|
|
1811
|
-
|
|
1827
|
+
logerr(err);
|
|
1812
1828
|
return retObj;
|
|
1813
1829
|
}
|
|
1814
1830
|
}
|
|
@@ -1884,7 +1900,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1884
1900
|
await this.#frame.evaluate(async (ev) => await ev.click(), this.#$ele);
|
|
1885
1901
|
await this.#$ele.click(actOptions);
|
|
1886
1902
|
} else {
|
|
1887
|
-
(0,
|
|
1903
|
+
(0, import_utils6.unreachable)(clickType);
|
|
1888
1904
|
}
|
|
1889
1905
|
return true;
|
|
1890
1906
|
}
|
|
@@ -1974,7 +1990,7 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1974
1990
|
}
|
|
1975
1991
|
break;
|
|
1976
1992
|
default:
|
|
1977
|
-
(0,
|
|
1993
|
+
(0, import_utils6.unreachable)(type);
|
|
1978
1994
|
}
|
|
1979
1995
|
return true;
|
|
1980
1996
|
}
|
|
@@ -1990,7 +2006,6 @@ var PuppeteerElement = class _PuppeteerElement {
|
|
|
1990
2006
|
};
|
|
1991
2007
|
|
|
1992
2008
|
// src/puppeteer/page.ts
|
|
1993
|
-
var import_utils6 = require("@letsscrapedata/utils");
|
|
1994
2009
|
var PuppeteerPage = class extends import_node_events4.default {
|
|
1995
2010
|
#lsdBrowserContext;
|
|
1996
2011
|
#page;
|
|
@@ -2150,9 +2165,9 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2150
2165
|
const page = this.#page;
|
|
2151
2166
|
const pageId = this.#pageId;
|
|
2152
2167
|
page.on("close", async () => {
|
|
2153
|
-
|
|
2168
|
+
loginfo(`##browser ${pageId} closed`);
|
|
2154
2169
|
if (!page.pageInfo) {
|
|
2155
|
-
|
|
2170
|
+
logerr(`Logic error in page.on("close")`);
|
|
2156
2171
|
}
|
|
2157
2172
|
this.emit("pageClose");
|
|
2158
2173
|
this.#lsdBrowserContext.emit("pageClose", this);
|
|
@@ -2171,12 +2186,12 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2171
2186
|
pageInfo.relatedId = page.pageInfo.taskId;
|
|
2172
2187
|
}
|
|
2173
2188
|
} else {
|
|
2174
|
-
|
|
2189
|
+
logerr(`##browser ${pageId} has popup without page.pageInfo`);
|
|
2175
2190
|
}
|
|
2176
|
-
|
|
2191
|
+
loginfo(`##browser ${pageId} has popup ${popupPageId}`);
|
|
2177
2192
|
this.emit("pagePopup", evtData);
|
|
2178
2193
|
} else {
|
|
2179
|
-
|
|
2194
|
+
logerr(`##browser ${pageId} has popup page with null page`);
|
|
2180
2195
|
}
|
|
2181
2196
|
});
|
|
2182
2197
|
}
|
|
@@ -2188,7 +2203,7 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2188
2203
|
this.#lsdBrowserContext = browserContext;
|
|
2189
2204
|
this.#page = page;
|
|
2190
2205
|
this.#status = "free";
|
|
2191
|
-
const currentTime = (0,
|
|
2206
|
+
const currentTime = (0, import_utils7.getCurrentUnixTime)();
|
|
2192
2207
|
const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
|
|
2193
2208
|
this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
|
|
2194
2209
|
this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
|
|
@@ -2582,7 +2597,7 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2582
2597
|
});
|
|
2583
2598
|
break;
|
|
2584
2599
|
default:
|
|
2585
|
-
(0,
|
|
2600
|
+
(0, import_utils7.unreachable)(action);
|
|
2586
2601
|
}
|
|
2587
2602
|
return true;
|
|
2588
2603
|
}
|
|
@@ -2786,7 +2801,6 @@ var PuppeteerPage = class extends import_node_events4.default {
|
|
|
2786
2801
|
};
|
|
2787
2802
|
|
|
2788
2803
|
// src/puppeteer/context.ts
|
|
2789
|
-
var import_utils7 = require("@letsscrapedata/utils");
|
|
2790
2804
|
var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
2791
2805
|
#lsdBrowser;
|
|
2792
2806
|
#browserIdx;
|
|
@@ -2815,7 +2829,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2815
2829
|
}
|
|
2816
2830
|
const pages = await this.#browserContext.pages();
|
|
2817
2831
|
const openType = this.#lsdBrowser.browserCreationMethod();
|
|
2818
|
-
const lastStatusUpdateTime = (0,
|
|
2832
|
+
const lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2819
2833
|
for (const page of pages) {
|
|
2820
2834
|
const pageInfo = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType, openTime: this.#createTime, lastStatusUpdateTime, taskId: 0, relatedId: 0, misc: {} };
|
|
2821
2835
|
const lsdPage = new PuppeteerPage(this, page, pageInfo);
|
|
@@ -2826,7 +2840,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2826
2840
|
await lsdPage.setUserAgent(this.#userAgent);
|
|
2827
2841
|
}
|
|
2828
2842
|
this.#lsdPages.push(lsdPage);
|
|
2829
|
-
|
|
2843
|
+
loginfo(`##browser ${lsdPage.id()} ${openType}ed`);
|
|
2830
2844
|
}
|
|
2831
2845
|
}
|
|
2832
2846
|
constructor(lsdBrowser, browserContext, browserContextCreationMethod, incognito = false, proxy = null, browserIdx = 0, browserContextIdx = 0, maxPagesPerBrowserContext = 20, maxPageFreeSeconds = 0, userAgent = "", maxViewportOfNewPage = true) {
|
|
@@ -2843,7 +2857,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2843
2857
|
this.#browserContext = browserContext;
|
|
2844
2858
|
this.#browserContextCreationMethod = browserContextCreationMethod;
|
|
2845
2859
|
this.#userAgent = userAgent;
|
|
2846
|
-
const currentTime = (0,
|
|
2860
|
+
const currentTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2847
2861
|
this.#createTime = currentTime;
|
|
2848
2862
|
this.#lastStatusUpdateTime = currentTime;
|
|
2849
2863
|
this.#status = "free";
|
|
@@ -2865,9 +2879,9 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2865
2879
|
const pageInfo = page.pageInfo;
|
|
2866
2880
|
if (pageInfo) {
|
|
2867
2881
|
const { browserIdx: browserIdx2, browserContextIdx: browserContextIdx2, pageIdx } = pageInfo;
|
|
2868
|
-
|
|
2882
|
+
logwarn(`##browser page-${browserIdx2}-${browserContextIdx2}-${pageIdx} has been already created`);
|
|
2869
2883
|
} else {
|
|
2870
|
-
const currentTime2 = (0,
|
|
2884
|
+
const currentTime2 = (0, import_utils8.getCurrentUnixTime)();
|
|
2871
2885
|
const pageInfo2 = { browserIdx: this.#browserIdx, browserContextIdx: this.#browserContextIdx, pageIdx: this.#nextPageIdx++, openType: "other", openTime: currentTime2, lastStatusUpdateTime: currentTime2, taskId: 0, relatedId: 0, misc: {} };
|
|
2872
2886
|
const lsdPage = new PuppeteerPage(this, page, pageInfo2);
|
|
2873
2887
|
if (this.#maxViewportOfNewPage) {
|
|
@@ -2877,18 +2891,18 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2877
2891
|
await lsdPage.setUserAgent(this.#userAgent);
|
|
2878
2892
|
}
|
|
2879
2893
|
this.#lsdPages.push(lsdPage);
|
|
2880
|
-
|
|
2894
|
+
loginfo(`##page ${lsdPage.id()} created`);
|
|
2881
2895
|
}
|
|
2882
2896
|
}
|
|
2883
2897
|
});
|
|
2884
2898
|
this.on("pageClose", (lsdPage) => {
|
|
2885
2899
|
if (!(lsdPage instanceof PuppeteerPage)) {
|
|
2886
|
-
|
|
2900
|
+
logerr(`Invalid data in LsdBrowserContext.on("pageClose)`);
|
|
2887
2901
|
return;
|
|
2888
2902
|
}
|
|
2889
2903
|
const idx = this.#lsdPages.findIndex((p) => p === lsdPage);
|
|
2890
2904
|
if (idx < 0) {
|
|
2891
|
-
|
|
2905
|
+
logerr(`Invalid lsdPage in LsdBrowserContext.on("pageClose)`);
|
|
2892
2906
|
return;
|
|
2893
2907
|
}
|
|
2894
2908
|
this.#lsdPages.splice(idx, 1);
|
|
@@ -2907,8 +2921,8 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2907
2921
|
async close() {
|
|
2908
2922
|
if (this.#browserContext) {
|
|
2909
2923
|
this.#status = "closed";
|
|
2910
|
-
this.#lastStatusUpdateTime = (0,
|
|
2911
|
-
|
|
2924
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
2925
|
+
loginfo(`browserContext ${this.id()} closed at ${this.#lastStatusUpdateTime}`);
|
|
2912
2926
|
if (this.#browserContextCreationMethod !== "launch") {
|
|
2913
2927
|
await this.#browserContext.close();
|
|
2914
2928
|
}
|
|
@@ -2922,15 +2936,15 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2922
2936
|
this.#gettingPage = true;
|
|
2923
2937
|
return true;
|
|
2924
2938
|
} else {
|
|
2925
|
-
await (0,
|
|
2939
|
+
await (0, import_utils8.sleep)(200);
|
|
2926
2940
|
}
|
|
2927
2941
|
}
|
|
2928
|
-
|
|
2942
|
+
logwarn(`Cannot get the gettingLock.`);
|
|
2929
2943
|
return false;
|
|
2930
2944
|
}
|
|
2931
2945
|
#freeGettingLock() {
|
|
2932
2946
|
if (!this.#gettingPage) {
|
|
2933
|
-
|
|
2947
|
+
logwarn(`Getting lock is already free now.`);
|
|
2934
2948
|
}
|
|
2935
2949
|
this.#gettingPage = false;
|
|
2936
2950
|
}
|
|
@@ -2939,7 +2953,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2939
2953
|
maxPageFreeSeconds = this.#maxPageFreeSeconds;
|
|
2940
2954
|
}
|
|
2941
2955
|
if (maxPageFreeSeconds <= 0) {
|
|
2942
|
-
|
|
2956
|
+
logwarn(`Please set valid maxPageFreeSeconds to close free pages`);
|
|
2943
2957
|
return false;
|
|
2944
2958
|
}
|
|
2945
2959
|
const gotLock = await this.#tryToGetGettingLock();
|
|
@@ -2947,7 +2961,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2947
2961
|
return false;
|
|
2948
2962
|
}
|
|
2949
2963
|
try {
|
|
2950
|
-
const maxUpdateTime = (0,
|
|
2964
|
+
const maxUpdateTime = (0, import_utils8.getCurrentUnixTime)() - this.#maxPageFreeSeconds;
|
|
2951
2965
|
let freePages = this.#lsdPages.filter((p) => p.isFree() && p.pageInfo().lastStatusUpdateTime < maxUpdateTime);
|
|
2952
2966
|
if (freePages.length === this.#lsdPages.length) {
|
|
2953
2967
|
freePages = freePages.slice(1);
|
|
@@ -2958,7 +2972,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2958
2972
|
this.#freeGettingLock();
|
|
2959
2973
|
return true;
|
|
2960
2974
|
} catch (err) {
|
|
2961
|
-
|
|
2975
|
+
logerr(err);
|
|
2962
2976
|
this.#freeGettingLock();
|
|
2963
2977
|
return false;
|
|
2964
2978
|
}
|
|
@@ -2983,7 +2997,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2983
2997
|
}
|
|
2984
2998
|
try {
|
|
2985
2999
|
if (this.#lsdPages.length === 0) {
|
|
2986
|
-
await (0,
|
|
3000
|
+
await (0, import_utils8.sleep)(1e3);
|
|
2987
3001
|
}
|
|
2988
3002
|
let lsdPage = this.#lsdPages.find((p) => p.isFree());
|
|
2989
3003
|
if (lsdPage) {
|
|
@@ -2995,7 +3009,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
2995
3009
|
return null;
|
|
2996
3010
|
}
|
|
2997
3011
|
const page = await this.#browserContext.newPage();
|
|
2998
|
-
await (0,
|
|
3012
|
+
await (0, import_utils8.sleep)(2e3);
|
|
2999
3013
|
const pageInfo = page.pageInfo;
|
|
3000
3014
|
if (!pageInfo) {
|
|
3001
3015
|
throw new Error(`Logic error in getPage`);
|
|
@@ -3012,7 +3026,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3012
3026
|
return null;
|
|
3013
3027
|
}
|
|
3014
3028
|
} catch (err) {
|
|
3015
|
-
|
|
3029
|
+
logerr(err);
|
|
3016
3030
|
this.#freeGettingLock();
|
|
3017
3031
|
return null;
|
|
3018
3032
|
}
|
|
@@ -3020,7 +3034,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3020
3034
|
free(clearStateData = false) {
|
|
3021
3035
|
if (this.#status === "busy") {
|
|
3022
3036
|
this.#status = "free";
|
|
3023
|
-
this.#lastStatusUpdateTime = (0,
|
|
3037
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
3024
3038
|
if (clearStateData) {
|
|
3025
3039
|
}
|
|
3026
3040
|
return true;
|
|
@@ -3089,12 +3103,12 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3089
3103
|
}, localStorage);
|
|
3090
3104
|
}
|
|
3091
3105
|
}
|
|
3092
|
-
await (0,
|
|
3106
|
+
await (0, import_utils8.sleep)(2e3);
|
|
3093
3107
|
await origPage.setRequestInterception(false);
|
|
3094
3108
|
await page.free();
|
|
3095
3109
|
return true;
|
|
3096
3110
|
} catch (err) {
|
|
3097
|
-
|
|
3111
|
+
logerr(err);
|
|
3098
3112
|
return false;
|
|
3099
3113
|
}
|
|
3100
3114
|
}
|
|
@@ -3104,7 +3118,7 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3104
3118
|
use() {
|
|
3105
3119
|
if (this.#status === "free") {
|
|
3106
3120
|
this.#status = "busy";
|
|
3107
|
-
this.#lastStatusUpdateTime = (0,
|
|
3121
|
+
this.#lastStatusUpdateTime = (0, import_utils8.getCurrentUnixTime)();
|
|
3108
3122
|
return true;
|
|
3109
3123
|
} else {
|
|
3110
3124
|
return false;
|
|
@@ -3116,7 +3130,6 @@ var PuppeteerBrowserContext = class extends import_node_events5.default {
|
|
|
3116
3130
|
};
|
|
3117
3131
|
|
|
3118
3132
|
// src/puppeteer/browser.ts
|
|
3119
|
-
var import_utils8 = require("@letsscrapedata/utils");
|
|
3120
3133
|
var PuppeteerBrowser = class extends import_node_events6.default {
|
|
3121
3134
|
#browser;
|
|
3122
3135
|
#browserIdx;
|
|
@@ -3158,7 +3171,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3158
3171
|
this.#browser = browser;
|
|
3159
3172
|
this.#browserIdx = browserIdx;
|
|
3160
3173
|
this.#pid = pid;
|
|
3161
|
-
this.#createTime = (0,
|
|
3174
|
+
this.#createTime = (0, import_utils9.getCurrentUnixTime)();
|
|
3162
3175
|
this.#lsdBrowserContexts = [];
|
|
3163
3176
|
this.#browserControllerType = "puppeteer";
|
|
3164
3177
|
this.#browserType = browerType;
|
|
@@ -3169,35 +3182,35 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3169
3182
|
this.#executablePath = executablePath;
|
|
3170
3183
|
this.#nextBrowserContextIdx = 1;
|
|
3171
3184
|
this.#closeFreePagesIntervalId = null;
|
|
3172
|
-
|
|
3185
|
+
loginfo(`##browser ${this.#browserType} ${this.id()} ${this.#browserCreationMethod}ed by ${this.#browserControllerType}`);
|
|
3173
3186
|
const browserContexts = browser.browserContexts();
|
|
3174
3187
|
const incognito = typeof options?.incognito === "boolean" ? options.incognito : false;
|
|
3175
3188
|
for (const browserContext of browserContexts) {
|
|
3176
3189
|
const lsdBrowserContext = new PuppeteerBrowserContext(this, browserContext, "launch", incognito, this.#proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), this.#userAgent(), maxViewportOfNewPage);
|
|
3177
3190
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
3178
|
-
|
|
3191
|
+
loginfo(`##browser ${lsdBrowserContext.id()} ${this.#browserCreationMethod}ed`);
|
|
3179
3192
|
}
|
|
3180
3193
|
browser.on("disconnected", () => {
|
|
3181
|
-
|
|
3194
|
+
loginfo(`##browser ${this.id()} disconnected`);
|
|
3182
3195
|
if (this.#lsdBrowserContexts.length > 0) {
|
|
3183
|
-
|
|
3196
|
+
logerr(`${this.id()} has browserContexts when disconnected`);
|
|
3184
3197
|
}
|
|
3185
3198
|
});
|
|
3186
3199
|
this.on("browserContextClose", (lsdBrowserContext) => {
|
|
3187
3200
|
if (!(lsdBrowserContext instanceof PuppeteerBrowserContext)) {
|
|
3188
|
-
|
|
3201
|
+
logerr(`Invalid data in LsdBrowser.on("browserContextClose)`);
|
|
3189
3202
|
return;
|
|
3190
3203
|
}
|
|
3191
3204
|
const idx = this.#lsdBrowserContexts.findIndex((bc) => bc === lsdBrowserContext);
|
|
3192
3205
|
if (idx < 0) {
|
|
3193
|
-
|
|
3206
|
+
logerr(`Invalid lsdBrowserContext in LsdBrowser.on("browserContextClose)`);
|
|
3194
3207
|
return;
|
|
3195
3208
|
}
|
|
3196
|
-
|
|
3209
|
+
loginfo(`##browser ${lsdBrowserContext.id()} closed
|
|
3197
3210
|
`);
|
|
3198
3211
|
this.#lsdBrowserContexts.splice(idx, 1);
|
|
3199
3212
|
if (this.#lsdBrowserContexts.length === 0) {
|
|
3200
|
-
|
|
3213
|
+
loginfo(`##browser ${this.id()} has no browserContexts now`);
|
|
3201
3214
|
}
|
|
3202
3215
|
return;
|
|
3203
3216
|
});
|
|
@@ -3214,7 +3227,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3214
3227
|
}
|
|
3215
3228
|
async newBrowserContext(options) {
|
|
3216
3229
|
if (this.#lsdBrowserContexts.length >= this.#maxBrowserContextsPerBrowser()) {
|
|
3217
|
-
|
|
3230
|
+
logwarn(`##browser ${this.id()} can not create more new browserContext`);
|
|
3218
3231
|
return null;
|
|
3219
3232
|
}
|
|
3220
3233
|
const browserContextOptions = {};
|
|
@@ -3227,7 +3240,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3227
3240
|
const { maxViewportOfNewPage = this.#options.maxViewportOfNewPage } = options ? options : {};
|
|
3228
3241
|
const lsdBrowserContext = new PuppeteerBrowserContext(this, browserContext, "new", true, proxy, this.#browserIdx, this.#nextBrowserContextIdx++, this.#maxPagesPerBrowserContext(), this.#maxPageFreeSeconds(), userAgent, maxViewportOfNewPage);
|
|
3229
3242
|
this.#lsdBrowserContexts.push(lsdBrowserContext);
|
|
3230
|
-
|
|
3243
|
+
loginfo(`##browser ${lsdBrowserContext.id()} created`);
|
|
3231
3244
|
return lsdBrowserContext;
|
|
3232
3245
|
}
|
|
3233
3246
|
async close() {
|
|
@@ -3279,7 +3292,7 @@ var PuppeteerBrowser = class extends import_node_events6.default {
|
|
|
3279
3292
|
}
|
|
3280
3293
|
async pidUsage() {
|
|
3281
3294
|
if (this.#pid > 0) {
|
|
3282
|
-
const usage = await (0,
|
|
3295
|
+
const usage = await (0, import_utils9.getPerformanceOfPidTree)(this.#pid, "MB");
|
|
3283
3296
|
return usage;
|
|
3284
3297
|
} else {
|
|
3285
3298
|
return { cpu: 0, memory: 0 };
|
|
@@ -3626,10 +3639,10 @@ var CheerioPage = class extends import_node_events7.default {
|
|
|
3626
3639
|
var import_os = __toESM(require("os"), 1);
|
|
3627
3640
|
var import_puppeteer = __toESM(require("puppeteer"), 1);
|
|
3628
3641
|
var import_playwright = __toESM(require("playwright"), 1);
|
|
3629
|
-
var import_utils9 = require("@letsscrapedata/utils");
|
|
3630
3642
|
var import_puppeteer_extra = __toESM(require("puppeteer-extra"), 1);
|
|
3631
3643
|
var playwrightExtra = __toESM(require("playwright-extra"), 1);
|
|
3632
3644
|
var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
|
|
3645
|
+
var import_utils10 = require("@letsscrapedata/utils");
|
|
3633
3646
|
var LsdBrowserController = class _LsdBrowserController {
|
|
3634
3647
|
static #forbidConstructor = false;
|
|
3635
3648
|
#puppeteer;
|
|
@@ -3702,7 +3715,7 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3702
3715
|
}
|
|
3703
3716
|
break;
|
|
3704
3717
|
default:
|
|
3705
|
-
(0,
|
|
3718
|
+
(0, import_utils10.unreachable)(browserType);
|
|
3706
3719
|
}
|
|
3707
3720
|
return true;
|
|
3708
3721
|
}
|
|
@@ -3730,22 +3743,22 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3730
3743
|
const actOptions = { closeFreePagesIntervalSeconds, maxBrowserContextsPerBrowser, maxPagesPerBrowserContext, maxPageFreeSeconds, maxViewportOfNewPage, proxy, timeout, args, executablePath, maxWindowSize, headless, minBrowserContexts, incognito, proxyPerBrowserContext, userDataDir, userAgent };
|
|
3731
3744
|
let idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--incoginto"));
|
|
3732
3745
|
if (idx >= 0) {
|
|
3733
|
-
|
|
3746
|
+
logwarn(`Please use options.incognito instead when launching new browser.`);
|
|
3734
3747
|
args.splice(idx, 1);
|
|
3735
3748
|
}
|
|
3736
3749
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--proxy-server"));
|
|
3737
3750
|
if (idx >= 0) {
|
|
3738
|
-
|
|
3751
|
+
logwarn(`Please use options.proxy instead when launching new browser.`);
|
|
3739
3752
|
args.splice(idx, 1);
|
|
3740
3753
|
}
|
|
3741
3754
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--user-data-dir"));
|
|
3742
3755
|
if (idx >= 0) {
|
|
3743
|
-
|
|
3756
|
+
logwarn(`Please use options.userDataDir instead when launching new browser.`);
|
|
3744
3757
|
args.splice(idx, 1);
|
|
3745
3758
|
}
|
|
3746
3759
|
idx = args.findIndex((arg) => arg.toLowerCase().startsWith("--start-maximized"));
|
|
3747
3760
|
if (idx >= 0) {
|
|
3748
|
-
|
|
3761
|
+
logwarn(`Please use options.maxWindowSize instead when launching new browser.`);
|
|
3749
3762
|
args.splice(idx, 1);
|
|
3750
3763
|
}
|
|
3751
3764
|
let lsdBrowser;
|
|
@@ -3855,10 +3868,10 @@ var LsdBrowserController = class _LsdBrowserController {
|
|
|
3855
3868
|
}
|
|
3856
3869
|
const u = new URL(browserUrl);
|
|
3857
3870
|
const port = u.port ? parseInt(u.port) : 80;
|
|
3858
|
-
const pids = await (0,
|
|
3871
|
+
const pids = await (0, import_utils10.getPidsListeningOnPort)(port);
|
|
3859
3872
|
let browserPid = 0;
|
|
3860
3873
|
if (pids.length !== 1) {
|
|
3861
|
-
|
|
3874
|
+
logerr(`##browser pids.length ${pids.length} is not 1 when trying to connect to browserUrl ${browserUrl}`);
|
|
3862
3875
|
} else {
|
|
3863
3876
|
browserPid = pids[0];
|
|
3864
3877
|
}
|