@skrillex1224/playwright-toolkit 2.1.22 → 2.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -17
- package/dist/index.cjs +246 -162
- package/dist/index.cjs.map +3 -3
- package/dist/index.js +246 -162
- package/dist/index.js.map +3 -3
- package/index.d.ts +28 -10
- package/package.json +2 -4
package/dist/index.js
CHANGED
|
@@ -323,133 +323,243 @@ var Utils = {
|
|
|
323
323
|
};
|
|
324
324
|
|
|
325
325
|
// src/stealth.js
|
|
326
|
-
var logger3 = createLogger("
|
|
327
|
-
var
|
|
326
|
+
var logger3 = createLogger("AntiDetect");
|
|
327
|
+
var BASE_CONFIG = Object.freeze({
|
|
328
|
+
locale: "zh-CN",
|
|
329
|
+
acceptLanguage: "zh-CN,zh;q=0.9",
|
|
330
|
+
timezoneId: "Asia/Shanghai",
|
|
331
|
+
timezoneOffset: -480,
|
|
332
|
+
geolocation: null
|
|
333
|
+
});
|
|
334
|
+
var DEFAULT_LAUNCH_ARGS = [
|
|
335
|
+
"--disable-blink-features=AutomationControlled",
|
|
336
|
+
"--no-sandbox",
|
|
337
|
+
"--disable-setuid-sandbox",
|
|
338
|
+
"--window-position=0,0",
|
|
339
|
+
`--lang=${BASE_CONFIG.locale}`
|
|
340
|
+
];
|
|
341
|
+
var ADVANCED_LAUNCH_ARGS = [
|
|
342
|
+
...DEFAULT_LAUNCH_ARGS,
|
|
343
|
+
"--disable-dev-shm-usage",
|
|
344
|
+
"--disable-background-networking",
|
|
345
|
+
"--disable-default-apps",
|
|
346
|
+
"--disable-extensions",
|
|
347
|
+
"--disable-sync",
|
|
348
|
+
"--disable-translate",
|
|
349
|
+
"--metrics-recording-only",
|
|
350
|
+
"--mute-audio",
|
|
351
|
+
"--no-first-run"
|
|
352
|
+
];
|
|
353
|
+
var CONTEXT_CONFIG_CACHE = /* @__PURE__ */ new WeakMap();
|
|
354
|
+
function buildFingerprintOptions(locale) {
|
|
355
|
+
return {
|
|
356
|
+
browsers: [{ name: "chrome", minVersion: 110 }],
|
|
357
|
+
devices: ["desktop"],
|
|
358
|
+
operatingSystems: ["windows", "macos", "linux"],
|
|
359
|
+
locales: [locale]
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
function parseAcceptLanguage(acceptLanguage) {
|
|
363
|
+
if (!acceptLanguage) return [];
|
|
364
|
+
return acceptLanguage.split(",").map((part) => part.trim().split(";")[0]).filter(Boolean);
|
|
365
|
+
}
|
|
366
|
+
function normalizeLanguages(acceptLanguage, fallbackLocale) {
|
|
367
|
+
const languages = parseAcceptLanguage(acceptLanguage);
|
|
368
|
+
if (languages.length === 0) return [fallbackLocale];
|
|
369
|
+
if (!languages.includes(fallbackLocale)) {
|
|
370
|
+
return [fallbackLocale, ...languages];
|
|
371
|
+
}
|
|
372
|
+
return languages;
|
|
373
|
+
}
|
|
374
|
+
function getOperatingSystemsFromUserAgent(userAgent) {
|
|
375
|
+
const lowerUA = userAgent.toLowerCase();
|
|
376
|
+
if (lowerUA.includes("windows")) return ["windows"];
|
|
377
|
+
if (lowerUA.includes("mac os") || lowerUA.includes("macintosh")) return ["macos"];
|
|
378
|
+
if (lowerUA.includes("linux")) return ["linux"];
|
|
379
|
+
return [];
|
|
380
|
+
}
|
|
381
|
+
function buildContextConfigKey(config) {
|
|
382
|
+
return JSON.stringify({
|
|
383
|
+
locale: config.locale,
|
|
384
|
+
acceptLanguage: config.acceptLanguage,
|
|
385
|
+
timezoneId: config.timezoneId,
|
|
386
|
+
timezoneOffset: config.timezoneOffset
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
async function applyContextSettings(context, config, languages, permissions, injectLocaleTimezone) {
|
|
390
|
+
const contextKey = buildContextConfigKey(config);
|
|
391
|
+
const cached = CONTEXT_CONFIG_CACHE.get(context);
|
|
392
|
+
const isFirstInit = !cached;
|
|
393
|
+
const effectiveConfig = cached?.config || config;
|
|
394
|
+
const effectiveLanguages = cached?.languages || languages;
|
|
395
|
+
if (isFirstInit) {
|
|
396
|
+
CONTEXT_CONFIG_CACHE.set(context, {
|
|
397
|
+
key: contextKey,
|
|
398
|
+
config,
|
|
399
|
+
languages
|
|
400
|
+
});
|
|
401
|
+
} else if (cached.key !== contextKey) {
|
|
402
|
+
logger3.warn("applyContext", "Context already initialized; ignore conflicting locale/timezone.");
|
|
403
|
+
}
|
|
404
|
+
await context.setExtraHTTPHeaders({
|
|
405
|
+
"accept-language": effectiveConfig.acceptLanguage
|
|
406
|
+
});
|
|
407
|
+
if (isFirstInit) {
|
|
408
|
+
await context.addInitScript(({ locale, timezoneId, timezoneOffset, languages: languages2, applyLocaleTimezone }) => {
|
|
409
|
+
const originalDateTimeFormat = Intl.DateTimeFormat;
|
|
410
|
+
if (applyLocaleTimezone) {
|
|
411
|
+
Intl.DateTimeFormat = function(locales, initOptions) {
|
|
412
|
+
const nextLocales = locales || locale;
|
|
413
|
+
const nextOptions = initOptions ? { ...initOptions } : {};
|
|
414
|
+
nextOptions.timeZone = nextOptions.timeZone || timezoneId;
|
|
415
|
+
return new originalDateTimeFormat(nextLocales, nextOptions);
|
|
416
|
+
};
|
|
417
|
+
Intl.DateTimeFormat.prototype = originalDateTimeFormat.prototype;
|
|
418
|
+
Date.prototype.getTimezoneOffset = function() {
|
|
419
|
+
return timezoneOffset;
|
|
420
|
+
};
|
|
421
|
+
Object.defineProperty(navigator, "language", { get: () => languages2[0] });
|
|
422
|
+
Object.defineProperty(navigator, "languages", { get: () => languages2 });
|
|
423
|
+
}
|
|
424
|
+
Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
|
|
425
|
+
}, {
|
|
426
|
+
locale: effectiveConfig.locale,
|
|
427
|
+
timezoneId: effectiveConfig.timezoneId,
|
|
428
|
+
timezoneOffset: effectiveConfig.timezoneOffset,
|
|
429
|
+
languages: effectiveLanguages,
|
|
430
|
+
applyLocaleTimezone: injectLocaleTimezone
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
if (effectiveConfig.geolocation) {
|
|
434
|
+
await context.setGeolocation(effectiveConfig.geolocation);
|
|
435
|
+
await context.grantPermissions(["geolocation"]);
|
|
436
|
+
}
|
|
437
|
+
if (permissions?.length) {
|
|
438
|
+
await context.grantPermissions(permissions);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
function resolveConfig(overrides = {}) {
|
|
442
|
+
return {
|
|
443
|
+
...BASE_CONFIG,
|
|
444
|
+
...overrides,
|
|
445
|
+
geolocation: overrides.geolocation === null ? null : overrides.geolocation || BASE_CONFIG.geolocation
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
var AntiDetect = {
|
|
449
|
+
/**
|
|
450
|
+
* 获取统一的基础配置(中国、桌面端、中文语言)。
|
|
451
|
+
*/
|
|
452
|
+
getBaseConfig() {
|
|
453
|
+
return { ...BASE_CONFIG };
|
|
454
|
+
},
|
|
455
|
+
/**
|
|
456
|
+
* 用于 Crawlee fingerprint generator 的统一配置(桌面端)。
|
|
457
|
+
*/
|
|
458
|
+
getFingerprintGeneratorOptions() {
|
|
459
|
+
return buildFingerprintOptions(BASE_CONFIG.locale);
|
|
460
|
+
},
|
|
461
|
+
/**
|
|
462
|
+
* 获取基础启动参数。
|
|
463
|
+
*/
|
|
464
|
+
getLaunchArgs() {
|
|
465
|
+
return [...DEFAULT_LAUNCH_ARGS];
|
|
466
|
+
},
|
|
467
|
+
/**
|
|
468
|
+
* 获取增强启动参数(高风险场景)。
|
|
469
|
+
*/
|
|
470
|
+
getAdvancedLaunchArgs() {
|
|
471
|
+
return [...ADVANCED_LAUNCH_ARGS];
|
|
472
|
+
},
|
|
473
|
+
/**
|
|
474
|
+
* 统一应用到 BrowserContext(时区/语言/权限/地理位置)。
|
|
475
|
+
*
|
|
476
|
+
* @param {import('playwright').BrowserContext} context
|
|
477
|
+
* @param {Object} [options]
|
|
478
|
+
* @param {string} [options.locale]
|
|
479
|
+
* @param {string} [options.acceptLanguage]
|
|
480
|
+
* @param {string} [options.timezoneId]
|
|
481
|
+
* @param {number} [options.timezoneOffset]
|
|
482
|
+
* @param {import('playwright').Geolocation|null} [options.geolocation]
|
|
483
|
+
* @param {string[]} [options.permissions]
|
|
484
|
+
*/
|
|
485
|
+
async applyContext(context, options = {}) {
|
|
486
|
+
const config = resolveConfig(options);
|
|
487
|
+
const languages = normalizeLanguages(config.acceptLanguage, config.locale);
|
|
488
|
+
const permissions = Array.isArray(options.permissions) ? options.permissions : [];
|
|
489
|
+
await applyContextSettings(context, config, languages, permissions, true);
|
|
490
|
+
logger3.success("applyContext", `${config.locale} | ${config.timezoneId}`);
|
|
491
|
+
},
|
|
492
|
+
/**
|
|
493
|
+
* 统一应用到 Page(Context + 视口同步)。
|
|
494
|
+
*
|
|
495
|
+
* @param {import('playwright').Page} page
|
|
496
|
+
* @param {Object} [options] - 传递给 applyContext 的选项
|
|
497
|
+
*/
|
|
498
|
+
async applyPage(page, options = {}) {
|
|
499
|
+
const config = resolveConfig(options);
|
|
500
|
+
const languages = normalizeLanguages(config.acceptLanguage, config.locale);
|
|
501
|
+
const permissions = Array.isArray(options.permissions) ? options.permissions : [];
|
|
502
|
+
let injectLocaleTimezone = true;
|
|
503
|
+
try {
|
|
504
|
+
const env = await page.evaluate(() => ({
|
|
505
|
+
language: navigator.language,
|
|
506
|
+
languages: Array.isArray(navigator.languages) ? navigator.languages : [],
|
|
507
|
+
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
508
|
+
tzOffset: (/* @__PURE__ */ new Date()).getTimezoneOffset()
|
|
509
|
+
}));
|
|
510
|
+
const languageMatch = env.language === languages[0];
|
|
511
|
+
const timeZoneMatch = env.timeZone === config.timezoneId && env.tzOffset === config.timezoneOffset;
|
|
512
|
+
injectLocaleTimezone = !(languageMatch && timeZoneMatch);
|
|
513
|
+
} catch (e) {
|
|
514
|
+
injectLocaleTimezone = true;
|
|
515
|
+
}
|
|
516
|
+
await applyContextSettings(page.context(), config, languages, permissions, injectLocaleTimezone);
|
|
517
|
+
await this.syncViewportWithScreen(page);
|
|
518
|
+
},
|
|
328
519
|
/**
|
|
329
|
-
*
|
|
330
|
-
* 防止 "Viewport Mismatch" 类型的反爬检测。
|
|
331
|
-
* @param {import('playwright').Page} page
|
|
520
|
+
* 同步 Page 视口到 window.screen,避免视口/屏幕不一致检测。
|
|
332
521
|
*/
|
|
333
522
|
async syncViewportWithScreen(page) {
|
|
334
523
|
try {
|
|
335
524
|
const screen = await page.evaluate(() => ({
|
|
336
525
|
width: window.screen.width,
|
|
337
|
-
height: window.screen.height
|
|
338
|
-
availWidth: window.screen.availWidth,
|
|
339
|
-
availHeight: window.screen.availHeight
|
|
526
|
+
height: window.screen.height
|
|
340
527
|
}));
|
|
341
528
|
await page.setViewportSize({
|
|
342
529
|
width: screen.width,
|
|
343
530
|
height: screen.height
|
|
344
531
|
});
|
|
345
|
-
logger3.success("
|
|
532
|
+
logger3.success("syncViewport", `size=${screen.width}x${screen.height}`);
|
|
346
533
|
} catch (e) {
|
|
347
|
-
logger3.warn(`
|
|
534
|
+
logger3.warn(`syncViewport \u5931\u8D25: ${e.message}\uFF0C\u56DE\u9000\u5230 1920x1080`);
|
|
348
535
|
await page.setViewportSize({ width: 1920, height: 1080 });
|
|
349
536
|
}
|
|
350
537
|
},
|
|
351
538
|
/**
|
|
352
|
-
*
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
await page.addInitScript(() => {
|
|
356
|
-
Object.defineProperty(navigator, "webdriver", {
|
|
357
|
-
get: () => false
|
|
358
|
-
});
|
|
359
|
-
});
|
|
360
|
-
logger3.success("hideWebdriver");
|
|
361
|
-
},
|
|
362
|
-
/**
|
|
363
|
-
* 获取推荐的 Stealth 启动参数
|
|
364
|
-
*/
|
|
365
|
-
getStealthLaunchArgs() {
|
|
366
|
-
return [
|
|
367
|
-
"--disable-blink-features=AutomationControlled",
|
|
368
|
-
"--no-sandbox",
|
|
369
|
-
"--disable-setuid-sandbox",
|
|
370
|
-
"--disable-infobars",
|
|
371
|
-
"--window-position=0,0",
|
|
372
|
-
"--ignore-certificate-errors",
|
|
373
|
-
"--disable-web-security"
|
|
374
|
-
// 注意:不建议这里强制指定 window-size,让 syncViewportWithScreen 去动态调整
|
|
375
|
-
// '--window-size=1920,1080'
|
|
376
|
-
];
|
|
377
|
-
},
|
|
378
|
-
/**
|
|
379
|
-
* 获取增强版 Stealth 启动参数 (推荐用于高风险反爬场景)
|
|
380
|
-
* 包含更多针对自动化检测的防护
|
|
381
|
-
*/
|
|
382
|
-
getAdvancedStealthArgs() {
|
|
383
|
-
return [
|
|
384
|
-
...this.getStealthLaunchArgs(),
|
|
385
|
-
// 禁用各种可能暴露自动化的特征
|
|
386
|
-
"--disable-dev-shm-usage",
|
|
387
|
-
"--disable-accelerated-2d-canvas",
|
|
388
|
-
"--disable-gpu-sandbox",
|
|
389
|
-
"--disable-background-networking",
|
|
390
|
-
"--disable-default-apps",
|
|
391
|
-
"--disable-extensions",
|
|
392
|
-
"--disable-sync",
|
|
393
|
-
"--disable-translate",
|
|
394
|
-
"--metrics-recording-only",
|
|
395
|
-
"--mute-audio",
|
|
396
|
-
"--no-first-run",
|
|
397
|
-
// 模拟真实用户配置
|
|
398
|
-
"--lang=zh-CN,zh"
|
|
399
|
-
];
|
|
400
|
-
},
|
|
401
|
-
/**
|
|
402
|
-
* 设置中国时区 (Asia/Shanghai, UTC+8)
|
|
403
|
-
*
|
|
404
|
-
* 防止时区不一致的检测。对于中国境内爬取强烈推荐使用。
|
|
405
|
-
* 应在 preNavigationHooks 或 BrowserContext 创建后调用。
|
|
406
|
-
*
|
|
407
|
-
* @param {import('playwright').BrowserContext} context
|
|
539
|
+
* 为 got-scraping 生成与浏览器一致的 TLS 指纹配置(桌面端)。
|
|
540
|
+
*
|
|
541
|
+
* @param {string} [userAgent]
|
|
408
542
|
*/
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
return new originalDateTimeFormat(locales, options);
|
|
416
|
-
};
|
|
417
|
-
Intl.DateTimeFormat.prototype = originalDateTimeFormat.prototype;
|
|
418
|
-
Date.prototype.getTimezoneOffset = function() {
|
|
419
|
-
return -480;
|
|
420
|
-
};
|
|
421
|
-
});
|
|
422
|
-
logger3.success("setChinaTimezone", "Asia/Shanghai (UTC+8)");
|
|
543
|
+
getTlsFingerprintOptions(userAgent = "", acceptLanguage = "") {
|
|
544
|
+
const primaryLocale = parseAcceptLanguage(acceptLanguage || BASE_CONFIG.acceptLanguage)[0] || BASE_CONFIG.locale;
|
|
545
|
+
const fingerprint = buildFingerprintOptions(primaryLocale);
|
|
546
|
+
const os = getOperatingSystemsFromUserAgent(userAgent);
|
|
547
|
+
if (os.length > 0) fingerprint.operatingSystems = os;
|
|
548
|
+
return fingerprint;
|
|
423
549
|
},
|
|
424
550
|
/**
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
* @
|
|
551
|
+
* 规范化请求头,确保语言与浏览器一致。
|
|
552
|
+
*
|
|
553
|
+
* @param {Record<string, string>} headers
|
|
554
|
+
* @returns {Record<string, string>}
|
|
429
555
|
*/
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
if (
|
|
434
|
-
|
|
435
|
-
else if (lowerUA.includes("linux")) os.push("linux");
|
|
436
|
-
else if (lowerUA.includes("android")) os.push("android");
|
|
437
|
-
else if (lowerUA.includes("iphone") || lowerUA.includes("ipad")) os.push("ios");
|
|
438
|
-
if (os.length === 0) os.push("windows", "macos");
|
|
439
|
-
const devices = [];
|
|
440
|
-
if (lowerUA.includes("mobile") || lowerUA.includes("android") || lowerUA.includes("iphone")) {
|
|
441
|
-
devices.push("mobile");
|
|
442
|
-
} else {
|
|
443
|
-
devices.push("desktop");
|
|
556
|
+
applyLocaleHeaders(headers, acceptLanguage = "") {
|
|
557
|
+
if (acceptLanguage) {
|
|
558
|
+
headers["accept-language"] = acceptLanguage;
|
|
559
|
+
} else if (!headers["accept-language"]) {
|
|
560
|
+
headers["accept-language"] = BASE_CONFIG.acceptLanguage;
|
|
444
561
|
}
|
|
445
|
-
return
|
|
446
|
-
browsers: [{ name: "chrome", minVersion: 110 }],
|
|
447
|
-
// 保持较新的 Chrome 版本
|
|
448
|
-
devices,
|
|
449
|
-
locales: ["zh-CN", "en-US"],
|
|
450
|
-
// 保持与 setChinaTimezone 一致的中文优先
|
|
451
|
-
operatingSystems: os
|
|
452
|
-
};
|
|
562
|
+
return headers;
|
|
453
563
|
}
|
|
454
564
|
};
|
|
455
565
|
|
|
@@ -928,12 +1038,11 @@ var Humanize = {
|
|
|
928
1038
|
};
|
|
929
1039
|
|
|
930
1040
|
// src/launch.js
|
|
931
|
-
var logger5 = createLogger("Launch");
|
|
932
1041
|
var Launch = {
|
|
933
1042
|
getLaunchOptions(customArgs = []) {
|
|
934
1043
|
return {
|
|
935
1044
|
args: [
|
|
936
|
-
...
|
|
1045
|
+
...AntiDetect.getLaunchArgs(),
|
|
937
1046
|
...customArgs
|
|
938
1047
|
],
|
|
939
1048
|
ignoreDefaultArgs: ["--enable-automation"]
|
|
@@ -945,7 +1054,7 @@ var Launch = {
|
|
|
945
1054
|
getAdvancedLaunchOptions(customArgs = []) {
|
|
946
1055
|
return {
|
|
947
1056
|
args: [
|
|
948
|
-
...
|
|
1057
|
+
...AntiDetect.getAdvancedLaunchArgs(),
|
|
949
1058
|
...customArgs
|
|
950
1059
|
],
|
|
951
1060
|
ignoreDefaultArgs: ["--enable-automation"]
|
|
@@ -956,42 +1065,14 @@ var Launch = {
|
|
|
956
1065
|
* 确保生成的是桌面端、较新的 Chrome,以匹配我们的脚本逻辑
|
|
957
1066
|
*/
|
|
958
1067
|
getFingerprintGeneratorOptions() {
|
|
959
|
-
return
|
|
960
|
-
browsers: [{ name: "chrome", minVersion: 110 }],
|
|
961
|
-
devices: ["desktop"],
|
|
962
|
-
operatingSystems: ["windows", "linux"]
|
|
963
|
-
// 包含 Linux 兼容容器
|
|
964
|
-
};
|
|
965
|
-
},
|
|
966
|
-
/**
|
|
967
|
-
* 创建已注册 Stealth 插件的 Chromium 实例
|
|
968
|
-
*
|
|
969
|
-
* 封装了 `chromium.use(stealthPlugin())` 的常用模式
|
|
970
|
-
*
|
|
971
|
-
* @example
|
|
972
|
-
* ```javascript
|
|
973
|
-
* import { chromium } from 'playwright-extra';
|
|
974
|
-
* import stealthPlugin from 'puppeteer-extra-plugin-stealth';
|
|
975
|
-
*
|
|
976
|
-
* const stealthChromium = Launch.createStealthChromium(chromium, stealthPlugin);
|
|
977
|
-
* // 现在 stealthChromium 已注册 stealth 插件,可用于 launchContext.launcher
|
|
978
|
-
* ```
|
|
979
|
-
*
|
|
980
|
-
* @param {import('playwright-extra').ChromiumExtra} chromium - playwright-extra 的 chromium
|
|
981
|
-
* @param {Function} stealthPlugin - puppeteer-extra-plugin-stealth 的默认导出
|
|
982
|
-
* @returns {import('playwright-extra').ChromiumExtra} 已注册 stealth 的 chromium
|
|
983
|
-
*/
|
|
984
|
-
createStealthChromium(chromium, stealthPlugin) {
|
|
985
|
-
chromium.use(stealthPlugin());
|
|
986
|
-
logger5.success("createStealthChromium", "Stealth plugin registered");
|
|
987
|
-
return chromium;
|
|
1068
|
+
return AntiDetect.getFingerprintGeneratorOptions();
|
|
988
1069
|
}
|
|
989
1070
|
};
|
|
990
1071
|
|
|
991
1072
|
// src/live-view.js
|
|
992
1073
|
import express from "express";
|
|
993
1074
|
import { Actor } from "apify";
|
|
994
|
-
var
|
|
1075
|
+
var logger5 = createLogger("LiveView");
|
|
995
1076
|
async function startLiveViewServer(liveViewKey) {
|
|
996
1077
|
const app = express();
|
|
997
1078
|
app.get("/", async (req, res) => {
|
|
@@ -1016,13 +1097,13 @@ async function startLiveViewServer(liveViewKey) {
|
|
|
1016
1097
|
</html>
|
|
1017
1098
|
`);
|
|
1018
1099
|
} catch (error) {
|
|
1019
|
-
|
|
1100
|
+
logger5.fail("Live View Server", error);
|
|
1020
1101
|
res.status(500).send(`\u65E0\u6CD5\u52A0\u8F7D\u5C4F\u5E55\u622A\u56FE: ${error.message}`);
|
|
1021
1102
|
}
|
|
1022
1103
|
});
|
|
1023
1104
|
const port = process.env.APIFY_CONTAINER_PORT || 4321;
|
|
1024
1105
|
app.listen(port, () => {
|
|
1025
|
-
|
|
1106
|
+
logger5.success("startLiveViewServer", `\u76D1\u542C\u7AEF\u53E3 ${port}`);
|
|
1026
1107
|
});
|
|
1027
1108
|
}
|
|
1028
1109
|
async function takeLiveScreenshot(liveViewKey, page, logMessage) {
|
|
@@ -1030,10 +1111,10 @@ async function takeLiveScreenshot(liveViewKey, page, logMessage) {
|
|
|
1030
1111
|
const buffer = await page.screenshot({ type: "png" });
|
|
1031
1112
|
await Actor.setValue(liveViewKey, buffer, { contentType: "image/png" });
|
|
1032
1113
|
if (logMessage) {
|
|
1033
|
-
|
|
1114
|
+
logger5.info(`(\u622A\u56FE): ${logMessage}`);
|
|
1034
1115
|
}
|
|
1035
1116
|
} catch (e) {
|
|
1036
|
-
|
|
1117
|
+
logger5.warn(`\u65E0\u6CD5\u6355\u83B7 Live View \u5C4F\u5E55\u622A\u56FE: ${e.message}`);
|
|
1037
1118
|
}
|
|
1038
1119
|
}
|
|
1039
1120
|
var useLiveView = (liveViewKey = PresetOfLiveViewKey) => {
|
|
@@ -1052,7 +1133,7 @@ var LiveView = {
|
|
|
1052
1133
|
|
|
1053
1134
|
// src/captcha-monitor.js
|
|
1054
1135
|
import { v4 as uuidv4 } from "uuid";
|
|
1055
|
-
var
|
|
1136
|
+
var logger6 = createLogger("Captcha");
|
|
1056
1137
|
function useCaptchaMonitor(page, options) {
|
|
1057
1138
|
const { domSelector, urlPattern, onDetected } = options;
|
|
1058
1139
|
if (!domSelector && !urlPattern) {
|
|
@@ -1124,7 +1205,7 @@ function useCaptchaMonitor(page, options) {
|
|
|
1124
1205
|
};
|
|
1125
1206
|
})();
|
|
1126
1207
|
}, { selector: domSelector, callbackName: exposedFunctionName, cleanerName });
|
|
1127
|
-
|
|
1208
|
+
logger6.success("useCaptchaMonitor", `DOM \u76D1\u63A7\u5DF2\u542F\u7528: ${domSelector}`);
|
|
1128
1209
|
cleanupFns.push(async () => {
|
|
1129
1210
|
try {
|
|
1130
1211
|
await page.evaluate((name) => {
|
|
@@ -1147,14 +1228,14 @@ function useCaptchaMonitor(page, options) {
|
|
|
1147
1228
|
}
|
|
1148
1229
|
};
|
|
1149
1230
|
page.on("framenavigated", frameHandler);
|
|
1150
|
-
|
|
1231
|
+
logger6.success("useCaptchaMonitor", `URL \u76D1\u63A7\u5DF2\u542F\u7528: ${urlPattern}`);
|
|
1151
1232
|
cleanupFns.push(async () => {
|
|
1152
1233
|
page.off("framenavigated", frameHandler);
|
|
1153
1234
|
});
|
|
1154
1235
|
}
|
|
1155
1236
|
return {
|
|
1156
1237
|
stop: async () => {
|
|
1157
|
-
|
|
1238
|
+
logger6.info("useCaptchaMonitor", "\u6B63\u5728\u505C\u6B62\u76D1\u63A7...");
|
|
1158
1239
|
for (const fn of cleanupFns) {
|
|
1159
1240
|
await fn();
|
|
1160
1241
|
}
|
|
@@ -1169,7 +1250,7 @@ var Captcha = {
|
|
|
1169
1250
|
// src/sse.js
|
|
1170
1251
|
import https from "https";
|
|
1171
1252
|
import { URL as URL2 } from "url";
|
|
1172
|
-
var
|
|
1253
|
+
var logger7 = createLogger("Sse");
|
|
1173
1254
|
var Sse = {
|
|
1174
1255
|
/**
|
|
1175
1256
|
* 解析 SSE 流文本
|
|
@@ -1188,11 +1269,11 @@ var Sse = {
|
|
|
1188
1269
|
events.push(JSON.parse(jsonContent));
|
|
1189
1270
|
}
|
|
1190
1271
|
} catch (e) {
|
|
1191
|
-
|
|
1272
|
+
logger7.debug("parseSseStream", `JSON \u89E3\u6790\u5931\u8D25: ${e.message}, line: ${line.substring(0, 100)}...`);
|
|
1192
1273
|
}
|
|
1193
1274
|
}
|
|
1194
1275
|
}
|
|
1195
|
-
|
|
1276
|
+
logger7.success("parseSseStream", `\u89E3\u6790\u5B8C\u6210, events \u6570\u91CF: ${events.length}`);
|
|
1196
1277
|
return events;
|
|
1197
1278
|
},
|
|
1198
1279
|
/**
|
|
@@ -1227,7 +1308,7 @@ var Sse = {
|
|
|
1227
1308
|
const workPromise = new Promise((resolve, reject) => {
|
|
1228
1309
|
page.route(urlPattern, async (route) => {
|
|
1229
1310
|
const request = route.request();
|
|
1230
|
-
|
|
1311
|
+
logger7.info(`[MITM] \u5DF2\u62E6\u622A\u8BF7\u6C42: ${request.url()}`);
|
|
1231
1312
|
try {
|
|
1232
1313
|
const headers = await request.allHeaders();
|
|
1233
1314
|
const postData = request.postData();
|
|
@@ -1252,7 +1333,7 @@ var Sse = {
|
|
|
1252
1333
|
clearTimeout(initialTimer);
|
|
1253
1334
|
initialTimer = null;
|
|
1254
1335
|
}
|
|
1255
|
-
|
|
1336
|
+
logger7.debug("[Intercept] \u5DF2\u63A5\u6536\u521D\u59CB\u6570\u636E");
|
|
1256
1337
|
}
|
|
1257
1338
|
chunks.push(chunk);
|
|
1258
1339
|
const textChunk = chunk.toString("utf-8");
|
|
@@ -1261,18 +1342,18 @@ var Sse = {
|
|
|
1261
1342
|
try {
|
|
1262
1343
|
onData(textChunk, resolve, accumulatedText);
|
|
1263
1344
|
} catch (e) {
|
|
1264
|
-
|
|
1345
|
+
logger7.fail(`onData \u9519\u8BEF`, e);
|
|
1265
1346
|
}
|
|
1266
1347
|
}
|
|
1267
1348
|
});
|
|
1268
1349
|
res.on("end", () => {
|
|
1269
|
-
|
|
1350
|
+
logger7.info("[MITM] \u4E0A\u6E38\u54CD\u5E94\u7ED3\u675F");
|
|
1270
1351
|
clearAllTimers();
|
|
1271
1352
|
if (onEnd) {
|
|
1272
1353
|
try {
|
|
1273
1354
|
onEnd(accumulatedText, resolve);
|
|
1274
1355
|
} catch (e) {
|
|
1275
|
-
|
|
1356
|
+
logger7.fail(`onEnd \u9519\u8BEF`, e);
|
|
1276
1357
|
}
|
|
1277
1358
|
} else if (!onData) {
|
|
1278
1359
|
resolve(accumulatedText);
|
|
@@ -1354,7 +1435,7 @@ var Sse = {
|
|
|
1354
1435
|
import { gotScraping } from "got-scraping";
|
|
1355
1436
|
import { Agent as HttpAgent } from "http";
|
|
1356
1437
|
import { Agent as HttpsAgent } from "https";
|
|
1357
|
-
var
|
|
1438
|
+
var logger8 = createLogger("Interception");
|
|
1358
1439
|
var SHARED_HTTP_AGENT = new HttpAgent({ keepAlive: false });
|
|
1359
1440
|
var SHARED_HTTPS_AGENT = new HttpsAgent({ keepAlive: false, rejectUnauthorized: false });
|
|
1360
1441
|
var DirectConfig = {
|
|
@@ -1485,7 +1566,7 @@ var Interception = {
|
|
|
1485
1566
|
if (mergedBlockingConfig.blockFont) enabledCategories.push("\u5B57\u4F53");
|
|
1486
1567
|
if (mergedBlockingConfig.blockCss) enabledCategories.push("CSS");
|
|
1487
1568
|
if (mergedBlockingConfig.blockOther) enabledCategories.push("\u5176\u4ED6");
|
|
1488
|
-
|
|
1569
|
+
logger8.start("setup", hasDirectDomains ? `\u76F4\u8FDE\u57DF\u540D: [${directDomains.length} \u4E2A] | \u5C4F\u853D: [${enabledCategories.join(", ")}]` : `\u4EC5\u8D44\u6E90\u5C4F\u853D\u6A21\u5F0F | \u5C4F\u853D: [${enabledCategories.join(", ")}]`);
|
|
1489
1570
|
await page.route("**/*", async (route) => {
|
|
1490
1571
|
let handled = false;
|
|
1491
1572
|
try {
|
|
@@ -1512,6 +1593,9 @@ var Interception = {
|
|
|
1512
1593
|
try {
|
|
1513
1594
|
const reqHeaders = await request.allHeaders();
|
|
1514
1595
|
delete reqHeaders["host"];
|
|
1596
|
+
const currentAcceptLanguage = reqHeaders["accept-language"] || "";
|
|
1597
|
+
AntiDetect.applyLocaleHeaders(reqHeaders, currentAcceptLanguage);
|
|
1598
|
+
const resolvedAcceptLanguage = reqHeaders["accept-language"] || "";
|
|
1515
1599
|
const userAgent = reqHeaders["user-agent"] || "";
|
|
1516
1600
|
const method = request.method();
|
|
1517
1601
|
const postData = method !== "GET" && method !== "HEAD" ? request.postDataBuffer() : void 0;
|
|
@@ -1525,7 +1609,7 @@ var Interception = {
|
|
|
1525
1609
|
responseType: "buffer",
|
|
1526
1610
|
// 强制获取 Buffer
|
|
1527
1611
|
// 模拟浏览器 TLS 指纹
|
|
1528
|
-
headerGeneratorOptions:
|
|
1612
|
+
headerGeneratorOptions: AntiDetect.getTlsFingerprintOptions(userAgent, resolvedAcceptLanguage),
|
|
1529
1613
|
// 使用共享的 Agent 单例(keepAlive: false,不会池化连接)
|
|
1530
1614
|
agent: {
|
|
1531
1615
|
http: SHARED_HTTP_AGENT,
|
|
@@ -1547,7 +1631,7 @@ var Interception = {
|
|
|
1547
1631
|
delete resHeaders["transfer-encoding"];
|
|
1548
1632
|
delete resHeaders["connection"];
|
|
1549
1633
|
delete resHeaders["keep-alive"];
|
|
1550
|
-
isSilent ?
|
|
1634
|
+
isSilent ? logger8.debug(`\u76F4\u8FDE\u6210\u529F: ${urlPath}`) : logger8.info(`\u76F4\u8FDE\u6210\u529F: ${urlPath}`);
|
|
1551
1635
|
await safeFulfill(route, {
|
|
1552
1636
|
status: response.statusCode,
|
|
1553
1637
|
headers: resHeaders,
|
|
@@ -1559,7 +1643,7 @@ var Interception = {
|
|
|
1559
1643
|
const isTimeout = e.code === "ETIMEDOUT" || e.message.toLowerCase().includes("timeout");
|
|
1560
1644
|
const action = fallbackToProxy ? "\u56DE\u9000\u4EE3\u7406" : "\u5DF2\u653E\u5F03";
|
|
1561
1645
|
const reason = isTimeout ? `\u8D85\u65F6(${DirectConfig.directTimeout}s)` : `\u5F02\u5E38: ${e.message}`;
|
|
1562
|
-
|
|
1646
|
+
logger8.warn(`\u76F4\u8FDE${reason}\uFF0C${action}: ${urlPath}`);
|
|
1563
1647
|
if (fallbackToProxy) {
|
|
1564
1648
|
await safeContinue(route);
|
|
1565
1649
|
} else {
|
|
@@ -1572,7 +1656,7 @@ var Interception = {
|
|
|
1572
1656
|
await safeContinue(route);
|
|
1573
1657
|
handled = true;
|
|
1574
1658
|
} catch (err) {
|
|
1575
|
-
|
|
1659
|
+
logger8.warn(`\u8DEF\u7531\u5904\u7406\u5F02\u5E38: ${err.message}`);
|
|
1576
1660
|
if (!handled) {
|
|
1577
1661
|
try {
|
|
1578
1662
|
await route.continue();
|
|
@@ -1609,7 +1693,7 @@ function isIgnorableError(error) {
|
|
|
1609
1693
|
var usePlaywrightToolKit = () => {
|
|
1610
1694
|
return {
|
|
1611
1695
|
ApifyKit,
|
|
1612
|
-
|
|
1696
|
+
AntiDetect,
|
|
1613
1697
|
Humanize,
|
|
1614
1698
|
Launch,
|
|
1615
1699
|
LiveView,
|