jumpy-lion 0.1.6-beta.7 → 1.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -607
- package/browser/fonts/macos-bundle/40-macos-aliases.conf +515 -0
- package/browser/fonts/macos-bundle/README.md +24 -0
- package/browser/fonts/macos-bundle/install.sh +83 -0
- package/dist/browser-controller.d.ts +47 -4
- package/dist/browser-controller.d.ts.map +1 -1
- package/dist/browser-controller.js +219 -278
- package/dist/browser-controller.js.map +1 -1
- package/dist/browser-plugin.d.ts +82 -110
- package/dist/browser-plugin.d.ts.map +1 -1
- package/dist/browser-plugin.js +376 -379
- package/dist/browser-plugin.js.map +1 -1
- package/dist/browser-process/align-fingerprint-config.d.ts +36 -0
- package/dist/browser-process/align-fingerprint-config.d.ts.map +1 -0
- package/dist/browser-process/align-fingerprint-config.js +74 -0
- package/dist/browser-process/align-fingerprint-config.js.map +1 -0
- package/dist/browser-process/anti-detect-browser.d.ts +2 -2
- package/dist/browser-process/anti-detect-browser.js +7 -7
- package/dist/browser-process/anti-detect-config.d.ts +131 -2
- package/dist/browser-process/anti-detect-config.d.ts.map +1 -1
- package/dist/browser-process/anti-detect-config.js +337 -127
- package/dist/browser-process/anti-detect-config.js.map +1 -1
- package/dist/browser-process/browser.d.ts +25 -92
- package/dist/browser-process/browser.d.ts.map +1 -1
- package/dist/browser-process/browser.js +429 -73
- package/dist/browser-process/browser.js.map +1 -1
- package/dist/browser-process/fingerprint-config.d.ts +146 -0
- package/dist/browser-process/fingerprint-config.d.ts.map +1 -0
- package/dist/browser-process/fingerprint-config.js +308 -0
- package/dist/browser-process/fingerprint-config.js.map +1 -0
- package/dist/browser-process/get-chrome-executable.d.ts +6 -0
- package/dist/browser-process/get-chrome-executable.d.ts.map +1 -1
- package/dist/browser-process/get-chrome-executable.js +79 -4
- package/dist/browser-process/get-chrome-executable.js.map +1 -1
- package/dist/browser-process/gpu-family-profiles.d.ts +58 -0
- package/dist/browser-process/gpu-family-profiles.d.ts.map +1 -0
- package/dist/browser-process/gpu-family-profiles.js +480 -0
- package/dist/browser-process/gpu-family-profiles.js.map +1 -0
- package/dist/browser-process/launch-options.d.ts +301 -0
- package/dist/browser-process/launch-options.d.ts.map +1 -0
- package/dist/browser-process/launch-options.js +2 -0
- package/dist/browser-process/launch-options.js.map +1 -0
- package/dist/browser-process/port-allocator.d.ts +64 -0
- package/dist/browser-process/port-allocator.d.ts.map +1 -0
- package/dist/browser-process/port-allocator.js +198 -0
- package/dist/browser-process/port-allocator.js.map +1 -0
- package/dist/browser-process/process.d.ts +85 -0
- package/dist/browser-process/process.d.ts.map +1 -1
- package/dist/browser-process/process.js +235 -19
- package/dist/browser-process/process.js.map +1 -1
- package/dist/browser-process/signal-cleanup.d.ts +40 -0
- package/dist/browser-process/signal-cleanup.d.ts.map +1 -0
- package/dist/browser-process/signal-cleanup.js +105 -0
- package/dist/browser-process/signal-cleanup.js.map +1 -0
- package/dist/browser-process/widevine-cdm.d.ts +144 -0
- package/dist/browser-process/widevine-cdm.d.ts.map +1 -0
- package/dist/browser-process/widevine-cdm.js +298 -0
- package/dist/browser-process/widevine-cdm.js.map +1 -0
- package/dist/browser-profiles/chrome/populate-profile.d.ts.map +1 -1
- package/dist/browser-profiles/chrome/populate-profile.js +21 -22
- package/dist/browser-profiles/chrome/populate-profile.js.map +1 -1
- package/dist/connection/cdp-reconnection.d.ts +50 -0
- package/dist/connection/cdp-reconnection.d.ts.map +1 -0
- package/dist/connection/cdp-reconnection.js +75 -0
- package/dist/connection/cdp-reconnection.js.map +1 -0
- package/dist/connection/page-session.d.ts +114 -0
- package/dist/connection/page-session.d.ts.map +1 -0
- package/dist/connection/page-session.js +237 -0
- package/dist/connection/page-session.js.map +1 -0
- package/dist/connection/session-client.d.ts +34 -0
- package/dist/connection/session-client.d.ts.map +1 -0
- package/dist/connection/session-client.js +129 -0
- package/dist/connection/session-client.js.map +1 -0
- package/dist/crawler.d.ts +9 -88
- package/dist/crawler.d.ts.map +1 -1
- package/dist/crawler.js +157 -8
- package/dist/crawler.js.map +1 -1
- package/dist/fingerprinting/fingerprint-injector.d.ts +24 -114
- package/dist/fingerprinting/fingerprint-injector.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-injector.js +305 -611
- package/dist/fingerprinting/fingerprint-injector.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/datadome-bypass.d.ts +1 -1
- package/dist/fingerprinting/fingerprint-overrides/datadome-bypass.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/datadome-bypass.js +16 -14
- package/dist/fingerprinting/fingerprint-overrides/datadome-bypass.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/index.d.ts +6 -2
- package/dist/fingerprinting/fingerprint-overrides/index.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/index.js +6 -2
- package/dist/fingerprinting/fingerprint-overrides/index.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/keyboard-humanization.d.ts +1 -1
- package/dist/fingerprinting/fingerprint-overrides/keyboard-humanization.js +4 -4
- package/dist/fingerprinting/fingerprint-overrides/keyboard-humanization.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/locale-spoofing.js +4 -4
- package/dist/fingerprinting/fingerprint-overrides/navigator-override.d.ts +18 -0
- package/dist/fingerprinting/fingerprint-overrides/navigator-override.d.ts.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/navigator-override.js +136 -0
- package/dist/fingerprinting/fingerprint-overrides/navigator-override.js.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/override.d.ts +151 -0
- package/dist/fingerprinting/fingerprint-overrides/override.d.ts.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/override.js +14 -0
- package/dist/fingerprinting/fingerprint-overrides/override.js.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/platform-consistency.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/platform-consistency.js +46 -15
- package/dist/fingerprinting/fingerprint-overrides/platform-consistency.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/registry.d.ts +43 -0
- package/dist/fingerprinting/fingerprint-overrides/registry.d.ts.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/registry.js +342 -0
- package/dist/fingerprinting/fingerprint-overrides/registry.js.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/screen-override.d.ts +16 -0
- package/dist/fingerprinting/fingerprint-overrides/screen-override.d.ts.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/screen-override.js +175 -0
- package/dist/fingerprinting/fingerprint-overrides/screen-override.js.map +1 -0
- package/dist/fingerprinting/fingerprint-overrides/scroll-humanization.d.ts +1 -1
- package/dist/fingerprinting/fingerprint-overrides/scroll-humanization.js +4 -4
- package/dist/fingerprinting/fingerprint-overrides/scroll-humanization.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/stealth-script.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/stealth-script.js +75 -9
- package/dist/fingerprinting/fingerprint-overrides/stealth-script.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/ua-ch.d.ts +1 -1
- package/dist/fingerprinting/fingerprint-overrides/ua-ch.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/ua-ch.js +34 -20
- package/dist/fingerprinting/fingerprint-overrides/ua-ch.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/webgl-spoofing.d.ts.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/webgl-spoofing.js +25 -6
- package/dist/fingerprinting/fingerprint-overrides/webgl-spoofing.js.map +1 -1
- package/dist/fingerprinting/fingerprint-overrides/webrtc-spoofing.js +1 -1
- package/dist/fingerprinting/hardware-realism.d.ts +83 -0
- package/dist/fingerprinting/hardware-realism.d.ts.map +1 -0
- package/dist/fingerprinting/hardware-realism.js +156 -0
- package/dist/fingerprinting/hardware-realism.js.map +1 -0
- package/dist/fingerprinting/injection-planner.d.ts +133 -0
- package/dist/fingerprinting/injection-planner.d.ts.map +1 -0
- package/dist/fingerprinting/injection-planner.js +497 -0
- package/dist/fingerprinting/injection-planner.js.map +1 -0
- package/dist/fingerprinting/locale-resolver.d.ts +8 -1
- package/dist/fingerprinting/locale-resolver.d.ts.map +1 -1
- package/dist/fingerprinting/locale-resolver.js +26 -3
- package/dist/fingerprinting/locale-resolver.js.map +1 -1
- package/dist/fingerprinting/non-apify-fingerprint-generator.d.ts +4 -4
- package/dist/fingerprinting/non-apify-fingerprint-generator.d.ts.map +1 -1
- package/dist/fingerprinting/non-apify-fingerprint-generator.js +24 -58
- package/dist/fingerprinting/non-apify-fingerprint-generator.js.map +1 -1
- package/dist/fingerprinting/non-apify-profiles.json +10515 -8494
- package/dist/fingerprinting/os-consistency.d.ts +28 -0
- package/dist/fingerprinting/os-consistency.d.ts.map +1 -0
- package/dist/fingerprinting/os-consistency.js +45 -0
- package/dist/fingerprinting/os-consistency.js.map +1 -0
- package/dist/fingerprinting/profile-quality.d.ts +25 -0
- package/dist/fingerprinting/profile-quality.d.ts.map +1 -0
- package/dist/fingerprinting/profile-quality.js +310 -0
- package/dist/fingerprinting/profile-quality.js.map +1 -0
- package/dist/fingerprinting/profile-selector.d.ts +102 -0
- package/dist/fingerprinting/profile-selector.d.ts.map +1 -0
- package/dist/fingerprinting/profile-selector.js +168 -0
- package/dist/fingerprinting/profile-selector.js.map +1 -0
- package/dist/fingerprinting/ua-alignment.d.ts +83 -0
- package/dist/fingerprinting/ua-alignment.d.ts.map +1 -0
- package/dist/fingerprinting/ua-alignment.js +197 -0
- package/dist/fingerprinting/ua-alignment.js.map +1 -0
- package/dist/fingerprinting/ua-ch-brands.d.ts +88 -0
- package/dist/fingerprinting/ua-ch-brands.d.ts.map +1 -0
- package/dist/fingerprinting/ua-ch-brands.js +174 -0
- package/dist/fingerprinting/ua-ch-brands.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/input/dropdown-selector.d.ts +90 -0
- package/dist/input/dropdown-selector.d.ts.map +1 -0
- package/dist/input/dropdown-selector.js +394 -0
- package/dist/input/dropdown-selector.js.map +1 -0
- package/dist/input/element-target.d.ts +152 -0
- package/dist/input/element-target.d.ts.map +1 -0
- package/dist/input/element-target.js +461 -0
- package/dist/input/element-target.js.map +1 -0
- package/dist/input/input-emulator.d.ts +85 -0
- package/dist/input/input-emulator.d.ts.map +1 -0
- package/dist/input/input-emulator.js +325 -0
- package/dist/input/input-emulator.js.map +1 -0
- package/dist/input/input-transport.d.ts +60 -0
- package/dist/input/input-transport.d.ts.map +1 -0
- package/dist/input/input-transport.js +28 -0
- package/dist/input/input-transport.js.map +1 -0
- package/dist/input/recording-transport.d.ts +32 -0
- package/dist/input/recording-transport.d.ts.map +1 -0
- package/dist/input/recording-transport.js +43 -0
- package/dist/input/recording-transport.js.map +1 -0
- package/dist/keyboard-layout.d.ts +2 -2
- package/dist/keyboard-layout.js +2 -2
- package/dist/navigation/page-navigation.d.ts +67 -0
- package/dist/navigation/page-navigation.d.ts.map +1 -0
- package/dist/navigation/page-navigation.js +122 -0
- package/dist/navigation/page-navigation.js.map +1 -0
- package/dist/network/network-watch.d.ts +110 -0
- package/dist/network/network-watch.d.ts.map +1 -0
- package/dist/network/network-watch.js +223 -0
- package/dist/network/network-watch.js.map +1 -0
- package/dist/page.d.ts +118 -104
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +311 -1130
- package/dist/page.js.map +1 -1
- package/dist/session/inject-hook.d.ts +24 -0
- package/dist/session/inject-hook.d.ts.map +1 -0
- package/dist/session/inject-hook.js +80 -0
- package/dist/session/inject-hook.js.map +1 -0
- package/dist/session/save.d.ts +21 -0
- package/dist/session/save.d.ts.map +1 -0
- package/dist/session/save.js +163 -0
- package/dist/session/save.js.map +1 -0
- package/dist/session/tar-data-dir.d.ts +18 -0
- package/dist/session/tar-data-dir.d.ts.map +1 -0
- package/dist/session/tar-data-dir.js +105 -0
- package/dist/session/tar-data-dir.js.map +1 -0
- package/dist/session/types.d.ts +80 -0
- package/dist/session/types.d.ts.map +1 -0
- package/dist/session/types.js +29 -0
- package/dist/session/types.js.map +1 -0
- package/dist/session-profile.d.ts +102 -0
- package/dist/session-profile.d.ts.map +1 -0
- package/dist/session-profile.js +163 -0
- package/dist/session-profile.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +17 -6
- package/scripts/postinstall.cjs +58 -0
- package/dist/fingerprinting/custom-fingerprint-injector.d.ts +0 -87
- package/dist/fingerprinting/custom-fingerprint-injector.d.ts.map +0 -1
- package/dist/fingerprinting/custom-fingerprint-injector.js +0 -342
- package/dist/fingerprinting/custom-fingerprint-injector.js.map +0 -1
- package/dist/launcher-wrap.d.ts +0 -10
- package/dist/launcher-wrap.d.ts.map +0 -1
- package/dist/launcher-wrap.js +0 -11
- package/dist/launcher-wrap.js.map +0 -1
|
@@ -62,14 +62,112 @@
|
|
|
62
62
|
*/
|
|
63
63
|
import { log } from 'crawlee';
|
|
64
64
|
import { gotScraping } from 'got-scraping';
|
|
65
|
-
import {
|
|
66
|
-
import {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
65
|
+
import { normalizePlatformVersion } from './hardware-realism.js';
|
|
66
|
+
import { buildInjectionPlan, isNativeStealth, needsIpTimezoneLookup, } from './injection-planner.js';
|
|
67
|
+
import { alignUserAgentWithBrowserVersion, alignUAClientHintsVersion } from './ua-alignment.js';
|
|
68
|
+
import { canonicalBrandList } from './ua-ch-brands.js';
|
|
69
|
+
/**
|
|
70
|
+
* Per-proxy-session cache of the ipinfo.io timezone lookup, held for process
|
|
71
|
+
* lifetime. Keyed by proxyUrl so concurrent pages on the SAME session share one
|
|
72
|
+
* in-flight fetch and every later page reuses the result - but two different
|
|
73
|
+
* proxy sessions get independent entries and NEVER bleed a timezone into one
|
|
74
|
+
* another. No-proxy lookups get their own dedicated key.
|
|
75
|
+
*/
|
|
76
|
+
const NO_PROXY_TIMEZONE_KEY = '__no_proxy__';
|
|
77
|
+
const timezoneByProxy = new Map();
|
|
78
|
+
/**
|
|
79
|
+
* Per-browser-process cache of the CDP `Browser.getVersion` probe. Keyed by the
|
|
80
|
+
* CDP connection's host:port - Chrome's remote-debugging port is unique per
|
|
81
|
+
* browser process, so every page/target of one browser shares a single probe
|
|
82
|
+
* while distinct browser processes stay independent. This is a version value,
|
|
83
|
+
* never proxy-session identity, so per-process keying is the correct scope.
|
|
84
|
+
*/
|
|
85
|
+
const chromeVersionByEndpoint = new Map();
|
|
86
|
+
/**
|
|
87
|
+
* UA-CH platform/arch/version constants. Hoisted to module scope so they are
|
|
88
|
+
* allocated once per process rather than rebuilt on every page's
|
|
89
|
+
* setUserAgentWithMetadata call.
|
|
90
|
+
*/
|
|
91
|
+
const UA_PLATFORM_NAME_MAP = {
|
|
92
|
+
'Win32': 'Windows',
|
|
93
|
+
'MacIntel': 'macOS',
|
|
94
|
+
'Linux x86_64': 'Linux',
|
|
95
|
+
};
|
|
96
|
+
const UA_ARCH_MAP = {
|
|
97
|
+
'Win32': 'x86',
|
|
98
|
+
'MacIntel': 'arm',
|
|
99
|
+
'Linux x86_64': 'x86',
|
|
100
|
+
};
|
|
101
|
+
// The platformVersion candidate tables live in hardware-realism.ts so this path
|
|
102
|
+
// and the native FingerprintConfig path cannot disagree - they did, and a Linux
|
|
103
|
+
// profile ended up advertising a macOS version (issue #137).
|
|
104
|
+
/**
|
|
105
|
+
* Cache of the deterministic UA-seed hash per userAgent string. The hash feeds
|
|
106
|
+
* the platformVersion candidate pick; it is a pure function of the UA string,
|
|
107
|
+
* so it is computed once per distinct UA rather than on every page injection.
|
|
108
|
+
*/
|
|
109
|
+
const uaSeedHashCache = new Map();
|
|
110
|
+
/**
|
|
111
|
+
* Stable hash of the UA string. Same UA → same value across reloads/sessions;
|
|
112
|
+
* a different UA (different fingerprint seed) → a different value. Math-only
|
|
113
|
+
* hash (mod 2^31-1) to satisfy the no-bitwise lint rule.
|
|
114
|
+
*/
|
|
115
|
+
function uaSeedHash(userAgent) {
|
|
116
|
+
const cached = uaSeedHashCache.get(userAgent);
|
|
117
|
+
if (cached !== undefined)
|
|
118
|
+
return cached;
|
|
119
|
+
let hash = 0xCAFE1003;
|
|
120
|
+
for (let i = 0; i < userAgent.length; i++) {
|
|
121
|
+
hash = (hash * 31 + userAgent.charCodeAt(i)) % 0x7FFFFFFF;
|
|
122
|
+
}
|
|
123
|
+
uaSeedHashCache.set(userAgent, hash);
|
|
124
|
+
return hash;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* The Apify-platform recommended settings, frozen once at module load. These
|
|
128
|
+
* are per-launch constants (no per-page or per-session variation), so the graph
|
|
129
|
+
* is built exactly once instead of on every page. Callers only ever spread this
|
|
130
|
+
* value into a fresh object; the deep freeze guards against accidental in-place
|
|
131
|
+
* mutation bleeding across pages.
|
|
132
|
+
*/
|
|
133
|
+
const APIFY_RECOMMENDED_SETTINGS = Object.freeze({
|
|
134
|
+
debug: false, // Disabled - anti-bot services detect debug console prefixes
|
|
135
|
+
enableAdvancedStealth: true,
|
|
136
|
+
bypassRuntimeEnable: true,
|
|
137
|
+
platform: 'Win32', // Spoof as Windows instead of Linux for better evasion
|
|
138
|
+
spoofWebGL: true,
|
|
139
|
+
spoofAudioContext: true,
|
|
140
|
+
spoofClientRects: true,
|
|
141
|
+
detectTimezone: true, // Auto-detect timezone from proxy
|
|
142
|
+
// New advanced spoofing options for 2025
|
|
143
|
+
spoofFonts: true,
|
|
144
|
+
spoofPerformance: true,
|
|
145
|
+
spoofLocale: true,
|
|
146
|
+
useFingerprintDefaults: true, // Use fingerprint-generator Windows values
|
|
147
|
+
// 2024-2025 DataDome bypass options
|
|
148
|
+
bypassCDPDetection: true,
|
|
149
|
+
spoofStorage: true,
|
|
150
|
+
spoofTiming: true,
|
|
151
|
+
enableDataDomeBypass: false, // Enable only when needed
|
|
152
|
+
// Humanization disabled by default for performance
|
|
153
|
+
humanization: Object.freeze({
|
|
154
|
+
mouse: true,
|
|
155
|
+
keyboard: true,
|
|
156
|
+
scroll: true,
|
|
157
|
+
}),
|
|
158
|
+
viewport: Object.freeze({
|
|
159
|
+
width: 1920,
|
|
160
|
+
height: 1080,
|
|
161
|
+
deviceScaleFactor: 1,
|
|
162
|
+
mobile: false,
|
|
163
|
+
}),
|
|
164
|
+
});
|
|
165
|
+
/** Test-only: drop the process-level runtime caches so runs start fresh. */
|
|
166
|
+
export function resetInjectorRuntimeCachesForTesting() {
|
|
167
|
+
timezoneByProxy.clear();
|
|
168
|
+
chromeVersionByEndpoint.clear();
|
|
169
|
+
uaSeedHashCache.clear();
|
|
170
|
+
}
|
|
73
171
|
export class CdpFingerprintInjector {
|
|
74
172
|
constructor(page) {
|
|
75
173
|
Object.defineProperty(this, "page", {
|
|
@@ -89,38 +187,9 @@ export class CdpFingerprintInjector {
|
|
|
89
187
|
* Use getDataDomeSettings() or enable humanization manually for anti-bot sites.
|
|
90
188
|
*/
|
|
91
189
|
static getApifyRecommendedSettings() {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
bypassRuntimeEnable: true,
|
|
96
|
-
platform: 'Win32', // Spoof as Windows instead of Linux for better evasion
|
|
97
|
-
spoofWebGL: true,
|
|
98
|
-
spoofAudioContext: true,
|
|
99
|
-
spoofClientRects: true,
|
|
100
|
-
detectTimezone: true, // Auto-detect timezone from proxy
|
|
101
|
-
// New advanced spoofing options for 2025
|
|
102
|
-
spoofFonts: true,
|
|
103
|
-
spoofPerformance: true,
|
|
104
|
-
spoofLocale: true,
|
|
105
|
-
useFingerprintDefaults: true, // Use fingerprint-generator Windows values
|
|
106
|
-
// 2024-2025 DataDome bypass options
|
|
107
|
-
bypassCDPDetection: true,
|
|
108
|
-
spoofStorage: true,
|
|
109
|
-
spoofTiming: true,
|
|
110
|
-
enableDataDomeBypass: false, // Enable only when needed
|
|
111
|
-
// Humanization disabled by default for performance
|
|
112
|
-
humanization: {
|
|
113
|
-
mouse: true,
|
|
114
|
-
keyboard: true,
|
|
115
|
-
scroll: true,
|
|
116
|
-
},
|
|
117
|
-
viewport: {
|
|
118
|
-
width: 1920,
|
|
119
|
-
height: 1080,
|
|
120
|
-
deviceScaleFactor: 1,
|
|
121
|
-
mobile: false,
|
|
122
|
-
},
|
|
123
|
-
};
|
|
190
|
+
// Frozen module-level constant - see APIFY_RECOMMENDED_SETTINGS. Callers
|
|
191
|
+
// spread it into a fresh object, so returning the shared value is safe.
|
|
192
|
+
return APIFY_RECOMMENDED_SETTINGS;
|
|
124
193
|
}
|
|
125
194
|
/**
|
|
126
195
|
* Get settings specifically optimized for DataDome-protected sites.
|
|
@@ -165,12 +234,12 @@ export class CdpFingerprintInjector {
|
|
|
165
234
|
process.env.CRAWLEE_HEADLESS === '1' ||
|
|
166
235
|
process.env.HEADLESS === 'true';
|
|
167
236
|
if (!isHeadless) {
|
|
168
|
-
log.
|
|
169
|
-
log.
|
|
237
|
+
log.debug('⚠️ Headful mode detected - SKIPPING viewport override to preserve natural window behavior');
|
|
238
|
+
log.debug('✅ Browser will use OS window dimensions - scrolling will work naturally');
|
|
170
239
|
return; // Skip viewport override in headful mode
|
|
171
240
|
}
|
|
172
241
|
// Only apply viewport override in headless mode
|
|
173
|
-
log.
|
|
242
|
+
log.debug('Headless mode detected - applying viewport override');
|
|
174
243
|
let defaultViewport = {
|
|
175
244
|
width: 1920,
|
|
176
245
|
height: 1080,
|
|
@@ -189,7 +258,7 @@ export class CdpFingerprintInjector {
|
|
|
189
258
|
deviceScaleFactor: fingerprint.screen.devicePixelRatio || 1,
|
|
190
259
|
mobile: false,
|
|
191
260
|
};
|
|
192
|
-
log.
|
|
261
|
+
log.debug('Viewport configured for scrolling', {
|
|
193
262
|
screenHeight,
|
|
194
263
|
viewportHeight,
|
|
195
264
|
reservedSpace: 150,
|
|
@@ -198,10 +267,10 @@ export class CdpFingerprintInjector {
|
|
|
198
267
|
});
|
|
199
268
|
}
|
|
200
269
|
const finalViewport = Object.assign(defaultViewport, viewport);
|
|
201
|
-
log.
|
|
270
|
+
log.debug('Final viewport settings (headless mode)', finalViewport);
|
|
202
271
|
// CDP call to override device metrics (headless mode only)
|
|
203
272
|
await this.page.client.Emulation.setDeviceMetricsOverride(finalViewport);
|
|
204
|
-
log.
|
|
273
|
+
log.debug('setDeviceMetricsOverride completed (headless mode)');
|
|
205
274
|
}
|
|
206
275
|
async setExtraHTTPHeaders(headers) {
|
|
207
276
|
await this.page.client.Network.setExtraHTTPHeaders({ headers });
|
|
@@ -212,35 +281,43 @@ export class CdpFingerprintInjector {
|
|
|
212
281
|
* Without this, the browser sends its real client hints headers which
|
|
213
282
|
* don't match the spoofed navigator.userAgentData values.
|
|
214
283
|
*/
|
|
215
|
-
async setUserAgentWithMetadata(userAgent, platform, uaClientHints) {
|
|
216
|
-
const
|
|
217
|
-
'Win32': 'Windows',
|
|
218
|
-
'MacIntel': 'macOS',
|
|
219
|
-
'Linux x86_64': 'Linux',
|
|
220
|
-
};
|
|
221
|
-
const platformName = platformMap[platform] || 'Windows';
|
|
284
|
+
async setUserAgentWithMetadata(userAgent, platform, uaClientHints, binaryFullVersion) {
|
|
285
|
+
const platformName = UA_PLATFORM_NAME_MAP[platform] || 'Windows';
|
|
222
286
|
// Determine Chrome version from UA string
|
|
223
287
|
const chromeVersionMatch = userAgent.match(/Chrome\/(\d+)\.(\d+)\.(\d+)\.(\d+)/);
|
|
224
288
|
if (!chromeVersionMatch) {
|
|
225
289
|
throw new Error(`Cannot derive Chrome version from user agent: ${userAgent}`);
|
|
226
290
|
}
|
|
227
291
|
const majorVersion = chromeVersionMatch[1];
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
}
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
'
|
|
242
|
-
|
|
243
|
-
|
|
292
|
+
// UA-CH `fullVersion` must be the REAL build (147.0.7727.56), never the
|
|
293
|
+
// UA string's version. UA Reduction (Chrome 107+) freezes the UA to
|
|
294
|
+
// MAJOR.0.0.0 but leaves UA-CH untouched, so deriving it from the UA -
|
|
295
|
+
// as this method used to - put "147.0.0.0" into
|
|
296
|
+
// `sec-ch-ua-full-version` / `-full-version-list`: a shape no Chrome
|
|
297
|
+
// emits. It stayed hidden because high-entropy hints only ship after the
|
|
298
|
+
// origin asks via `accept-ch`, so the first document passed and every
|
|
299
|
+
// later XHR was refused (issue #131). Order: the probed binary build →
|
|
300
|
+
// the profile's own build → the reduced UA value, which is a bug worth
|
|
301
|
+
// logging rather than a value worth sending.
|
|
302
|
+
const uaDerivedVersion = `${majorVersion}.${chromeVersionMatch[2]}.${chromeVersionMatch[3]}.${chromeVersionMatch[4]}`;
|
|
303
|
+
const fullVersion = binaryFullVersion || uaClientHints?.uaFullVersion || uaDerivedVersion;
|
|
304
|
+
if (/^\d+\.0\.0\.0$/.test(fullVersion)) {
|
|
305
|
+
log.warning('[AntiDetect] UA-CH full version is the UA-reduced MAJOR.0.0.0 shape - no real Chrome '
|
|
306
|
+
+ 'reports this in sec-ch-ua-full-version(-list). The binary version probe and the '
|
|
307
|
+
+ 'profile both failed to supply a build number.', { fullVersion, userAgent });
|
|
308
|
+
}
|
|
309
|
+
// No profile brands (useFingerprintDefaults path): synthesize the exact
|
|
310
|
+
// list a real Chrome of this major emits - derived GREASE token, real
|
|
311
|
+
// brands, Chromium's own ordering. A hardcoded literal here paired a
|
|
312
|
+
// fixed GREASE with a moving major, which is what Naver rejects (#131).
|
|
313
|
+
const brands = uaClientHints?.brands || canonicalBrandList(majorVersion);
|
|
314
|
+
const fullVersionList = uaClientHints?.fullVersionList || canonicalBrandList(fullVersion);
|
|
315
|
+
// Stable hash of the UA string → index into the candidate list.
|
|
316
|
+
// Same UA → same platformVersion across reloads/sessions; different UA
|
|
317
|
+
// (different fingerprint seed) → different version. The hash is memoized
|
|
318
|
+
// per UA string (see uaSeedHashCache).
|
|
319
|
+
const _uaHash = uaSeedHash(userAgent);
|
|
320
|
+
const resolvedPlatformVersion = normalizePlatformVersion(platform, uaClientHints?.platformVersion, _uaHash);
|
|
244
321
|
try {
|
|
245
322
|
// NOTE: do NOT pass the top-level `platform` field. Chrome maps it
|
|
246
323
|
// to `Settings::NavigatorPlatformOverride`, which then overrides
|
|
@@ -248,19 +325,31 @@ export class CdpFingerprintInjector {
|
|
|
248
325
|
// (third_party/blink/renderer/core/frame/navigator.cc). The value
|
|
249
326
|
// sent here is UA-CH form ("macOS"/"Windows"/"Linux"), but the
|
|
250
327
|
// LEGACY `navigator.platform` must be "MacIntel"/"Win32"/
|
|
251
|
-
// "Linux x86_64"
|
|
328
|
+
// "Linux x86_64" - passing the UA-CH string leaks a Chrome-
|
|
252
329
|
// invariant violation that CreepJS flags immediately.
|
|
253
330
|
// The UA-CH value belongs only in userAgentMetadata.platform.
|
|
254
331
|
await this.page.client.Network.setUserAgentOverride({
|
|
255
332
|
userAgent,
|
|
333
|
+
// `fullVersionList` is missing from the pinned
|
|
334
|
+
// devtools-protocol typings but has been part of
|
|
335
|
+
// Emulation.UserAgentMetadata since Chrome 100 (it superseded
|
|
336
|
+
// `fullVersion`). Sending it explicitly is what makes
|
|
337
|
+
// `sec-ch-ua-full-version-list` correct: given only
|
|
338
|
+
// `fullVersion`, Chrome pairs EVERY brand with it - including
|
|
339
|
+
// GREASE, which then carries the real build instead of its
|
|
340
|
+
// padded "<v>.0.0.0" (issue #131). We send both so an older
|
|
341
|
+
// browser still gets the legacy field.
|
|
256
342
|
userAgentMetadata: {
|
|
257
343
|
brands: brands.map(b => ({ brand: b.brand, version: b.version })),
|
|
344
|
+
fullVersionList: fullVersionList.map(b => ({ brand: b.brand, version: b.version })),
|
|
258
345
|
fullVersion,
|
|
259
346
|
platform: platformName,
|
|
260
|
-
platformVersion:
|
|
261
|
-
architecture: uaClientHints?.architecture ||
|
|
347
|
+
platformVersion: resolvedPlatformVersion,
|
|
348
|
+
architecture: uaClientHints?.architecture || UA_ARCH_MAP[platform] || 'x86',
|
|
262
349
|
model: uaClientHints?.model || '',
|
|
263
350
|
mobile: false,
|
|
351
|
+
// The cast covers `fullVersionList`, absent from the pinned
|
|
352
|
+
// devtools-protocol typings (see the note above).
|
|
264
353
|
},
|
|
265
354
|
});
|
|
266
355
|
log.debug('Set User-Agent with full client hints metadata');
|
|
@@ -272,18 +361,29 @@ export class CdpFingerprintInjector {
|
|
|
272
361
|
}
|
|
273
362
|
}
|
|
274
363
|
async getTimezoneFromIp(proxyUrl) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
364
|
+
// Cache per proxy session for process lifetime: concurrent pages on the
|
|
365
|
+
// same session share the single in-flight fetch, and later pages reuse
|
|
366
|
+
// the resolved value. A failed lookup evicts its entry so a later page
|
|
367
|
+
// can retry instead of pinning a wrong timezone for the whole session.
|
|
368
|
+
const key = proxyUrl ?? NO_PROXY_TIMEZONE_KEY;
|
|
369
|
+
let pending = timezoneByProxy.get(key);
|
|
370
|
+
if (!pending) {
|
|
371
|
+
pending = this.fetchTimezoneFromIp(proxyUrl).catch((error) => {
|
|
372
|
+
timezoneByProxy.delete(key);
|
|
373
|
+
log.error('Failed to get timezone from IP:', { error });
|
|
374
|
+
return 'UTC';
|
|
280
375
|
});
|
|
281
|
-
|
|
282
|
-
}
|
|
283
|
-
catch (error) {
|
|
284
|
-
log.error('Failed to get timezone from IP:', { error });
|
|
285
|
-
return 'UTC';
|
|
376
|
+
timezoneByProxy.set(key, pending);
|
|
286
377
|
}
|
|
378
|
+
return pending;
|
|
379
|
+
}
|
|
380
|
+
async fetchTimezoneFromIp(proxyUrl) {
|
|
381
|
+
const response = await gotScraping({
|
|
382
|
+
url: 'https://ipinfo.io/json',
|
|
383
|
+
proxyUrl,
|
|
384
|
+
responseType: 'json',
|
|
385
|
+
});
|
|
386
|
+
return response.body.timezone;
|
|
287
387
|
}
|
|
288
388
|
async setTimezoneOverride(timezone) {
|
|
289
389
|
await this.page.client.Emulation.setTimezoneOverride({
|
|
@@ -293,7 +393,7 @@ export class CdpFingerprintInjector {
|
|
|
293
393
|
/**
|
|
294
394
|
* Override ICU's default locale for the page via CDP. This makes Intl.DateTimeFormat,
|
|
295
395
|
* Intl.NumberFormat, and Date.toLocaleString default to the supplied locale regardless
|
|
296
|
-
* of the browser binary's --lang setting
|
|
396
|
+
* of the browser binary's --lang setting - our reliable knob for the stock-Chrome path
|
|
297
397
|
* where we don't control the startup args.
|
|
298
398
|
*/
|
|
299
399
|
async setLocaleOverride(locale) {
|
|
@@ -308,561 +408,160 @@ export class CdpFingerprintInjector {
|
|
|
308
408
|
// into the single injection to avoid multiple addScriptToEvaluateOnNewDocument calls
|
|
309
409
|
// which is a detection vector.
|
|
310
410
|
/**
|
|
311
|
-
*
|
|
411
|
+
* Identify the browser process behind this page's CDP connection. The
|
|
412
|
+
* remote-debugging host:port is unique per Chrome process, so it keys the
|
|
413
|
+
* per-browser-process version cache without bleeding across processes.
|
|
312
414
|
*/
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
// CDP Fingerprint Override Debug Logs
|
|
317
|
-
const DEBUG_PREFIX = '[CDP-FP-DEBUG]';
|
|
318
|
-
|
|
319
|
-
console.log(DEBUG_PREFIX, '=== Fingerprint Override Status ===');
|
|
320
|
-
console.log(DEBUG_PREFIX, 'Timestamp:', new Date().toISOString());
|
|
321
|
-
|
|
322
|
-
// Check navigator properties
|
|
323
|
-
console.log(DEBUG_PREFIX, 'Navigator Checks:');
|
|
324
|
-
console.log(DEBUG_PREFIX, ' - platform:', navigator.platform);
|
|
325
|
-
console.log(DEBUG_PREFIX, ' - userAgent:', navigator.userAgent.substring(0, 80) + '...');
|
|
326
|
-
console.log(DEBUG_PREFIX, ' - webdriver:', navigator.webdriver);
|
|
327
|
-
console.log(DEBUG_PREFIX, ' - hardwareConcurrency:', navigator.hardwareConcurrency);
|
|
328
|
-
console.log(DEBUG_PREFIX, ' - deviceMemory:', navigator.deviceMemory);
|
|
329
|
-
console.log(DEBUG_PREFIX, ' - language:', navigator.language);
|
|
330
|
-
console.log(DEBUG_PREFIX, ' - languages:', navigator.languages);
|
|
331
|
-
console.log(DEBUG_PREFIX, ' - maxTouchPoints:', navigator.maxTouchPoints);
|
|
332
|
-
console.log(DEBUG_PREFIX, ' - pdfViewerEnabled:', navigator.pdfViewerEnabled);
|
|
333
|
-
console.log(DEBUG_PREFIX, ' - plugins.length:', navigator.plugins?.length);
|
|
334
|
-
|
|
335
|
-
// Check screen properties
|
|
336
|
-
console.log(DEBUG_PREFIX, 'Screen Checks:');
|
|
337
|
-
console.log(DEBUG_PREFIX, ' - colorDepth:', screen.colorDepth);
|
|
338
|
-
console.log(DEBUG_PREFIX, ' - pixelDepth:', screen.pixelDepth);
|
|
339
|
-
console.log(DEBUG_PREFIX, ' - availWidth:', screen.availWidth);
|
|
340
|
-
console.log(DEBUG_PREFIX, ' - availHeight:', screen.availHeight);
|
|
341
|
-
console.log(DEBUG_PREFIX, ' - devicePixelRatio:', window.devicePixelRatio);
|
|
342
|
-
|
|
343
|
-
// Check window properties
|
|
344
|
-
console.log(DEBUG_PREFIX, 'Window Checks:');
|
|
345
|
-
console.log(DEBUG_PREFIX, ' - outerWidth:', window.outerWidth);
|
|
346
|
-
console.log(DEBUG_PREFIX, ' - outerHeight:', window.outerHeight);
|
|
347
|
-
console.log(DEBUG_PREFIX, ' - screenX:', window.screenX);
|
|
348
|
-
console.log(DEBUG_PREFIX, ' - screenY:', window.screenY);
|
|
349
|
-
|
|
350
|
-
// Check chrome object
|
|
351
|
-
console.log(DEBUG_PREFIX, 'Chrome Object:');
|
|
352
|
-
console.log(DEBUG_PREFIX, ' - chrome exists:', typeof window.chrome !== 'undefined');
|
|
353
|
-
console.log(DEBUG_PREFIX, ' - chrome.runtime exists:', typeof window.chrome?.runtime !== 'undefined');
|
|
354
|
-
|
|
355
|
-
// Check WebGL
|
|
356
|
-
try {
|
|
357
|
-
const canvas = document.createElement('canvas');
|
|
358
|
-
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
|
|
359
|
-
if (gl) {
|
|
360
|
-
const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
|
|
361
|
-
if (debugInfo) {
|
|
362
|
-
console.log(DEBUG_PREFIX, 'WebGL Checks:');
|
|
363
|
-
console.log(DEBUG_PREFIX, ' - vendor:', gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL));
|
|
364
|
-
console.log(DEBUG_PREFIX, ' - renderer:', gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL));
|
|
365
|
-
} else {
|
|
366
|
-
console.log(DEBUG_PREFIX, 'WebGL debug extension not available');
|
|
367
|
-
}
|
|
368
|
-
} else {
|
|
369
|
-
console.log(DEBUG_PREFIX, 'WebGL context not available');
|
|
370
|
-
}
|
|
371
|
-
} catch (e) {
|
|
372
|
-
console.log(DEBUG_PREFIX, 'WebGL check failed:', e.message);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// Check automation markers
|
|
376
|
-
console.log(DEBUG_PREFIX, 'Automation Marker Checks:');
|
|
377
|
-
console.log(DEBUG_PREFIX, ' - $cdc_ properties:', Object.keys(window).filter(k => k.startsWith('$cdc_') || k.startsWith('cdc_')).length === 0 ? 'CLEAN' : 'DETECTED');
|
|
378
|
-
console.log(DEBUG_PREFIX, ' - webdriver properties:', Object.keys(window).filter(k => k.toLowerCase().includes('webdriver')).length === 0 ? 'CLEAN' : 'DETECTED');
|
|
379
|
-
|
|
380
|
-
// Check permissions API
|
|
381
|
-
if (navigator.permissions) {
|
|
382
|
-
navigator.permissions.query({ name: 'notifications' }).then(result => {
|
|
383
|
-
console.log(DEBUG_PREFIX, 'Permissions:');
|
|
384
|
-
console.log(DEBUG_PREFIX, ' - notifications:', result.state);
|
|
385
|
-
}).catch(e => {
|
|
386
|
-
console.log(DEBUG_PREFIX, 'Permissions check failed:', e.message);
|
|
387
|
-
});
|
|
415
|
+
browserEndpointKey() {
|
|
416
|
+
const client = this.page.client;
|
|
417
|
+
return `${client?.host ?? 'localhost'}:${client?.port ?? ''}`;
|
|
388
418
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
419
|
+
async getActualChromeVersion() {
|
|
420
|
+
// Probe Browser.getVersion once per browser process. All of a browser's
|
|
421
|
+
// pages/targets share the same endpoint key, so the version is fetched
|
|
422
|
+
// once and reused; distinct browser processes stay independent. A failed
|
|
423
|
+
// probe evicts its entry so a later page can retry.
|
|
424
|
+
const key = this.browserEndpointKey();
|
|
425
|
+
let pending = chromeVersionByEndpoint.get(key);
|
|
426
|
+
if (!pending) {
|
|
427
|
+
pending = this.probeActualChromeVersion().catch(() => {
|
|
428
|
+
chromeVersionByEndpoint.delete(key);
|
|
429
|
+
return null;
|
|
430
|
+
});
|
|
431
|
+
chromeVersionByEndpoint.set(key, pending);
|
|
432
|
+
}
|
|
433
|
+
return pending;
|
|
399
434
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
console.log(DEBUG_PREFIX, ' - bypassCDPDetection:', ${options.bypassCDPDetection !== false});
|
|
406
|
-
console.log(DEBUG_PREFIX, ' - spoofStorage:', ${options.spoofStorage !== false});
|
|
407
|
-
console.log(DEBUG_PREFIX, ' - spoofTiming:', ${options.spoofTiming !== false});
|
|
408
|
-
console.log(DEBUG_PREFIX, ' - enableDataDomeBypass:', ${options.enableDataDomeBypass || false});
|
|
409
|
-
console.log(DEBUG_PREFIX, ' - humanization.mouse:', ${options.humanization?.mouse || false});
|
|
410
|
-
console.log(DEBUG_PREFIX, ' - humanization.keyboard:', ${options.humanization?.keyboard || false});
|
|
411
|
-
console.log(DEBUG_PREFIX, ' - humanization.scroll:', ${options.humanization?.scroll || false});
|
|
412
|
-
|
|
413
|
-
console.log(DEBUG_PREFIX, '=== End Fingerprint Override Status ===');
|
|
414
|
-
})();
|
|
415
|
-
`;
|
|
435
|
+
async probeActualChromeVersion() {
|
|
436
|
+
const versionInfo = await this.page.client.Browser.getVersion();
|
|
437
|
+
const product = versionInfo.product || '';
|
|
438
|
+
const match = product.match(/(?:Chrome|HeadlessChrome)\/(\d+\.\d+\.\d+\.\d+)/);
|
|
439
|
+
return match ? match[1] : null;
|
|
416
440
|
}
|
|
417
441
|
/**
|
|
418
|
-
*
|
|
442
|
+
* Mutate the shared fingerprint's navigator UA surfaces (userAgent, appVersion,
|
|
443
|
+
* userAgentData brands, and the headers UA) to the real binary version, in
|
|
444
|
+
* place. This is the SAME object Crawlee/BrowserPool's built-in
|
|
445
|
+
* FingerprintInjector reads in its postPageCreate hook, so aligning it here
|
|
446
|
+
* keeps navigator.userAgent, the Sec-CH-UA brands, and the wire User-Agent
|
|
447
|
+
* header all on the real binary version instead of the generator's (#104).
|
|
448
|
+
* No-op when the version couldn't be probed or the profile has no UA.
|
|
419
449
|
*/
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
platformConfig.oscpu = fingerprint.navigator.oscpu;
|
|
442
|
-
platformConfig.userAgent = fingerprint.navigator.userAgent;
|
|
443
|
-
platformConfig.appVersion = fingerprint.navigator.appVersion;
|
|
444
|
-
platformConfig.appName = fingerprint.navigator.appName;
|
|
445
|
-
platformConfig.appCodeName = fingerprint.navigator.appCodeName;
|
|
446
|
-
platformConfig.product = fingerprint.navigator.product;
|
|
447
|
-
// Extract UA-CH data from generated fingerprint
|
|
448
|
-
// This ensures navigator.userAgentData brands/versions match the UA string
|
|
449
|
-
if (fingerprint.navigator.userAgentData) {
|
|
450
|
-
const uad = fingerprint.navigator.userAgentData;
|
|
451
|
-
uaClientHints = {
|
|
452
|
-
brands: uad.brands,
|
|
453
|
-
fullVersionList: uad.fullVersionList,
|
|
454
|
-
mobile: uad.mobile,
|
|
455
|
-
platform: uad.platform,
|
|
456
|
-
architecture: uad.architecture,
|
|
457
|
-
bitness: uad.bitness,
|
|
458
|
-
model: uad.model,
|
|
459
|
-
platformVersion: uad.platformVersion,
|
|
460
|
-
uaFullVersion: uad.uaFullVersion,
|
|
461
|
-
wow64: uad.wow64,
|
|
462
|
-
};
|
|
450
|
+
alignFingerprintUserAgent(fp, actualChromeVersion) {
|
|
451
|
+
if (!actualChromeVersion)
|
|
452
|
+
return;
|
|
453
|
+
const navigator = fp?.fingerprint?.navigator;
|
|
454
|
+
if (!navigator?.userAgent)
|
|
455
|
+
return;
|
|
456
|
+
navigator.userAgent = alignUserAgentWithBrowserVersion(navigator.userAgent, actualChromeVersion);
|
|
457
|
+
if (navigator.appVersion) {
|
|
458
|
+
navigator.appVersion = alignUserAgentWithBrowserVersion(navigator.appVersion, actualChromeVersion);
|
|
459
|
+
}
|
|
460
|
+
if (navigator.userAgentData) {
|
|
461
|
+
navigator.userAgentData = alignUAClientHintsVersion(navigator.userAgentData, actualChromeVersion);
|
|
462
|
+
}
|
|
463
|
+
// Keep the generator headers coherent too, so anything that replays them
|
|
464
|
+
// (Crawlee's hook, header injection) carries the aligned UA.
|
|
465
|
+
const headers = fp?.headers;
|
|
466
|
+
if (headers) {
|
|
467
|
+
for (const key of Object.keys(headers)) {
|
|
468
|
+
if (key.toLowerCase() === 'user-agent') {
|
|
469
|
+
headers[key] = alignUserAgentWithBrowserVersion(headers[key], actualChromeVersion);
|
|
470
|
+
}
|
|
463
471
|
}
|
|
464
472
|
}
|
|
465
|
-
if (fingerprint.screen) {
|
|
466
|
-
platformConfig.devicePixelRatio = fingerprint.screen.devicePixelRatio;
|
|
467
|
-
platformConfig.screenDepth = fingerprint.screen.pixelDepth;
|
|
468
|
-
platformConfig.colorDepth = fingerprint.screen.colorDepth;
|
|
469
|
-
}
|
|
470
|
-
return { webglConfig, platformConfig, uaClientHints };
|
|
471
|
-
}
|
|
472
|
-
async getActualChromeVersion() {
|
|
473
|
-
try {
|
|
474
|
-
const versionInfo = await this.page.client.Browser.getVersion();
|
|
475
|
-
const product = versionInfo.product || '';
|
|
476
|
-
const match = product.match(/(?:Chrome|HeadlessChrome)\/(\d+\.\d+\.\d+\.\d+)/);
|
|
477
|
-
return match ? match[1] : null;
|
|
478
|
-
}
|
|
479
|
-
catch {
|
|
480
|
-
return null;
|
|
481
|
-
}
|
|
482
473
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
if (
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
const frozenVersion = `${majorVersion}.0.0.0`;
|
|
497
|
-
if (uaMatch[1] === frozenVersion) {
|
|
498
|
-
return userAgent;
|
|
474
|
+
async attachToPage(browserFingerprintWithHeaders, options) {
|
|
475
|
+
// Native-stealth fast-path: the plan is built WITHOUT any CDP probe.
|
|
476
|
+
// Issuing CDP calls here (Browser.getVersion, Emulation.* overrides,
|
|
477
|
+
// Page.enable) would introduce exactly the artifacts the C++
|
|
478
|
+
// source-tree edits are designed to hide.
|
|
479
|
+
if (isNativeStealth(options)) {
|
|
480
|
+
const plan = buildInjectionPlan({
|
|
481
|
+
browserFingerprintWithHeaders,
|
|
482
|
+
options,
|
|
483
|
+
actualChromeVersion: null,
|
|
484
|
+
});
|
|
485
|
+
await this.applyPlan(plan, browserFingerprintWithHeaders, options);
|
|
486
|
+
return;
|
|
499
487
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
488
|
+
// Legacy timezone path still needs the ipinfo.io lookup; the resolved
|
|
489
|
+
// triple (when present) wins inside the planner.
|
|
490
|
+
const ipTimezone = needsIpTimezoneLookup(options)
|
|
491
|
+
? await this.getTimezoneFromIp(options?.proxyUrl)
|
|
492
|
+
: undefined;
|
|
493
|
+
const actualChromeVersion = await this.getActualChromeVersion();
|
|
494
|
+
// Align the SHARED fingerprint's navigator UA fields to the real binary
|
|
495
|
+
// version, in place, before building the plan. Crawlee/BrowserPool's own
|
|
496
|
+
// FingerprintInjector runs its postPageCreate hook AFTER this injector and
|
|
497
|
+
// writes navigator.userAgent from this very fingerprint object - with the
|
|
498
|
+
// raw generator version (e.g. Chrome/144) while our wire header + this
|
|
499
|
+
// page are aligned to the real binary (e.g. Chrome/150). That leaves a
|
|
500
|
+
// navigator-vs-header (and vs Browser.getVersion) mismatch - an instant
|
|
501
|
+
// detection signal (issue #104). Aligning the shared object here makes
|
|
502
|
+
// Crawlee's later injection emit the same aligned version everywhere.
|
|
503
|
+
this.alignFingerprintUserAgent(browserFingerprintWithHeaders, actualChromeVersion);
|
|
504
|
+
const plan = buildInjectionPlan({
|
|
505
|
+
browserFingerprintWithHeaders,
|
|
506
|
+
options,
|
|
507
|
+
actualChromeVersion,
|
|
508
|
+
ipTimezone,
|
|
505
509
|
});
|
|
506
|
-
|
|
510
|
+
await this.applyPlan(plan, browserFingerprintWithHeaders, options);
|
|
507
511
|
}
|
|
508
512
|
/**
|
|
509
|
-
*
|
|
510
|
-
*
|
|
511
|
-
* preserving the "Not_A Brand" GREASE token from the profile as-is.
|
|
513
|
+
* Apply an InjectionPlan to the page. All CDP traffic for fingerprint
|
|
514
|
+
* injection happens here; every decision was made in the planner.
|
|
512
515
|
*/
|
|
513
|
-
|
|
514
|
-
const
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
return {
|
|
518
|
-
...uaClientHints,
|
|
519
|
-
brands: uaClientHints.brands ? alignBrands(uaClientHints.brands) : uaClientHints.brands,
|
|
520
|
-
fullVersionList: uaClientHints.fullVersionList
|
|
521
|
-
? alignFullList(uaClientHints.fullVersionList)
|
|
522
|
-
: uaClientHints.fullVersionList,
|
|
523
|
-
uaFullVersion: fullVersion,
|
|
524
|
-
};
|
|
525
|
-
}
|
|
526
|
-
async attachToPage(browserFingerprintWithHeaders, options) {
|
|
527
|
-
const { fingerprint, headers } = browserFingerprintWithHeaders;
|
|
528
|
-
// Locale/timezone resolution priority:
|
|
529
|
-
// 1. Resolved triple from browser-plugin (user locale → proxy country → fingerprint)
|
|
530
|
-
// 2. Legacy options.timezone / options.detectTimezone → ipinfo.io lookup
|
|
531
|
-
// Resolved values always win so fingerprintOptions.locale: 'ko-KR' coherently
|
|
532
|
-
// propagates to every locale surface.
|
|
533
|
-
const resolvedTimezone = options?.resolvedLocale?.timezone;
|
|
534
|
-
const resolvedLocale = options?.resolvedLocale?.locale;
|
|
535
|
-
const shouldFetchTimezone = !resolvedTimezone && (options?.timezone || options?.detectTimezone);
|
|
536
|
-
const timezone = resolvedTimezone
|
|
537
|
-
?? (shouldFetchTimezone ? await this.getTimezoneFromIp(options?.proxyUrl) : undefined);
|
|
538
|
-
if (timezone) {
|
|
539
|
-
await this.setTimezoneOverride(timezone);
|
|
540
|
-
}
|
|
541
|
-
if (resolvedLocale) {
|
|
542
|
-
await this.setLocaleOverride(resolvedLocale);
|
|
543
|
-
}
|
|
544
|
-
// NOTE: Page.enable() is required for addScriptToEvaluateOnNewDocument to work
|
|
545
|
-
// We enable it lazily just before script injection to minimize detection surface
|
|
546
|
-
await this.page.ensurePageEnabled();
|
|
547
|
-
// Filter headers to only include those that can be injected via CDP
|
|
548
|
-
const filteredHeaders = {};
|
|
549
|
-
const allowedHeaders = ['accept', 'accept-encoding', 'accept-language', 'user-agent', 'referer', 'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform', 'sec-ch-ua-full-version-list'];
|
|
550
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
551
|
-
if (allowedHeaders.includes(key.toLowerCase()) && typeof value === 'string') {
|
|
552
|
-
filteredHeaders[key] = value;
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
// Resolved Accept-Language wins over whatever the fingerprint-generator headers had
|
|
556
|
-
// so that fingerprintOptions.locale: 'ko-KR' reaches the wire.
|
|
557
|
-
if (options?.resolvedLocale?.languages) {
|
|
558
|
-
filteredHeaders['accept-language'] = options.resolvedLocale.languages;
|
|
559
|
-
delete filteredHeaders['Accept-Language'];
|
|
560
|
-
}
|
|
561
|
-
const originalUserAgent = fingerprint?.navigator?.userAgent
|
|
562
|
-
|| filteredHeaders['user-agent']
|
|
563
|
-
|| filteredHeaders['User-Agent']
|
|
564
|
-
|| '';
|
|
565
|
-
const actualChromeVersion = await this.getActualChromeVersion();
|
|
566
|
-
const userAgent = this.alignUserAgentWithBrowserVersion(originalUserAgent, actualChromeVersion);
|
|
567
|
-
if (userAgent) {
|
|
568
|
-
filteredHeaders['user-agent'] = userAgent;
|
|
569
|
-
delete filteredHeaders['User-Agent'];
|
|
570
|
-
}
|
|
571
|
-
// Create custom fingerprint injection scripts
|
|
572
|
-
log.info('Creating custom fingerprint injection scripts');
|
|
573
|
-
const customFingerprintScripts = createCustomFingerprintInjection(fingerprint, {
|
|
574
|
-
debug: options?.debug !== false,
|
|
575
|
-
throwOnError: true,
|
|
576
|
-
});
|
|
577
|
-
// Prepare all scripts to inject - stealth utils and runtime bypass MUST be first
|
|
578
|
-
// These scripts set up protections BEFORE page JavaScript runs
|
|
579
|
-
// Check if DataDome bypass is enabled (via either option name)
|
|
580
|
-
const isDataDomeBypassEnabled = options?.enableDataDomeBypass || options?.bypassDatadome;
|
|
581
|
-
const shouldMaskAutomation = options?.maskAutomationFlags !== false;
|
|
582
|
-
// When the C++ anti-detect browser is in use, every fingerprint surface
|
|
583
|
-
// is handled by patches in /Users/petr/cromite-src/src/third_party/blink/:
|
|
584
|
-
// navigator (webdriver/platform/UA/UA-CH/plugins/deviceMemory/hwConc)
|
|
585
|
-
// screen / outerWidth / devicePixelRatio
|
|
586
|
-
// WebGL / WebGPU / canvas noise / audio noise / rect noise / measureText
|
|
587
|
-
// timezone / language / accept-language HTTP header / battery
|
|
588
|
-
// performance.now quantization / Runtime.enable bypass / CDP artifacts
|
|
589
|
-
// chrome.runtime stub / inspector sourceURL
|
|
590
|
-
//
|
|
591
|
-
// CDP-side overrides (Network.setUserAgentOverride, Emulation.setDeviceMetrics,
|
|
592
|
-
// Emulation.setTimezone, addScriptToEvaluateOnNewDocument) are strictly
|
|
593
|
-
// incompatible with native stealth: they either duplicate what the C++
|
|
594
|
-
// patches already do (double-override → inconsistency) or they introduce
|
|
595
|
-
// CDP artifacts the C++ patches are specifically trying to hide.
|
|
596
|
-
// → skip the ENTIRE injection pipeline. No JS overrides, no CDP overrides,
|
|
597
|
-
// no humanization — native C++ is the single source of truth.
|
|
598
|
-
const nativeStealth = options?.useNativeStealth === true;
|
|
599
|
-
if (nativeStealth) {
|
|
600
|
-
log.info('[AntiDetect] useNativeStealth=true: skipping ALL JS + CDP overrides '
|
|
516
|
+
async applyPlan(plan, browserFingerprintWithHeaders, options) {
|
|
517
|
+
const { fingerprint } = browserFingerprintWithHeaders;
|
|
518
|
+
if (plan.kind === 'native') {
|
|
519
|
+
log.debug('[AntiDetect] useNativeStealth=true: skipping ALL JS + CDP overrides '
|
|
601
520
|
+ '(viewport, UA, headers, timezone, mouse/keyboard/scroll humanization, '
|
|
602
521
|
+ 'event prototype patches). C++ patches are the single source of truth: '
|
|
603
522
|
+ '0040 (PointerEvent.pressure NaN→0.5), 0039 (getCoalescedEvents non-empty), '
|
|
604
523
|
+ 'and the upstream Chromium ConvertWidgetPointToScreenPoint / TimeClamper '
|
|
605
524
|
+ 'paths cover screenX/Y and event.timeStamp.');
|
|
606
|
-
// Exception:
|
|
607
|
-
//
|
|
608
|
-
//
|
|
609
|
-
//
|
|
610
|
-
//
|
|
611
|
-
|
|
612
|
-
if (options?.resolvedLocale?.languages) {
|
|
525
|
+
// Exception: Accept-Language header. Chrome's profile default is
|
|
526
|
+
// `en-US,en;q=0.9` even when `--lang=ko-KR` is set; native-stealth
|
|
527
|
+
// binaries predating the accept-language header patch leak that
|
|
528
|
+
// default. A single HTTP header is not a fingerprintable CDP
|
|
529
|
+
// artifact (unlike Runtime.enable or inspector traces).
|
|
530
|
+
if (plan.acceptLanguage) {
|
|
613
531
|
await this.page.ensureNetworkEnabled();
|
|
614
532
|
await this.page.client.Network.setExtraHTTPHeaders({
|
|
615
|
-
headers: { 'Accept-Language':
|
|
533
|
+
headers: { 'Accept-Language': plan.acceptLanguage },
|
|
616
534
|
});
|
|
617
535
|
}
|
|
618
536
|
return;
|
|
619
537
|
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
createPrototypeIntegrityScript(), // Second - provides native-like property descriptor masking
|
|
623
|
-
];
|
|
624
|
-
if (shouldMaskAutomation && !nativeStealth) {
|
|
625
|
-
// C++ patch 0005 strips CDP artifacts natively — skip JS bypass when native stealth active
|
|
626
|
-
scriptsToInject.push(createRuntimeEnableBypassScript()); // Protects against CDP detection via Error stack analysis
|
|
538
|
+
if (plan.timezone) {
|
|
539
|
+
await this.setTimezoneOverride(plan.timezone);
|
|
627
540
|
}
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
if (isDataDomeBypassEnabled) {
|
|
631
|
-
scriptsToInject.push(createDataDomeBypassScript());
|
|
632
|
-
log.info('DataDome bypass enabled - injected early in script chain');
|
|
633
|
-
}
|
|
634
|
-
// Add custom fingerprint scripts
|
|
635
|
-
scriptsToInject.push(...customFingerprintScripts);
|
|
636
|
-
// Bundle stealth protections into the main script instead of separate injection
|
|
637
|
-
// Having multiple addScriptToEvaluateOnNewDocument calls is a detection vector
|
|
638
|
-
if (shouldMaskAutomation) {
|
|
639
|
-
scriptsToInject.push(createStealthScript());
|
|
640
|
-
}
|
|
641
|
-
// Extract fingerprint data if we should use it
|
|
642
|
-
// When useFingerprintDefaults is true (or undefined), use hardcoded defaults
|
|
643
|
-
// When useFingerprintDefaults is false, use the generated fingerprint data
|
|
644
|
-
const shouldUseFingerprintData = options?.useFingerprintDefaults === false;
|
|
645
|
-
const { webglConfig, platformConfig, uaClientHints: rawUaClientHints } = shouldUseFingerprintData
|
|
646
|
-
? this.extractFingerprintData(fingerprint)
|
|
647
|
-
: { webglConfig: undefined, platformConfig: undefined, uaClientHints: undefined };
|
|
648
|
-
// Align UA-CH brand/full-version fields with the actual running binary so the
|
|
649
|
-
// fingerprint profile version (132-135) doesn't contradict the real binary (147+).
|
|
650
|
-
const uaClientHints = rawUaClientHints && actualChromeVersion
|
|
651
|
-
? this.alignUAClientHintsVersion(rawUaClientHints, actualChromeVersion)
|
|
652
|
-
: rawUaClientHints;
|
|
653
|
-
// Add advanced features based on options
|
|
654
|
-
if (options?.enableAdvancedStealth !== false) {
|
|
655
|
-
// UA Client Hints spoofing — skipped when C++ patch 0003 handles it natively
|
|
656
|
-
if (!nativeStealth) {
|
|
657
|
-
scriptsToInject.push(createUAClientHintsSpoofingScript(options?.platform, shouldUseFingerprintData ? uaClientHints : undefined));
|
|
658
|
-
}
|
|
659
|
-
// WebGL spoofing — skipped when C++ patch 0004 handles it natively
|
|
660
|
-
if (!nativeStealth && options?.spoofWebGL !== false) {
|
|
661
|
-
scriptsToInject.push(createWebGLSpoofingScript(shouldUseFingerprintData ? webglConfig : undefined));
|
|
662
|
-
}
|
|
663
|
-
// Audio fingerprint spoofing — skipped when C++ patch 0010 handles it natively
|
|
664
|
-
if (!nativeStealth && options?.spoofAudioContext !== false) {
|
|
665
|
-
scriptsToInject.push(createAudioFingerprintSpoofingScript());
|
|
666
|
-
}
|
|
667
|
-
// Canvas protection — skipped when C++ patch 0009 handles it natively
|
|
668
|
-
if (!nativeStealth) {
|
|
669
|
-
scriptsToInject.push(createAdvancedCanvasProtectionScript());
|
|
670
|
-
}
|
|
671
|
-
// Client rect spoofing — skipped when C++ patch 0011 handles it natively
|
|
672
|
-
if (!nativeStealth && options?.spoofClientRects !== false) {
|
|
673
|
-
scriptsToInject.push(createClientRectSpoofingScript());
|
|
674
|
-
}
|
|
675
|
-
// (Coalesced PointerEvents are now handled in C++ — see Cromite anti-detect
|
|
676
|
-
// edit in third_party/blink/renderer/core/events/pointer_event.cc::getCoalescedEvents.
|
|
677
|
-
// The previous JS spoof in coalesced-events-spoofing.ts has been removed because
|
|
678
|
-
// a prototype getter on PointerEvent.prototype is detectable; the C++ fix is invisible.)
|
|
679
|
-
// Media codec spoofing — always keep (canPlayType behavioral)
|
|
680
|
-
scriptsToInject.push(createMediaCodecsSpoofingScript());
|
|
681
|
-
// WebRTC handling policy — always keep (fake device IDs; C++ handles IP only)
|
|
682
|
-
if (options?.webRtcPolicy === 'disable') {
|
|
683
|
-
scriptsToInject.push(createWebRTCDisableScript());
|
|
684
|
-
}
|
|
685
|
-
else {
|
|
686
|
-
scriptsToInject.push(createWebRTCSpoofingScript());
|
|
687
|
-
}
|
|
688
|
-
// WebGPU spoofing — skipped when C++ patch 0012 handles it natively
|
|
689
|
-
if (!nativeStealth) {
|
|
690
|
-
scriptsToInject.push(createWebGPUSpoofingScript(options?.useExperimentalFingerprints));
|
|
691
|
-
}
|
|
692
|
-
// Font spoofing — skipped when Cromite built-in allowlist handles it
|
|
693
|
-
if (!nativeStealth && options?.spoofFonts !== false) {
|
|
694
|
-
scriptsToInject.push(createFontSpoofingScript());
|
|
695
|
-
}
|
|
696
|
-
// Performance spoofing — skipped when Cromite built-in perf.now clamping handles it
|
|
697
|
-
if (!nativeStealth && options?.spoofPerformance !== false) {
|
|
698
|
-
scriptsToInject.push(createPerformanceSpoofingScript());
|
|
699
|
-
}
|
|
700
|
-
// Locale spoofing — always keep (Intl.NumberFormat formatting part not in C++).
|
|
701
|
-
// Bake the resolved locale/languages in as literals so the script reports values
|
|
702
|
-
// that match the navigator/header overrides instead of hardcoded en-US.
|
|
703
|
-
if (options?.spoofLocale !== false) {
|
|
704
|
-
scriptsToInject.push(createLocaleSpoofingScript({
|
|
705
|
-
locale: options?.resolvedLocale?.locale ?? 'en-US',
|
|
706
|
-
languages: options?.resolvedLocale?.languages ?? 'en-US,en;q=0.9',
|
|
707
|
-
}));
|
|
708
|
-
}
|
|
709
|
-
// CDP Detection Bypass — skipped when C++ patches 0001+0005 handle it natively
|
|
710
|
-
if (!nativeStealth && options?.bypassCDPDetection !== false) {
|
|
711
|
-
scriptsToInject.push(createCDPDetectionBypassScript());
|
|
712
|
-
}
|
|
713
|
-
// Storage consistency — always keep (CDP-level quota spoofing)
|
|
714
|
-
if (options?.spoofStorage !== false) {
|
|
715
|
-
scriptsToInject.push(createStorageConsistencyScript());
|
|
716
|
-
}
|
|
717
|
-
// Timing consistency — always keep (Date.now alignment; C++ handles perf.now)
|
|
718
|
-
if (options?.spoofTiming !== false) {
|
|
719
|
-
scriptsToInject.push(createTimingConsistencyScript());
|
|
720
|
-
}
|
|
721
|
-
// DataDome-specific bypass - already injected early if enabled
|
|
722
|
-
// (see isDataDomeBypassEnabled check at the beginning of scriptsToInject)
|
|
723
|
-
// Mouse movement humanization (browser context) - disabled by default
|
|
724
|
-
// Bundled here so the bezier-curve utility counts as part of the single
|
|
725
|
-
// addScriptToEvaluateOnNewDocument call (multiple calls are detectable).
|
|
726
|
-
if (options?.humanization?.mouse || options?.humanizeInteractions) {
|
|
727
|
-
scriptsToInject.push(createMouseMovementHumanizationScript());
|
|
728
|
-
}
|
|
729
|
-
// Keyboard humanization (browser context) - disabled by default
|
|
730
|
-
if (options?.humanization?.keyboard) {
|
|
731
|
-
scriptsToInject.push(createKeyboardHumanizationScript());
|
|
732
|
-
}
|
|
733
|
-
// Scroll humanization (browser context) - disabled by default
|
|
734
|
-
if (options?.humanization?.scroll) {
|
|
735
|
-
scriptsToInject.push(createScrollHumanizationScript());
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
// Apply platform consistency script — skipped when C++ patches 0002+0006 handle it natively
|
|
739
|
-
if (!nativeStealth && options?.platform) {
|
|
740
|
-
// Build the config to pass: when using real fingerprint data, align its UA with
|
|
741
|
-
// the actual binary version; when using hardcoded defaults, inject a minimal
|
|
742
|
-
// version-aligned UA so we don't bake in the static Chrome/134.0.0.0 fallback.
|
|
743
|
-
let platformConfigForScript = shouldUseFingerprintData ? platformConfig : undefined;
|
|
744
|
-
if (platformConfigForScript?.userAgent && actualChromeVersion) {
|
|
745
|
-
// Align dynamic fingerprint UA with the actual running binary version.
|
|
746
|
-
platformConfigForScript = {
|
|
747
|
-
...platformConfigForScript,
|
|
748
|
-
userAgent: this.alignUserAgentWithBrowserVersion(platformConfigForScript.userAgent, actualChromeVersion),
|
|
749
|
-
appVersion: platformConfigForScript.appVersion
|
|
750
|
-
? this.alignUserAgentWithBrowserVersion(platformConfigForScript.appVersion, actualChromeVersion)
|
|
751
|
-
: platformConfigForScript.appVersion,
|
|
752
|
-
};
|
|
753
|
-
}
|
|
754
|
-
else if (!platformConfigForScript && actualChromeVersion && options.platform === 'Win32') {
|
|
755
|
-
// useFingerprintDefaults=true path: override only the UA/appVersion fields so the
|
|
756
|
-
// hardcoded Win32 defaults pick up the real Chrome version instead of Chrome/134.
|
|
757
|
-
const alignedUA = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${actualChromeVersion} Safari/537.36`;
|
|
758
|
-
platformConfigForScript = {
|
|
759
|
-
userAgent: alignedUA,
|
|
760
|
-
appVersion: `5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${actualChromeVersion} Safari/537.36`,
|
|
761
|
-
};
|
|
762
|
-
}
|
|
763
|
-
scriptsToInject.push(createPlatformConsistencyScript(options.platform, platformConfigForScript));
|
|
764
|
-
}
|
|
765
|
-
// CRITICAL: Clean scripts to remove CDP detection patterns
|
|
766
|
-
const isDebug = options?.debug === true;
|
|
767
|
-
const cleanScript = (script) => {
|
|
768
|
-
let cleaned = script
|
|
769
|
-
// Remove sourceURL patterns that reveal CDP injection
|
|
770
|
-
.replace(/\/\/# sourceURL=[^\n]*/g, '')
|
|
771
|
-
.replace(/\/\/@ sourceURL=[^\n]*/g, '')
|
|
772
|
-
// Remove sourceMappingURL patterns
|
|
773
|
-
.replace(/\/\/# sourceMappingURL=[^\n]*/g, '')
|
|
774
|
-
// Remove puppeteer/playwright markers
|
|
775
|
-
.replace(/__puppeteer_evaluation_script__/g, 'anonymous')
|
|
776
|
-
.replace(/__playwright_evaluation_script__/g, 'anonymous')
|
|
777
|
-
// Remove CDP-specific comments
|
|
778
|
-
.replace(/\/\*\s*__CDP_[A-Z_]+__\s*\*\//g, '');
|
|
779
|
-
// CRITICAL: Strip ALL console output from injected scripts in production.
|
|
780
|
-
// Anti-bot services hook console.log and detect debug prefixes like
|
|
781
|
-
// [CDP-FP-DEBUG], [DD-BYPASS], [RUNTIME-ENABLE-BYPASS], etc.
|
|
782
|
-
if (!isDebug) {
|
|
783
|
-
cleaned = cleaned
|
|
784
|
-
.replace(/^\s*console\.(log|warn|info|debug)\s*\([^;]*\);?\s*$/gm, '')
|
|
785
|
-
.replace(/^\s*const\s+(DEBUG_PREFIX|ERROR_PREFIX)\s*=\s*[^;]*;\s*$/gm, '');
|
|
786
|
-
}
|
|
787
|
-
return cleaned;
|
|
788
|
-
};
|
|
789
|
-
// Add debug scripts into the bundle only when explicitly enabled
|
|
790
|
-
if (isDebug) {
|
|
791
|
-
scriptsToInject.push(this.createDebugScript(options || {}));
|
|
792
|
-
}
|
|
793
|
-
// BUNDLE ALL SCRIPTS INTO ONE to minimize CDP interactions
|
|
794
|
-
log.info(`Bundling ${scriptsToInject.length} fingerprint scripts into single injection`);
|
|
795
|
-
// Extract proxy seed key for deterministic fingerprint seeding.
|
|
796
|
-
// Priority: session ID from Apify proxy URL > full proxy URL > null.
|
|
797
|
-
// Apify proxy URLs look like: http://groups-RESIDENTIAL,session-abc123:pw@proxy.apify.com:8000
|
|
798
|
-
let proxySeedKey = null;
|
|
799
|
-
if (options?.proxyUrl) {
|
|
800
|
-
try {
|
|
801
|
-
const proxyUsername = new URL(options.proxyUrl).username;
|
|
802
|
-
const sessionMatch = proxyUsername.match(/session-([^,&]+)/i);
|
|
803
|
-
proxySeedKey = sessionMatch ? sessionMatch[1] : options.proxyUrl;
|
|
804
|
-
}
|
|
805
|
-
catch {
|
|
806
|
-
proxySeedKey = options.proxyUrl;
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
// Preamble: create shared stealth store accessible by all module IIFEs via closure.
|
|
810
|
-
// seedKey: proxy session ID or proxy URL - used by _stableHash for deterministic fingerprints.
|
|
811
|
-
const preamble = `var __stealth = { debug: ${isDebug}, seedKey: ${proxySeedKey ? JSON.stringify(proxySeedKey) : 'null'} };`;
|
|
812
|
-
// Neutralize the CDP-injection stack-frame tell. V8 renders scripts
|
|
813
|
-
// registered via Page.addScriptToEvaluateOnNewDocument that carry no
|
|
814
|
-
// sourceURL as "<anonymous>" in Error.stack — a well-known detection
|
|
815
|
-
// vector (DataDome, CreepJS, fpscanner all check for it). Appending
|
|
816
|
-
// `//# sourceURL=about:blank` makes V8 render stack frames as if the
|
|
817
|
-
// script loaded from about:blank, which is indistinguishable from
|
|
818
|
-
// a normal bootstrap script on a fresh page.
|
|
819
|
-
//
|
|
820
|
-
// Note: the cleanScript pass above strips *existing* sourceURL comments
|
|
821
|
-
// in the per-module scripts (to avoid the comment naming a file path
|
|
822
|
-
// that reveals our repo layout). We then stamp exactly one sourceURL
|
|
823
|
-
// on the outer bundle so the final script V8 sees has a clean name.
|
|
824
|
-
const bundledScript = `${preamble}\n\n${scriptsToInject
|
|
825
|
-
.map(script => cleanScript(script))
|
|
826
|
-
.join('\n\n')}\n//# sourceURL=about:blank\n`;
|
|
827
|
-
// Validate bundled script for syntax errors
|
|
828
|
-
try {
|
|
829
|
-
// eslint-disable-next-line no-new-func, no-new
|
|
830
|
-
new Function(bundledScript);
|
|
831
|
-
log.info('✓ Bundled script passed syntax validation');
|
|
832
|
-
}
|
|
833
|
-
catch (error) {
|
|
834
|
-
log.error('✗ Bundled script has syntax error:', { error });
|
|
835
|
-
// Fall back to individual script validation to find the culprit
|
|
836
|
-
scriptsToInject.forEach((script, index) => {
|
|
837
|
-
try {
|
|
838
|
-
// eslint-disable-next-line no-new-func, no-new
|
|
839
|
-
new Function(script);
|
|
840
|
-
}
|
|
841
|
-
catch (e) {
|
|
842
|
-
const scriptPreview = script.substring(0, 300);
|
|
843
|
-
log.error(`✗ Error in script ${index + 1}:`, { error: e, scriptPreview });
|
|
844
|
-
}
|
|
845
|
-
});
|
|
846
|
-
throw new Error(`Bundled script has syntax error: ${error}`);
|
|
541
|
+
if (plan.locale) {
|
|
542
|
+
await this.setLocaleOverride(plan.locale);
|
|
847
543
|
}
|
|
544
|
+
// NOTE: Page.enable() is required for addScriptToEvaluateOnNewDocument to work
|
|
545
|
+
// We enable it lazily just before script injection to minimize detection surface
|
|
546
|
+
await this.page.ensurePageEnabled();
|
|
848
547
|
// Inject SINGLE bundled script - reduces CDP detection surface
|
|
849
548
|
try {
|
|
850
|
-
log.
|
|
549
|
+
log.debug('Injecting single bundled fingerprint script (minimizes CDP detection)');
|
|
851
550
|
await Promise.all([
|
|
852
|
-
this.setViewport(
|
|
853
|
-
this.setExtraHTTPHeaders(
|
|
551
|
+
this.setViewport(plan.viewport, fingerprint),
|
|
552
|
+
this.setExtraHTTPHeaders(plan.headers),
|
|
854
553
|
// Set User-Agent with full client hints metadata (Sec-CH-UA-* HTTP headers)
|
|
855
|
-
userAgent ? this.setUserAgentWithMetadata(userAgent,
|
|
554
|
+
plan.userAgent ? this.setUserAgentWithMetadata(plan.userAgent, plan.platform, plan.uaClientHints, plan.binaryFullVersion) : Promise.resolve(),
|
|
856
555
|
// Single script injection instead of many - reduces CDP detection surface
|
|
857
556
|
(async () => {
|
|
858
557
|
try {
|
|
859
558
|
await this.page.client.Page.addScriptToEvaluateOnNewDocument({
|
|
860
|
-
source: bundledScript,
|
|
559
|
+
source: plan.bundledScript,
|
|
861
560
|
});
|
|
862
|
-
log.
|
|
561
|
+
log.debug('✓ Bundled script injected successfully');
|
|
863
562
|
}
|
|
864
563
|
catch (error) {
|
|
865
|
-
const scriptPreview = bundledScript.substring(0, 200).split('\n')[0];
|
|
564
|
+
const scriptPreview = plan.bundledScript.substring(0, 200).split('\n')[0];
|
|
866
565
|
log.error('✗ Failed to inject bundled script:', {
|
|
867
566
|
error,
|
|
868
567
|
scriptPreview,
|
|
@@ -871,24 +570,19 @@ export class CdpFingerprintInjector {
|
|
|
871
570
|
}
|
|
872
571
|
})(),
|
|
873
572
|
]);
|
|
874
|
-
log.
|
|
573
|
+
log.debug('✓ Fingerprint injection complete (single bundled script)');
|
|
875
574
|
}
|
|
876
575
|
catch (error) {
|
|
877
576
|
log.error('✗ Failed to inject fingerprint scripts:', { error });
|
|
878
577
|
throw error; // Rethrow to make injection failures visible
|
|
879
578
|
}
|
|
880
|
-
// NOTE: Mouse/keyboard/scroll humanization browser-context scripts are
|
|
579
|
+
// NOTE: Mouse/keyboard/scroll humanization browser-context scripts are
|
|
881
580
|
// bundled into the single addScriptToEvaluateOnNewDocument call above.
|
|
882
581
|
// Multiple addScriptToEvaluateOnNewDocument registrations are themselves
|
|
883
|
-
// a bot-detection signal
|
|
884
|
-
|
|
885
|
-
// The bypass script is now included in the bundled stealth scripts injected via addScriptToEvaluateOnNewDocument
|
|
886
|
-
// This runs BEFORE page JavaScript, avoiding Runtime domain activation
|
|
887
|
-
const enableMouseHumanization = options?.humanization?.mouse || options?.humanizeInteractions;
|
|
888
|
-
// Debug and viewport debug scripts are now bundled into the main injection above
|
|
889
|
-
log.info('Enhanced fingerprint attached with settings:', {
|
|
582
|
+
// a bot-detection signal - keep the count at 1.
|
|
583
|
+
log.debug('Enhanced fingerprint attached with settings:', {
|
|
890
584
|
proxy: options?.proxyUrl,
|
|
891
|
-
timezone,
|
|
585
|
+
timezone: plan.timezone,
|
|
892
586
|
advancedStealth: options?.enableAdvancedStealth !== false,
|
|
893
587
|
bypassRuntimeEnable: options?.bypassRuntimeEnable,
|
|
894
588
|
platform: options?.platform,
|
|
@@ -900,7 +594,7 @@ export class CdpFingerprintInjector {
|
|
|
900
594
|
enableDataDomeBypass: options?.enableDataDomeBypass,
|
|
901
595
|
webRtcPolicy: options?.webRtcPolicy || 'spoof',
|
|
902
596
|
humanization: {
|
|
903
|
-
mouse:
|
|
597
|
+
mouse: Boolean(options?.humanization?.mouse || options?.humanizeInteractions),
|
|
904
598
|
keyboard: options?.humanization?.keyboard || false,
|
|
905
599
|
scroll: options?.humanization?.scroll || false,
|
|
906
600
|
},
|