crawlforge-mcp-server 6.6.2 → 6.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +18 -2
- package/README.md +20 -2
- package/package.json +4 -2
- package/server.js +37 -16
- package/src/constants/config.js +22 -0
- package/src/core/ResearchOrchestrator.js +76 -2
- package/src/core/StealthBrowserManager.js +1159 -247
- package/src/core/creatorMode.js +1 -1
- package/src/core/processing/BrowserProcessor.js +30 -4
- package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md +57 -9
- package/src/skills/agent-skills/crawlforge-stealth-browsing/references/engine-selection.md +64 -19
- package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md +12 -6
- package/src/tools/advanced/BrowserSessionTool.js +38 -3
- package/src/tools/advanced/ScrapeWithActionsTool.js +30 -2
- package/src/tools/scrape/escalation.js +6 -1
- package/src/tools/scrape/unifiedScrape.js +2 -1
- package/src/utils/challengeDetection.js +86 -3
- package/src/utils/firefoxPageErrorGuard.js +93 -0
- package/src/utils/stealthVerdict.js +36 -2
|
@@ -11,14 +11,34 @@
|
|
|
11
11
|
import { chromium } from 'playwright';
|
|
12
12
|
import { z } from 'zod';
|
|
13
13
|
import crypto from 'crypto';
|
|
14
|
+
import fs from 'fs';
|
|
15
|
+
import os from 'os';
|
|
16
|
+
import path from 'path';
|
|
17
|
+
import { createRequire } from 'module';
|
|
14
18
|
import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
|
|
15
19
|
import { BrowserContextPool } from './BrowserContextPool.js';
|
|
16
20
|
import { safeGoto } from '../utils/ssrfGuard.js';
|
|
17
|
-
import {
|
|
21
|
+
import { looksLikeInterstitial } from '../utils/challengeDetection.js';
|
|
22
|
+
import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
|
|
23
|
+
import { serverStealthProxies } from '../constants/config.js';
|
|
18
24
|
|
|
19
25
|
// Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
|
|
20
26
|
export const EMPTY_DOCUMENT_GRACE_MS = 8000;
|
|
21
27
|
|
|
28
|
+
// The Chrome major claimed when the installed binary cannot be read at all.
|
|
29
|
+
// Only ever a floor: installedChromeVersion() prefers playwright-core's own
|
|
30
|
+
// browsers.json, and a launched browser's real version overrides both.
|
|
31
|
+
const FALLBACK_CHROME_MAJOR = 151;
|
|
32
|
+
|
|
33
|
+
// The proxy a camoufox browser was launched with, kept on the browser itself so
|
|
34
|
+
// it survives being parked and restored by an engine switch.
|
|
35
|
+
const CAMOUFOX_PROXY = Symbol('crawlforge.camoufoxProxy');
|
|
36
|
+
|
|
37
|
+
// The locale that camoufox browser was launched with — null when geoip derived
|
|
38
|
+
// one from the proxy's exit IP and we therefore do not know it. Kept on the
|
|
39
|
+
// browser for the same reason as the proxy: camoufox fixes it at launch.
|
|
40
|
+
const CAMOUFOX_LOCALE = Symbol('crawlforge.camoufoxLocale');
|
|
41
|
+
|
|
22
42
|
const StealthConfigSchema = z.object({
|
|
23
43
|
level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
|
|
24
44
|
randomizeFingerprint: z.boolean().default(true),
|
|
@@ -61,8 +81,11 @@ const StealthConfigSchema = z.object({
|
|
|
61
81
|
hardwareSpoofing: z.boolean().default(true)
|
|
62
82
|
}).optional(),
|
|
63
83
|
|
|
64
|
-
// C2: browser engine selection — '
|
|
65
|
-
|
|
84
|
+
// C2: browser engine selection — 'auto' (default), 'chromium' or 'camoufox'
|
|
85
|
+
// (Firefox-based). 'auto' is resolved to one of the concrete two by
|
|
86
|
+
// resolveStealthEngine before anything reads this field; see it for why the
|
|
87
|
+
// default moved off chromium.
|
|
88
|
+
engine: z.enum(['chromium', 'camoufox', 'auto']).optional().default('auto')
|
|
66
89
|
});
|
|
67
90
|
|
|
68
91
|
/**
|
|
@@ -84,9 +107,73 @@ function isChromium(page) {
|
|
|
84
107
|
}
|
|
85
108
|
}
|
|
86
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Resolve a requested engine to the one that will actually launch.
|
|
112
|
+
*
|
|
113
|
+
* Camoufox is the engine that gets past the walls a stealth call is made for —
|
|
114
|
+
* it spoofs at the C++/Juggler level, where a page cannot see the seam — so
|
|
115
|
+
* 'auto', now the default everywhere, asks for it and drops to Chromium only
|
|
116
|
+
* when the package is not there. That is the semantic ResearchOrchestrator has
|
|
117
|
+
* run with since v4.6.6 (RESEARCH_STEALTH_ENGINE='auto'); it is lifted here so
|
|
118
|
+
* every stealth path shares one definition of it.
|
|
119
|
+
*
|
|
120
|
+
* A caller who NAMED an engine gets that engine and nothing else: 'camoufox'
|
|
121
|
+
* is never downgraded, so a missing install still fails at launch with the
|
|
122
|
+
* install instructions rather than quietly running a browser the caller
|
|
123
|
+
* rejected — the failure mode this whole review exists to remove.
|
|
124
|
+
*
|
|
125
|
+
* @param {'auto'|'chromium'|'camoufox'|'playwright'|null|undefined} requested
|
|
126
|
+
* 'playwright' is the tool layer's public name for chromium; null/undefined
|
|
127
|
+
* mean 'auto'.
|
|
128
|
+
* @returns {Promise<{engine: 'chromium'|'camoufox', fallbackWarning: string|null}>}
|
|
129
|
+
* fallbackWarning is set only when 'auto' wanted camoufox and could not have
|
|
130
|
+
* it, and is the one thing that stops that downgrade from being silent.
|
|
131
|
+
*/
|
|
132
|
+
export async function resolveStealthEngine(requested) {
|
|
133
|
+
if (requested === 'camoufox') return { engine: 'camoufox', fallbackWarning: null };
|
|
134
|
+
if (requested && requested !== 'auto') return { engine: 'chromium', fallbackWarning: null };
|
|
135
|
+
|
|
136
|
+
// Deployment-level answer for 'auto', because there is no correct global one.
|
|
137
|
+
// Which engine clears a wall depends on the exit IP: the 2026-09-21
|
|
138
|
+
// residential run had Camoufox clearing indeed.com where Chromium failed, and
|
|
139
|
+
// two hosted runs from a datacenter address found the exact reverse, with
|
|
140
|
+
// Camoufox winning no wall Chromium lost. An operator who knows which side of
|
|
141
|
+
// that their egress sits on can say so here; a caller naming an engine still
|
|
142
|
+
// overrides it. Anything unrecognised is ignored rather than fatal — a typo in
|
|
143
|
+
// an env var should not take stealth down.
|
|
144
|
+
const configured = String(process.env.CRAWLFORGE_STEALTH_ENGINE || '').trim().toLowerCase();
|
|
145
|
+
if (configured === 'chromium' || configured === 'playwright') {
|
|
146
|
+
return { engine: 'chromium', fallbackWarning: null };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
let reason = 'camoufox is not installed (npm install camoufox)';
|
|
150
|
+
try {
|
|
151
|
+
if (await new CamoufoxAdapter().isAvailable()) {
|
|
152
|
+
return { engine: 'camoufox', fallbackWarning: null };
|
|
153
|
+
}
|
|
154
|
+
} catch (error) {
|
|
155
|
+
// Installed-but-broken arrives as a throw, and it is a different problem
|
|
156
|
+
// from not installed — a fetched binary that will not load is worth
|
|
157
|
+
// naming. Either way it is a reason to run Chromium, not to fail a scrape
|
|
158
|
+
// the caller never asked to pin to one engine.
|
|
159
|
+
reason = error.message;
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
engine: 'chromium',
|
|
163
|
+
fallbackWarning:
|
|
164
|
+
`Stealth engine fell back to chromium: ${reason}. ` +
|
|
165
|
+
'Camoufox passes bot walls this Chromium does not; install it to use it.'
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
87
169
|
export class StealthBrowserManager {
|
|
88
170
|
constructor(options = {}) {
|
|
89
171
|
this.browser = null;
|
|
172
|
+
// Why the running browser is not the engine 'auto' asked for, or null when
|
|
173
|
+
// nothing was downgraded. Kept on the instance beside _launchedEngine so a
|
|
174
|
+
// tool layer can tell the caller which browser actually ran — a drop to
|
|
175
|
+
// Chromium that nobody reports reads as a camoufox result that failed.
|
|
176
|
+
this._engineFallbackWarning = null;
|
|
90
177
|
this._maxContexts = parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10);
|
|
91
178
|
this.contexts = this._createContextPool();
|
|
92
179
|
// D2.2: fingerprints Map is capped at _maxContexts to prevent unbounded growth.
|
|
@@ -149,25 +236,19 @@ export class StealthBrowserManager {
|
|
|
149
236
|
// UA thirty majors behind the engine — the pool sat at 119–121 while
|
|
150
237
|
// Chrome 152 shipped — is a staleness tell for anything that reads
|
|
151
238
|
// sec-ch-ua (R15, 2026-09-04).
|
|
239
|
+
//
|
|
240
|
+
// The Chrome major is now read from the installed binary rather than
|
|
241
|
+
// listed. A pool of four majors on one binary meant three of four personas
|
|
242
|
+
// mis-stated the version navigator.userAgentData reports — the UA claimed
|
|
243
|
+
// Chrome 149 on a 151 Chromium (2026-09-21 benchmark, ua-version-vs-binary).
|
|
244
|
+
// One string per OS is not lost entropy: Chrome's UA has been frozen since
|
|
245
|
+
// the UA-reduction rollout, so minor/build/patch are always 0.0.0 and the OS
|
|
246
|
+
// token is fixed, and that one string is the only one a real Chrome of that
|
|
247
|
+
// major sends. Variation lives in the display, fonts, hardware and persona.
|
|
248
|
+
this.chromeVersion = StealthBrowserManager.installedChromeVersion() || `${FALLBACK_CHROME_MAJOR}.0.0.0`;
|
|
249
|
+
this.chromeMajor = StealthBrowserManager.majorVersion(this.chromeVersion) || FALLBACK_CHROME_MAJOR;
|
|
152
250
|
this.userAgentPools = {
|
|
153
|
-
chrome:
|
|
154
|
-
windows: [
|
|
155
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
|
|
156
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
|
|
157
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
|
|
158
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
|
|
159
|
-
],
|
|
160
|
-
macos: [
|
|
161
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
|
|
162
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
|
|
163
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
|
|
164
|
-
],
|
|
165
|
-
linux: [
|
|
166
|
-
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
|
|
167
|
-
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
|
|
168
|
-
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
|
|
169
|
-
]
|
|
170
|
-
},
|
|
251
|
+
chrome: this.buildChromeUserAgents(this.chromeMajor),
|
|
171
252
|
firefox: {
|
|
172
253
|
windows: [
|
|
173
254
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0'
|
|
@@ -188,13 +269,9 @@ export class StealthBrowserManager {
|
|
|
188
269
|
}
|
|
189
270
|
};
|
|
190
271
|
|
|
191
|
-
//
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
macos: 0.15,
|
|
195
|
-
linux: 0.10
|
|
196
|
-
};
|
|
197
|
-
|
|
272
|
+
// No OS distribution any more: the persona's OS is the host's (see
|
|
273
|
+
// selectOS), not a draw from market share.
|
|
274
|
+
|
|
198
275
|
// Browser market share for realistic selection
|
|
199
276
|
this.browserDistribution = {
|
|
200
277
|
chrome: 0.65,
|
|
@@ -287,6 +364,12 @@ export class StealthBrowserManager {
|
|
|
287
364
|
*/
|
|
288
365
|
async launchStealthBrowser(config = {}) {
|
|
289
366
|
const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
|
|
367
|
+
// 'auto' has to become a concrete engine before the comparison below: an
|
|
368
|
+
// unresolved 'auto' never equals _launchedEngine, so every call would park
|
|
369
|
+
// a perfectly good browser and launch a second one beside it. This is the
|
|
370
|
+
// call that decides the engine — every other path reaches a browser
|
|
371
|
+
// through it — so it is the one that records the outcome.
|
|
372
|
+
await this._resolveConfigEngine(validatedConfig, true);
|
|
290
373
|
|
|
291
374
|
// A Chromium that was OOM-killed or crashed doesn't error on reuse — its
|
|
292
375
|
// protocol calls hang. Detect the corpse and relaunch instead.
|
|
@@ -328,11 +411,49 @@ export class StealthBrowserManager {
|
|
|
328
411
|
}
|
|
329
412
|
}
|
|
330
413
|
|
|
414
|
+
/**
|
|
415
|
+
* Rewrite config.engine in place to the engine that will actually launch,
|
|
416
|
+
* recording any downgrade on the instance.
|
|
417
|
+
*
|
|
418
|
+
* In place, because `engine` is read again further down — the park/reuse
|
|
419
|
+
* comparison, the fingerprint's browser pool, the camoufox context surgery —
|
|
420
|
+
* and every one of them has to see the same answer. Idempotent and cheap (a
|
|
421
|
+
* concrete engine resolves to itself; isAvailable()'s require is cached by
|
|
422
|
+
* the module loader), so the context path and the launch path can each call
|
|
423
|
+
* it without arranging who goes first.
|
|
424
|
+
*
|
|
425
|
+
* @param {{engine: string}} validatedConfig — mutated
|
|
426
|
+
* @param {boolean} [ownsTheDecision] — true for the call this launch's engine
|
|
427
|
+
* is decided by (launchStealthBrowser). _doLaunchStealthBrowser re-checks
|
|
428
|
+
* the very config that call already resolved, and recording "nothing was
|
|
429
|
+
* downgraded" there would erase the downgrade the first pass had just
|
|
430
|
+
* found — which is how the fallback would go silent again.
|
|
431
|
+
* @returns {Promise<string|null>} this call's fallback warning, if any
|
|
432
|
+
*/
|
|
433
|
+
async _resolveConfigEngine(validatedConfig, ownsTheDecision = false) {
|
|
434
|
+
// A concrete engine is already the answer and cannot have fallen back.
|
|
435
|
+
if (validatedConfig.engine && validatedConfig.engine !== 'auto') {
|
|
436
|
+
if (ownsTheDecision) this._engineFallbackWarning = null;
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const { engine, fallbackWarning } = await resolveStealthEngine(validatedConfig.engine);
|
|
441
|
+
validatedConfig.engine = engine;
|
|
442
|
+
this._engineFallbackWarning = fallbackWarning;
|
|
443
|
+
return fallbackWarning;
|
|
444
|
+
}
|
|
445
|
+
|
|
331
446
|
/**
|
|
332
447
|
* Actual browser launch, guarded by launchStealthBrowser's in-flight
|
|
333
448
|
* promise so only one launch can be in progress at a time.
|
|
334
449
|
*/
|
|
335
450
|
async _doLaunchStealthBrowser(validatedConfig) {
|
|
451
|
+
// Resolved again here rather than trusted: this method is reachable on its
|
|
452
|
+
// own, and an 'auto' that got this far unresolved would fall through to the
|
|
453
|
+
// chromium branch below — the exact silent downgrade 'auto' is meant to
|
|
454
|
+
// report.
|
|
455
|
+
await this._resolveConfigEngine(validatedConfig);
|
|
456
|
+
|
|
336
457
|
// C2: delegate to CamoufoxAdapter when engine === 'camoufox'
|
|
337
458
|
if (validatedConfig.engine === 'camoufox') {
|
|
338
459
|
const adapter = new CamoufoxAdapter();
|
|
@@ -342,10 +463,69 @@ export class StealthBrowserManager {
|
|
|
342
463
|
'camoufox is not installed. Run: npm install camoufox to use the Firefox-based stealth engine.'
|
|
343
464
|
);
|
|
344
465
|
}
|
|
345
|
-
|
|
346
|
-
|
|
466
|
+
// camoufox fixes its fingerprint — and, with geoip, its geolocation,
|
|
467
|
+
// timezone and locale — at launch, from the proxy it is launched with.
|
|
468
|
+
// So the proxy is resolved once here and reused for every context this
|
|
469
|
+
// browser serves (see createStealthContext): rotating underneath it would
|
|
470
|
+
// leave camoufox reporting the first proxy's city behind the second
|
|
471
|
+
// proxy's exit IP, which is a worse signal than not rotating at all.
|
|
472
|
+
// A rotation takes effect on the next launch, after cleanup().
|
|
473
|
+
const proxy = this.resolveProxy(validatedConfig);
|
|
474
|
+
// The caller's locale goes to the launcher, not to the context. camoufox
|
|
475
|
+
// sets language, Accept-Language and Intl together below the JS layer,
|
|
476
|
+
// where a Worker reads the same answer as the document; Playwright's
|
|
477
|
+
// Firefox context override reaches the document only, so asking for
|
|
478
|
+
// de-DE there produced ["de-DE"] in the page beside camoufox's own list
|
|
479
|
+
// in its worker (2026-09-21 benchmark, worker-languages). Fixed at
|
|
480
|
+
// launch, like the proxy: a later context asking for a different locale
|
|
481
|
+
// inherits the launched one until cleanup(). With a proxy, geoip derives
|
|
482
|
+
// the locale from the exit IP and must win — a persona from this call
|
|
483
|
+
// would name a country the address contradicts.
|
|
484
|
+
const locale = proxy ? null : validatedConfig.locale;
|
|
485
|
+
const browser = await adapter.launch({
|
|
486
|
+
// 'virtual' runs a real, windowed Firefox inside Xvfb instead of the
|
|
487
|
+
// headless build. Headless Firefox is its own tell — no window manager,
|
|
488
|
+
// no compositor, and a set of media/GL answers that differ from the
|
|
489
|
+
// browser everyone else runs — and camoufox ships this mode precisely
|
|
490
|
+
// to avoid it. Only on Linux: that is where a hosted image has no
|
|
491
|
+
// display and where the Dockerfile provides Xvfb; a Mac or Windows
|
|
492
|
+
// developer machine has neither, so plain headless stays there.
|
|
493
|
+
headless: process.platform === 'linux' ? 'virtual' : true,
|
|
494
|
+
proxy,
|
|
495
|
+
locale,
|
|
496
|
+
// camoufox draws its persona from ["windows","macos","linux"] when it
|
|
497
|
+
// is told nothing, so it claimed Windows on a Mac while still reporting
|
|
498
|
+
// an Apple M1 GPU and a `-apple-system: Mac` CSS platform hint
|
|
499
|
+
// (2026-09-21 benchmark, persona-os-vs-host). The host's OS is the only
|
|
500
|
+
// one the GPU strings, the font list and the TCP/IP fingerprint agree
|
|
501
|
+
// with, so it is the one camoufox is given.
|
|
502
|
+
//
|
|
503
|
+
// KNOWN NOT TO TAKE EFFECT on camoufox npm 0.1.19, measured 2026-09-21:
|
|
504
|
+
// ten launches asking for "macos" on a Mac produced a Mac persona about
|
|
505
|
+
// one time in three, which is the market-share draw, not the answer to a
|
|
506
|
+
// constraint. The client forwards `{ screen, os }` to
|
|
507
|
+
// fingerprint-generator (chunk-QNWJYPXY.js `generateFingerprint`), whose
|
|
508
|
+
// option for this is `operatingSystems: string[]` — `os` is not a key it
|
|
509
|
+
// knows, so it is dropped and the OS is drawn at random. The call below
|
|
510
|
+
// is the documented API and is kept: it is inert (camoufox only
|
|
511
|
+
// validates the value), it is correct the moment upstream honours it,
|
|
512
|
+
// and the alternative — generating a BrowserForge fingerprint ourselves
|
|
513
|
+
// and passing it through `fingerprint` — needs a dependency this package
|
|
514
|
+
// does not declare. Phase 2 of the stealth review owns the camoufox
|
|
515
|
+
// client; until then persona-os-vs-host stays baselined for camoufox.
|
|
516
|
+
os: this.hostOS(),
|
|
517
|
+
// Only ask for geoip when there is a proxy to derive it from. Without
|
|
518
|
+
// one it would look up this machine's own public IP — a network call,
|
|
519
|
+
// and an external service learning our address, to confirm a location
|
|
520
|
+
// the browser is already in.
|
|
521
|
+
geoip: !!proxy,
|
|
522
|
+
blockWebRTC: validatedConfig.blockWebRTC,
|
|
523
|
+
humanize: validatedConfig.simulateHumanBehavior,
|
|
347
524
|
launchOptions: {}
|
|
348
525
|
});
|
|
526
|
+
browser[CAMOUFOX_PROXY] = proxy;
|
|
527
|
+
browser[CAMOUFOX_LOCALE] = locale;
|
|
528
|
+
this.browser = browser;
|
|
349
529
|
this._launchedEngine = 'camoufox';
|
|
350
530
|
return this.browser;
|
|
351
531
|
}
|
|
@@ -356,9 +536,7 @@ export class StealthBrowserManager {
|
|
|
356
536
|
'--no-sandbox',
|
|
357
537
|
'--disable-dev-shm-usage',
|
|
358
538
|
'--disable-blink-features=AutomationControlled',
|
|
359
|
-
'--disable-web-security',
|
|
360
539
|
'--disable-features=VizDisplayCompositor',
|
|
361
|
-
'--disable-extensions',
|
|
362
540
|
'--disable-plugins',
|
|
363
541
|
'--disable-background-timer-throttling',
|
|
364
542
|
'--disable-backgrounding-occluded-windows',
|
|
@@ -374,12 +552,18 @@ export class StealthBrowserManager {
|
|
|
374
552
|
'--safebrowsing-disable-auto-update',
|
|
375
553
|
'--password-store=basic',
|
|
376
554
|
'--use-mock-keychain',
|
|
377
|
-
|
|
555
|
+
|
|
556
|
+
// A headless Chromium has no pointing device, so it reports
|
|
557
|
+
// primaryPointerType=coarse/none and hover:none — the media-query answer
|
|
558
|
+
// of a touchscreen kiosk under a desktop UA. These four blink settings
|
|
559
|
+
// (2 = HOVER_HOVER_TYPE, 4 = POINTER_FINE_TYPE) give it the mouse a
|
|
560
|
+
// desktop persona is supposed to have. Verified present in Chromium 151
|
|
561
|
+
// with `strings -a` on the binary.
|
|
562
|
+
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
|
|
563
|
+
|
|
378
564
|
// Additional stealth arguments
|
|
379
|
-
'--disable-default-apps',
|
|
380
565
|
'--disable-component-extensions-with-background-pages',
|
|
381
566
|
'--disable-background-networking',
|
|
382
|
-
'--disable-component-update',
|
|
383
567
|
'--disable-client-side-phishing-detection',
|
|
384
568
|
'--disable-domain-reliability',
|
|
385
569
|
'--disable-ipc-flooding-protection',
|
|
@@ -392,9 +576,7 @@ export class StealthBrowserManager {
|
|
|
392
576
|
if (validatedConfig.level === 'advanced') {
|
|
393
577
|
stealthArgs.push(
|
|
394
578
|
'--disable-gpu-sandbox',
|
|
395
|
-
'--disable-popup-blocking',
|
|
396
579
|
'--disable-setuid-sandbox',
|
|
397
|
-
'--disable-site-isolation-trials',
|
|
398
580
|
'--disable-threaded-animation',
|
|
399
581
|
'--disable-threaded-scrolling',
|
|
400
582
|
'--disable-in-process-stack-traces',
|
|
@@ -413,22 +595,22 @@ export class StealthBrowserManager {
|
|
|
413
595
|
);
|
|
414
596
|
}
|
|
415
597
|
|
|
416
|
-
// WebRTC blocking
|
|
598
|
+
// WebRTC blocking. The five flags that used to stand here disabled
|
|
599
|
+
// hardware codecs and extra routes — none of them stops an ICE candidate
|
|
600
|
+
// carrying the real address, which is what CreepJS read the host's IPv6
|
|
601
|
+
// out of (2026-09-21 benchmark, finding 9). This one is the switch that
|
|
602
|
+
// governs which local addresses WebRTC may use: with it, only the address
|
|
603
|
+
// the proxy already exposes is offered. Verified present in Chromium 151
|
|
604
|
+
// with `strings -a` on the binary.
|
|
417
605
|
if (validatedConfig.blockWebRTC) {
|
|
418
|
-
stealthArgs.push(
|
|
419
|
-
'--disable-webrtc-hw-decoding',
|
|
420
|
-
'--disable-webrtc-hw-encoding',
|
|
421
|
-
'--disable-webrtc-multiple-routes',
|
|
422
|
-
'--disable-webrtc-hw-vp8-encoding',
|
|
423
|
-
'--enforce-webrtc-ip-permission-check'
|
|
424
|
-
);
|
|
606
|
+
stealthArgs.push('--webrtc-ip-handling-policy=disable_non_proxied_udp');
|
|
425
607
|
}
|
|
426
608
|
|
|
427
|
-
//
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
609
|
+
// No proxy argument here. Chromium's --proxy-server= has no field for the
|
|
610
|
+
// user:pass every residential proxy requires, and a proxy fixed at launch
|
|
611
|
+
// could never rotate: the browser it was baked into is cached for the life
|
|
612
|
+
// of the process. Both engines take the proxy per context instead — see
|
|
613
|
+
// createStealthContext.
|
|
432
614
|
|
|
433
615
|
const browser = await chromium.launch({
|
|
434
616
|
headless: true,
|
|
@@ -436,12 +618,27 @@ export class StealthBrowserManager {
|
|
|
436
618
|
// never reads it) — see Dockerfile.
|
|
437
619
|
executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
|
|
438
620
|
args: stealthArgs,
|
|
621
|
+
// patchright's "marks you as a stealth driver" list. Playwright passes
|
|
622
|
+
// all four by default, so dropping them from stealthArgs above is only
|
|
623
|
+
// half the job — they have to be taken off the default set too, or the
|
|
624
|
+
// command line still carries them. --disable-web-security went with them:
|
|
625
|
+
// it is readable in one line from any page (a cross-origin fetch that
|
|
626
|
+
// should throw and does not), and it contradicts this file's own decision
|
|
627
|
+
// to leave bypassCSP unset (R15, 2026-09-04).
|
|
439
628
|
ignoreDefaultArgs: [
|
|
440
629
|
'--enable-blink-features=IdleDetection',
|
|
441
|
-
'--enable-automation'
|
|
630
|
+
'--enable-automation',
|
|
631
|
+
'--disable-component-update',
|
|
632
|
+
'--disable-default-apps',
|
|
633
|
+
'--disable-extensions',
|
|
634
|
+
'--disable-popup-blocking'
|
|
442
635
|
]
|
|
443
636
|
});
|
|
444
637
|
|
|
638
|
+
// A hosted image's system Chromium may not be the version browsers.json
|
|
639
|
+
// named, and the UA has to state the version that is actually running.
|
|
640
|
+
this._alignUserAgentsWithBinary(browser);
|
|
641
|
+
|
|
445
642
|
// If this Chromium dies (OOM kill, crash), drop the handle so the next
|
|
446
643
|
// call relaunches instead of reusing a corpse. The identity guard keeps a
|
|
447
644
|
// late event from an old instance from nulling a newer one.
|
|
@@ -460,12 +657,18 @@ export class StealthBrowserManager {
|
|
|
460
657
|
*/
|
|
461
658
|
async createStealthContext(config = {}) {
|
|
462
659
|
const validatedConfig = StealthConfigSchema.parse({ ...this.defaultConfig, ...config });
|
|
463
|
-
|
|
660
|
+
|
|
464
661
|
// Always go through launchStealthBrowser: it returns the running browser
|
|
465
662
|
// when the engine matches, and closes + relaunches on an engine mismatch.
|
|
466
663
|
// Guarding on `!this.browser` here skipped that mismatch check, so a
|
|
467
664
|
// camoufox request silently reused an already-running chromium browser.
|
|
468
665
|
await this.launchStealthBrowser(validatedConfig);
|
|
666
|
+
// The browser that came back is the authority on which engine this context
|
|
667
|
+
// is created over, and everything below reads it: the fingerprint's browser
|
|
668
|
+
// pool branches on it, and an 'auto' left unresolved here would draw a
|
|
669
|
+
// Chrome persona for a Firefox browser.
|
|
670
|
+
validatedConfig.engine = this._launchedEngine ?? validatedConfig.engine;
|
|
671
|
+
const engineFallbackWarning = this._engineFallbackWarning;
|
|
469
672
|
|
|
470
673
|
// Generate fingerprint for this context
|
|
471
674
|
const fingerprint = this.generateAdvancedFingerprint(validatedConfig);
|
|
@@ -507,19 +710,79 @@ export class StealthBrowserManager {
|
|
|
507
710
|
serviceWorkers: 'block'
|
|
508
711
|
};
|
|
509
712
|
|
|
510
|
-
//
|
|
511
|
-
//
|
|
512
|
-
//
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
//
|
|
713
|
+
// The proxy rides on the context, credentials included: Chromium takes it
|
|
714
|
+
// on Target.createBrowserContext and camoufox's Juggler on
|
|
715
|
+
// Browser.setContextProxy. Both were verified against a local authenticating
|
|
716
|
+
// proxy. camoufox additionally gets one at launch, because its geoip lookup
|
|
717
|
+
// runs there — but a launch-time proxy routes traffic and drops the
|
|
718
|
+
// credentials, so the context is what actually authenticates, on both
|
|
719
|
+
// engines. camoufox stays on the proxy it was launched with: its geolocation,
|
|
720
|
+
// timezone and locale were derived from that exit IP, and rotating
|
|
721
|
+
// underneath it would leave the first proxy's city behind the second
|
|
722
|
+
// proxy's address.
|
|
723
|
+
const proxy = this._launchedEngine === 'camoufox'
|
|
724
|
+
? (this.browser[CAMOUFOX_PROXY] || null)
|
|
725
|
+
: this.resolveProxy(validatedConfig);
|
|
726
|
+
if (proxy) {
|
|
727
|
+
contextOptions.proxy = proxy;
|
|
728
|
+
}
|
|
729
|
+
|
|
516
730
|
if (this._launchedEngine === 'camoufox') {
|
|
731
|
+
// camoufox's Firefox build predates the Browser.setDefaultViewport fields
|
|
732
|
+
// playwright-core 1.62 sends (screenSize, isMobile, ...) and rejects
|
|
733
|
+
// unknown properties, so any fixed viewport fails. viewport:null skips
|
|
734
|
+
// that protocol call entirely (deviceScaleFactor/isMobile/hasTouch/screen
|
|
735
|
+
// are invalid or meaningless without a viewport). camoufox generates its
|
|
736
|
+
// own screen and window and spoofs them below the JS layer.
|
|
517
737
|
contextOptions.viewport = null;
|
|
518
738
|
delete contextOptions.deviceScaleFactor;
|
|
519
739
|
delete contextOptions.isMobile;
|
|
520
740
|
delete contextOptions.hasTouch;
|
|
521
741
|
delete contextOptions.screen;
|
|
522
742
|
delete contextOptions.serviceWorkers;
|
|
743
|
+
|
|
744
|
+
// camoufox arrives with a complete Firefox identity of its own. Ours was
|
|
745
|
+
// overwriting it, and not with a Firefox one: for this engine the browser
|
|
746
|
+
// distribution is left open, so about two thirds of camoufox contexts were
|
|
747
|
+
// handed a Chrome User-Agent on a Gecko engine. Every one of them also
|
|
748
|
+
// sent sec-ch-ua, sec-ch-ua-mobile and sec-ch-ua-platform — client hints
|
|
749
|
+
// Firefox has never implemented and never sends. That is a decision a
|
|
750
|
+
// detector can make from the request headers alone, before a line of
|
|
751
|
+
// script runs.
|
|
752
|
+
delete contextOptions.userAgent;
|
|
753
|
+
delete contextOptions.extraHTTPHeaders;
|
|
754
|
+
|
|
755
|
+
// The locale goes the same way, proxy or no proxy — but it is not
|
|
756
|
+
// dropped: _doLaunchStealthBrowser hands the caller's locale to camoufox
|
|
757
|
+
// itself, which sets language, Accept-Language and Intl together below
|
|
758
|
+
// the JS layer. Playwright's Firefox context override reaches the
|
|
759
|
+
// document only, so setting it here reported ["de-DE"] in the page beside
|
|
760
|
+
// camoufox's own list in its worker (2026-09-21 benchmark,
|
|
761
|
+
// worker-languages).
|
|
762
|
+
delete contextOptions.locale;
|
|
763
|
+
|
|
764
|
+
// Behind a proxy, camoufox has already derived timezone and geolocation
|
|
765
|
+
// from the exit IP. Those agree with the address the site sees; a persona
|
|
766
|
+
// drawn here does not, so it must not override them.
|
|
767
|
+
if (proxy) {
|
|
768
|
+
delete contextOptions.timezoneId;
|
|
769
|
+
delete contextOptions.geolocation;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// Keep what we hand back honest. create_context returns this fingerprint,
|
|
773
|
+
// and reporting a Chrome user agent and a persona the browser never uses
|
|
774
|
+
// is worse than reporting nothing: null reads as "the engine owns this".
|
|
775
|
+
fingerprint.userAgent = null;
|
|
776
|
+
fingerprint.headers = {};
|
|
777
|
+
fingerprint.viewport = null;
|
|
778
|
+
fingerprint.hardware = { ...fingerprint.hardware, platform: null };
|
|
779
|
+
// The locale camoufox was actually launched with, which is this call's
|
|
780
|
+
// only when this call is the one that launched it — null behind a proxy,
|
|
781
|
+
// where geoip derived a locale we never see.
|
|
782
|
+
fingerprint.locale = this.browser[CAMOUFOX_LOCALE] ?? null;
|
|
783
|
+
if (proxy) {
|
|
784
|
+
fingerprint.timezone = null;
|
|
785
|
+
}
|
|
523
786
|
}
|
|
524
787
|
|
|
525
788
|
const context = await this.browser.newContext(contextOptions);
|
|
@@ -532,7 +795,11 @@ export class StealthBrowserManager {
|
|
|
532
795
|
// D2.2: enforce LRU cap on fingerprints Map
|
|
533
796
|
this._setFingerprint(contextId, fingerprint);
|
|
534
797
|
|
|
535
|
-
|
|
798
|
+
// The engine and its warning ride back with the context, not only on the
|
|
799
|
+
// instance: the instance fields follow the browser, which the next call
|
|
800
|
+
// may switch, and the caller of THIS context still has to be able to say
|
|
801
|
+
// which browser it got.
|
|
802
|
+
return { context, contextId, fingerprint, engine: validatedConfig.engine, engineFallbackWarning };
|
|
536
803
|
}
|
|
537
804
|
|
|
538
805
|
/**
|
|
@@ -590,8 +857,11 @@ export class StealthBrowserManager {
|
|
|
590
857
|
platform: fingerprint.hardware.platform,
|
|
591
858
|
locale: fingerprint.locale,
|
|
592
859
|
timezone: fingerprint.timezone,
|
|
593
|
-
// width/height only — the pool's selection weight is an internal.
|
|
594
|
-
|
|
860
|
+
// width/height only — the pool's selection weight is an internal. null on
|
|
861
|
+
// camoufox, which sizes its own window.
|
|
862
|
+
viewport: fingerprint.viewport
|
|
863
|
+
? { width: fingerprint.viewport.width, height: fingerprint.viewport.height }
|
|
864
|
+
: null
|
|
595
865
|
};
|
|
596
866
|
}
|
|
597
867
|
|
|
@@ -626,17 +896,33 @@ export class StealthBrowserManager {
|
|
|
626
896
|
|
|
627
897
|
/**
|
|
628
898
|
* Choose a single OS ('windows' | 'macos' | 'linux') for a fingerprint.
|
|
629
|
-
* A custom UA pins the OS to whatever that UA reports;
|
|
630
|
-
*
|
|
899
|
+
* A custom UA pins the OS to whatever that UA reports; every other path takes
|
|
900
|
+
* the host's OS.
|
|
901
|
+
*
|
|
902
|
+
* A persona drawn at random claimed Windows, then Linux, on a macOS host
|
|
903
|
+
* (2026-09-21 benchmark, persona-os-vs-host). Nothing about the machine
|
|
904
|
+
* follows the draw: the TCP/IP fingerprint is the host kernel's, the GPU
|
|
905
|
+
* strings and the CSS platform hints are the host's, and a WebGL renderer
|
|
906
|
+
* reading "Apple M1" under a Windows UA is a cleaner signal than no spoofing
|
|
907
|
+
* at all. So the OS is the one thing in this fingerprint that is not
|
|
908
|
+
* randomised — it is observed.
|
|
631
909
|
*/
|
|
632
910
|
selectOS(config = {}) {
|
|
633
911
|
if (config.customUserAgent) {
|
|
634
912
|
return this.inferOSFromUserAgent(config.customUserAgent);
|
|
635
913
|
}
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
914
|
+
return this.hostOS();
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* The host's own OS in the vocabulary the pools and generators use. Anything
|
|
919
|
+
* that is neither macOS nor Windows is treated as linux: it is the only other
|
|
920
|
+
* desktop persona modelled, and the closest to a BSD host's TCP fingerprint.
|
|
921
|
+
*/
|
|
922
|
+
hostOS() {
|
|
923
|
+
if (process.platform === 'darwin') return 'macos';
|
|
924
|
+
if (process.platform === 'win32') return 'windows';
|
|
925
|
+
return 'linux';
|
|
640
926
|
}
|
|
641
927
|
|
|
642
928
|
/**
|
|
@@ -648,6 +934,64 @@ export class StealthBrowserManager {
|
|
|
648
934
|
return 'windows';
|
|
649
935
|
}
|
|
650
936
|
|
|
937
|
+
/**
|
|
938
|
+
* The version of the Chromium that is actually installed, as
|
|
939
|
+
* `151.0.7922.34`. playwright-core's browsers.json names the build it
|
|
940
|
+
* downloads — the same file Scrapling reads — and is the only source
|
|
941
|
+
* available before a browser is launched; once one is running,
|
|
942
|
+
* _alignUserAgentsWithBinary() corrects for a hosted image that pointed
|
|
943
|
+
* PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH at a Chromium of its own.
|
|
944
|
+
*/
|
|
945
|
+
static installedChromeVersion() {
|
|
946
|
+
try {
|
|
947
|
+
const require = createRequire(import.meta.url);
|
|
948
|
+
const file = path.join(path.dirname(require.resolve('playwright-core')), 'browsers.json');
|
|
949
|
+
const entry = JSON.parse(fs.readFileSync(file, 'utf8')).browsers
|
|
950
|
+
.find((browser) => browser.name === 'chromium');
|
|
951
|
+
return entry?.browserVersion || null;
|
|
952
|
+
} catch {
|
|
953
|
+
return null;
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
/** The leading integer of a version string: `151.0.7922.34` -> 151. */
|
|
958
|
+
static majorVersion(version) {
|
|
959
|
+
const match = /^(\d+)/.exec(String(version ?? '').trim());
|
|
960
|
+
return match ? Number(match[1]) : null;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
/**
|
|
964
|
+
* One Chrome user agent per OS at `major`. Chrome froze every other token of
|
|
965
|
+
* the string years ago, so these are literal, not templates with room.
|
|
966
|
+
*/
|
|
967
|
+
buildChromeUserAgents(major) {
|
|
968
|
+
const chrome = `AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36`;
|
|
969
|
+
return {
|
|
970
|
+
windows: [`Mozilla/5.0 (Windows NT 10.0; Win64; x64) ${chrome}`],
|
|
971
|
+
macos: [`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ${chrome}`],
|
|
972
|
+
linux: [`Mozilla/5.0 (X11; Linux x86_64) ${chrome}`]
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Re-derive the Chrome pool from the browser that actually launched. Only
|
|
978
|
+
* does anything when the running binary is not the one browsers.json named —
|
|
979
|
+
* a hosted image's system Chromium — which is exactly the case where the
|
|
980
|
+
* pool would otherwise mis-state the version userAgentData reports.
|
|
981
|
+
*/
|
|
982
|
+
_alignUserAgentsWithBinary(browser) {
|
|
983
|
+
let version = null;
|
|
984
|
+
try {
|
|
985
|
+
version = browser.version();
|
|
986
|
+
} catch { /* a browser that cannot be asked keeps the installed version */ }
|
|
987
|
+
if (!version || version === this.chromeVersion) return;
|
|
988
|
+
this.chromeVersion = version;
|
|
989
|
+
const major = StealthBrowserManager.majorVersion(version);
|
|
990
|
+
if (!major || major === this.chromeMajor) return;
|
|
991
|
+
this.chromeMajor = major;
|
|
992
|
+
this.userAgentPools.chrome = this.buildChromeUserAgents(major);
|
|
993
|
+
}
|
|
994
|
+
|
|
651
995
|
/**
|
|
652
996
|
* Select realistic user agent based on market distribution
|
|
653
997
|
*/
|
|
@@ -657,12 +1001,14 @@ export class StealthBrowserManager {
|
|
|
657
1001
|
}
|
|
658
1002
|
|
|
659
1003
|
if (!config.useRandomUserAgent) {
|
|
660
|
-
|
|
1004
|
+
// Not "the Windows default" any more: a fixed UA still has to be the
|
|
1005
|
+
// host's, or it contradicts everything the machine itself reports.
|
|
1006
|
+
return this.userAgentPools.chrome[this.hostOS()][0];
|
|
661
1007
|
}
|
|
662
1008
|
|
|
663
|
-
// Use the OS chosen once for this fingerprint (falls back to
|
|
664
|
-
// if called without one,
|
|
665
|
-
selectedOS = selectedOS || this.
|
|
1009
|
+
// Use the OS chosen once for this fingerprint (falls back to the host's own
|
|
1010
|
+
// if called without one, which is what selectOS would have returned).
|
|
1011
|
+
selectedOS = selectedOS || this.hostOS();
|
|
666
1012
|
|
|
667
1013
|
// Select browser based on distribution and OS compatibility
|
|
668
1014
|
let availableBrowsers = { ...this.browserDistribution };
|
|
@@ -796,24 +1142,41 @@ export class StealthBrowserManager {
|
|
|
796
1142
|
}
|
|
797
1143
|
|
|
798
1144
|
/**
|
|
799
|
-
*
|
|
800
|
-
*
|
|
801
|
-
*
|
|
1145
|
+
* The brand list Chrome presents, in both places it is read: the sec-ch-ua
|
|
1146
|
+
* request header and navigator.userAgentData.brands.
|
|
1147
|
+
*
|
|
1148
|
+
* One source for both because the two used to disagree. The header was built
|
|
1149
|
+
* here and said "Google Chrome"; userAgentData was left to Chromium, which
|
|
1150
|
+
* filled it in from the binary and answered "HeadlessChrome 151, Chromium
|
|
1151
|
+
* 151" with no Google Chrome brand at all — rebrowser flagged the page as
|
|
1152
|
+
* Chrome for Testing (2026-09-21 benchmark, useragentdata-brands and
|
|
1153
|
+
* headless-markers). _applyEmulatedIdentity now hands this list to the
|
|
1154
|
+
* renderer, so the header and the JS API are the same list.
|
|
1155
|
+
*
|
|
802
1156
|
* @param {string} [userAgent] — the selected user agent string
|
|
1157
|
+
* @returns {Array<{brand:string, version:string}>}
|
|
803
1158
|
*/
|
|
804
|
-
|
|
1159
|
+
generateUserAgentBrands(userAgent = '') {
|
|
805
1160
|
// Extract Chrome major version from the UA (e.g. "Chrome/151.0.0.0" → "151").
|
|
806
|
-
// Fall back to the
|
|
807
|
-
const match = userAgent.match(/Chrome\/(\d+)/i);
|
|
808
|
-
const version = match ? match[1] :
|
|
1161
|
+
// Fall back to the installed Chromium's major if the UA is not a Chrome UA.
|
|
1162
|
+
const match = String(userAgent).match(/Chrome\/(\d+)/i);
|
|
1163
|
+
const version = match ? match[1] : String(this.chromeMajor);
|
|
809
1164
|
|
|
810
|
-
|
|
1165
|
+
return [
|
|
811
1166
|
{ brand: 'Not_A Brand', version: '8' },
|
|
812
1167
|
{ brand: 'Chromium', version },
|
|
813
1168
|
{ brand: 'Google Chrome', version }
|
|
814
1169
|
];
|
|
1170
|
+
}
|
|
815
1171
|
|
|
816
|
-
|
|
1172
|
+
/**
|
|
1173
|
+
* Generate sec-ch-ua header.
|
|
1174
|
+
* C2: brand versions are derived from the UA's Chrome major version so
|
|
1175
|
+
* sec-ch-ua and the User-Agent header stay consistent.
|
|
1176
|
+
* @param {string} [userAgent] — the selected user agent string
|
|
1177
|
+
*/
|
|
1178
|
+
generateSecChUaHeader(userAgent = '') {
|
|
1179
|
+
return this.generateUserAgentBrands(userAgent)
|
|
817
1180
|
.map(b => `"${b.brand}";v="${b.version}"`)
|
|
818
1181
|
.join(', ');
|
|
819
1182
|
}
|
|
@@ -828,7 +1191,7 @@ export class StealthBrowserManager {
|
|
|
828
1191
|
linux: '"Linux"'
|
|
829
1192
|
};
|
|
830
1193
|
|
|
831
|
-
selectedOS = selectedOS || this.
|
|
1194
|
+
selectedOS = selectedOS || this.hostOS();
|
|
832
1195
|
return platforms[selectedOS] || '"Windows"';
|
|
833
1196
|
}
|
|
834
1197
|
|
|
@@ -1093,7 +1456,7 @@ export class StealthBrowserManager {
|
|
|
1093
1456
|
* Generate realistic hardware fingerprint
|
|
1094
1457
|
*/
|
|
1095
1458
|
generateHardwareFingerprint(selectedOS) {
|
|
1096
|
-
selectedOS = selectedOS || this.
|
|
1459
|
+
selectedOS = selectedOS || this.hostOS();
|
|
1097
1460
|
|
|
1098
1461
|
const processors = [
|
|
1099
1462
|
{ cores: 4, threads: 8, name: 'Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz' },
|
|
@@ -1104,20 +1467,60 @@ export class StealthBrowserManager {
|
|
|
1104
1467
|
{ cores: 8, threads: 16, name: 'AMD Ryzen 7 3700X 8-Core Processor' }
|
|
1105
1468
|
];
|
|
1106
1469
|
|
|
1107
|
-
|
|
1470
|
+
// hardwareConcurrency and deviceMemory are observed, not drawn — the same
|
|
1471
|
+
// call taken for the persona OS, and for the same reason.
|
|
1472
|
+
//
|
|
1473
|
+
// Both are readable from a Worker. An init script does not run in one, and
|
|
1474
|
+
// Emulation.setHardwareConcurrencyOverride does not reach one either: a
|
|
1475
|
+
// Worker is a separate target and the Emulation domain is not available on
|
|
1476
|
+
// it. So a drawn value only ever landed in the document, beside the host's
|
|
1477
|
+
// real one in the worker — 16 cores against 32 (2026-09-21 benchmark,
|
|
1478
|
+
// worker-hardware-concurrency). A machine that reports its own cores is not
|
|
1479
|
+
// a signal. A machine that disagrees with itself is.
|
|
1480
|
+
const hardwareConcurrency = this.hostHardwareConcurrency();
|
|
1481
|
+
const matching = processors.filter((p) => p.threads === hardwareConcurrency);
|
|
1482
|
+
const selectedProcessor = matching.length
|
|
1483
|
+
? matching[Math.floor(Math.random() * matching.length)]
|
|
1484
|
+
// No modelled CPU has this many threads. Name one that could — the
|
|
1485
|
+
// processor string is reported in the fingerprint, never to a page.
|
|
1486
|
+
: {
|
|
1487
|
+
cores: Math.max(1, Math.round(hardwareConcurrency / 2)),
|
|
1488
|
+
threads: hardwareConcurrency,
|
|
1489
|
+
name: `AMD Ryzen ${Math.max(1, Math.round(hardwareConcurrency / 2))}-Core Processor`
|
|
1490
|
+
};
|
|
1108
1491
|
|
|
1109
1492
|
return {
|
|
1110
|
-
hardwareConcurrency
|
|
1493
|
+
hardwareConcurrency,
|
|
1111
1494
|
processor: selectedProcessor.name,
|
|
1112
1495
|
architecture: 'x86_64',
|
|
1113
1496
|
memory: Math.floor(Math.random() * 24) + 8, // 8-32 GB
|
|
1114
|
-
|
|
1115
|
-
// or 32 is a value no real browser reports (sannysoft CHR_MEMORY: FAIL).
|
|
1116
|
-
deviceMemory: Math.random() < 0.7 ? 8 : 4,
|
|
1497
|
+
deviceMemory: this.hostDeviceMemory(),
|
|
1117
1498
|
platform: this.selectRealisticPlatform(selectedOS)
|
|
1118
1499
|
};
|
|
1119
1500
|
}
|
|
1120
1501
|
|
|
1502
|
+
/**
|
|
1503
|
+
* The core count navigator.hardwareConcurrency will report: the host's own
|
|
1504
|
+
* logical CPU count, which is what Chromium answers with
|
|
1505
|
+
* (base::SysInfo::NumberOfProcessors) in the document and in every worker.
|
|
1506
|
+
*/
|
|
1507
|
+
hostHardwareConcurrency() {
|
|
1508
|
+
const count = os.cpus()?.length || 0;
|
|
1509
|
+
return count > 0 ? count : 4;
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
/**
|
|
1513
|
+
* navigator.deviceMemory as Chromium computes it: physical RAM rounded to the
|
|
1514
|
+
* nearest power of two, in GiB, then clamped to [0.25, 8]. The clamp is why
|
|
1515
|
+
* 16 or 32 is a value no real browser reports (sannysoft CHR_MEMORY: FAIL),
|
|
1516
|
+
* and why almost every desktop answers 8.
|
|
1517
|
+
*/
|
|
1518
|
+
hostDeviceMemory() {
|
|
1519
|
+
const gib = os.totalmem() / (1024 ** 3);
|
|
1520
|
+
if (!(gib > 0)) return 8;
|
|
1521
|
+
return Math.min(8, Math.max(0.25, 2 ** Math.round(Math.log2(gib))));
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1121
1524
|
/**
|
|
1122
1525
|
* Map the chosen OS to its navigator.platform value so it stays consistent
|
|
1123
1526
|
* with the user-agent and sec-ch-ua-platform header.
|
|
@@ -1249,17 +1652,67 @@ export class StealthBrowserManager {
|
|
|
1249
1652
|
* Apply advanced stealth configurations to browser context
|
|
1250
1653
|
*/
|
|
1251
1654
|
async applyAdvancedStealthConfigurations(context, config, fingerprint) {
|
|
1655
|
+
// Nothing is injected into camoufox.
|
|
1656
|
+
//
|
|
1657
|
+
// Every script below is Chromium-shaped — it patches navigator.webdriver,
|
|
1658
|
+
// installs a window.chrome, and patches getContext, AudioContext and font
|
|
1659
|
+
// metrics from the main world. camoufox does all of that in its own
|
|
1660
|
+
// C++/Juggler layer, where there is no JS seam to find, and a page that
|
|
1661
|
+
// compares property descriptors, Function.prototype.toString output or the
|
|
1662
|
+
// main thread against a Worker sees ours and not camoufox's. Injecting on
|
|
1663
|
+
// top of an engine built to need no injection only adds back the tells the
|
|
1664
|
+
// engine was chosen to avoid.
|
|
1665
|
+
//
|
|
1666
|
+
// This does not clear deviceandbrowserinfo.com's hasInconsistentWorkerValues
|
|
1667
|
+
// on camoufox: that flag stayed set with every one of these disabled, so it
|
|
1668
|
+
// is camoufox's own worker leak, not ours. What it removes is our share.
|
|
1669
|
+
if (this._launchedEngine === 'camoufox') {
|
|
1670
|
+
return;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1252
1673
|
// Enhanced initialization script with comprehensive stealth measures
|
|
1253
|
-
await context.addInitScript((
|
|
1254
|
-
//
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1674
|
+
await context.addInitScript(() => {
|
|
1675
|
+
// Everything this script changes about navigator goes on
|
|
1676
|
+
// Navigator.prototype, never on the navigator instance.
|
|
1677
|
+
//
|
|
1678
|
+
// A real Chrome answers Object.getOwnPropertyNames(navigator) with an
|
|
1679
|
+
// empty array: every property it has is inherited. So one
|
|
1680
|
+
// defineProperty(navigator, …) puts the spoof's own name in a list any
|
|
1681
|
+
// page can print in a line — rebrowser's bot detector prints exactly that
|
|
1682
|
+
// list, and it read ["connection","plugins","mimeTypes","getBattery"]
|
|
1683
|
+
// (2026-09-21 bench). It is the same class of tell as the deleted
|
|
1684
|
+
// webdriver property, one level up.
|
|
1685
|
+
//
|
|
1686
|
+
// A property that is not on the prototype is left alone rather than
|
|
1687
|
+
// invented: a shape that no Chromium has is worse than the real value.
|
|
1688
|
+
const defineOnPrototype = (key, descriptor) => {
|
|
1689
|
+
if (!(key in Navigator.prototype)) return false;
|
|
1690
|
+
try {
|
|
1691
|
+
// WebIDL attributes and operations are both enumerable and
|
|
1692
|
+
// configurable on the prototype; matching that keeps the descriptor
|
|
1693
|
+
// indistinguishable from the one it replaces.
|
|
1694
|
+
Object.defineProperty(Navigator.prototype, key, Object.assign(
|
|
1695
|
+
{ configurable: true, enumerable: true }, descriptor
|
|
1696
|
+
));
|
|
1697
|
+
return true;
|
|
1698
|
+
} catch (e) {
|
|
1699
|
+
return false;
|
|
1700
|
+
}
|
|
1701
|
+
};
|
|
1702
|
+
|
|
1703
|
+
// navigator.webdriver reports false; the property stays. Deleting it was
|
|
1704
|
+
// its own tell — rebrowser's bot detector marks a missing webdriver red,
|
|
1705
|
+
// because every real Chrome has the property and answers false
|
|
1706
|
+
// (2026-09-21 benchmark, navigator-webdriver). The launch flags do this
|
|
1707
|
+
// already (--disable-blink-features=AutomationControlled, and
|
|
1708
|
+
// --enable-automation taken off the default args), so on a normal launch
|
|
1709
|
+
// this is a no-op and the native prototype getter is left untouched; the
|
|
1710
|
+
// redefinition is only for a binary where the flag did not take.
|
|
1711
|
+
if (navigator.webdriver !== false) {
|
|
1712
|
+
defineOnPrototype('webdriver', { get: () => false });
|
|
1713
|
+
}
|
|
1259
1714
|
|
|
1260
1715
|
// Hide automation indicators
|
|
1261
|
-
delete window.navigator.__proto__.webdriver;
|
|
1262
|
-
delete window.navigator.webdriver;
|
|
1263
1716
|
delete window.webdriver;
|
|
1264
1717
|
delete window._phantom;
|
|
1265
1718
|
delete window.__nightmare;
|
|
@@ -1284,18 +1737,15 @@ export class StealthBrowserManager {
|
|
|
1284
1737
|
originalQuery(parameters)
|
|
1285
1738
|
);
|
|
1286
1739
|
|
|
1287
|
-
//
|
|
1288
|
-
//
|
|
1289
|
-
//
|
|
1290
|
-
//
|
|
1291
|
-
//
|
|
1292
|
-
|
|
1293
|
-
get: () => 4,
|
|
1294
|
-
configurable: true
|
|
1295
|
-
});
|
|
1740
|
+
// No hardwareConcurrency here, and none anywhere else either: the
|
|
1741
|
+
// fingerprint now carries the host's own core count
|
|
1742
|
+
// (generateHardwareFingerprint), so there is nothing to override. A
|
|
1743
|
+
// defineProperty here only ever reached the document, and the worker
|
|
1744
|
+
// answering with the real count beside it was the mismatch
|
|
1745
|
+
// bot.incolumitas.com scores.
|
|
1296
1746
|
|
|
1297
1747
|
// Spoof connection
|
|
1298
|
-
|
|
1748
|
+
defineOnPrototype('connection', {
|
|
1299
1749
|
get: () => ({
|
|
1300
1750
|
effectiveType: '4g',
|
|
1301
1751
|
rtt: 50 + Math.random() * 50,
|
|
@@ -1360,7 +1810,7 @@ export class StealthBrowserManager {
|
|
|
1360
1810
|
Object.defineProperty(pluginArray, index, { value: plugin, enumerable: true });
|
|
1361
1811
|
Object.defineProperty(pluginArray, plugin.name, { value: plugin });
|
|
1362
1812
|
});
|
|
1363
|
-
|
|
1813
|
+
defineOnPrototype('plugins', { get: () => pluginArray });
|
|
1364
1814
|
|
|
1365
1815
|
// Real Chrome pairs those plugins with two navigator.mimeTypes entries
|
|
1366
1816
|
// (application/pdf, text/pdf). Five plugins beside an empty mimeTypes
|
|
@@ -1379,31 +1829,29 @@ export class StealthBrowserManager {
|
|
|
1379
1829
|
Object.defineProperty(mimeTypeArray, index, { value: mimeType, enumerable: true });
|
|
1380
1830
|
Object.defineProperty(mimeTypeArray, mimeType.type, { value: mimeType });
|
|
1381
1831
|
});
|
|
1382
|
-
|
|
1832
|
+
defineOnPrototype('mimeTypes', { get: () => mimeTypeArray });
|
|
1383
1833
|
}
|
|
1384
1834
|
}
|
|
1385
1835
|
|
|
1386
|
-
//
|
|
1387
|
-
//
|
|
1388
|
-
// the
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
return
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
});
|
|
1406
|
-
};
|
|
1836
|
+
// No languages here either. The persona's language list is handed to
|
|
1837
|
+
// Chromium as the accept-language override in _applyEmulatedIdentity, so
|
|
1838
|
+
// the document and every worker parse the same list; this defineProperty
|
|
1839
|
+
// reached only the document, and the worker's ["en-US"] against the
|
|
1840
|
+
// document's ["en-US", "en"] was the difference the benchmark measured
|
|
1841
|
+
// (2026-09-21, worker-languages).
|
|
1842
|
+
|
|
1843
|
+
// Mock battery API with realistic values. A data property, not an
|
|
1844
|
+
// accessor: getBattery is a WebIDL operation, so on a real Navigator
|
|
1845
|
+
// prototype it is a writable function value and not a getter.
|
|
1846
|
+
defineOnPrototype('getBattery', {
|
|
1847
|
+
writable: true,
|
|
1848
|
+
value: function getBattery() {
|
|
1849
|
+
return Promise.resolve({
|
|
1850
|
+
charging: true,
|
|
1851
|
+
chargingTime: 0,
|
|
1852
|
+
dischargingTime: Infinity,
|
|
1853
|
+
level: 0.8 + Math.random() * 0.19 // 80-99%
|
|
1854
|
+
});
|
|
1407
1855
|
}
|
|
1408
1856
|
});
|
|
1409
1857
|
|
|
@@ -1435,7 +1883,7 @@ export class StealthBrowserManager {
|
|
|
1435
1883
|
return originalPrepareStackTrace.call(this, error, filteredStack);
|
|
1436
1884
|
};
|
|
1437
1885
|
}
|
|
1438
|
-
}
|
|
1886
|
+
});
|
|
1439
1887
|
|
|
1440
1888
|
// WebRTC leak prevention with advanced spoofing
|
|
1441
1889
|
if (config.blockWebRTC) {
|
|
@@ -1609,48 +2057,56 @@ export class StealthBrowserManager {
|
|
|
1609
2057
|
}, fingerprint.mediaDevices);
|
|
1610
2058
|
}
|
|
1611
2059
|
|
|
1612
|
-
// Hardware
|
|
1613
|
-
|
|
2060
|
+
// Hardware. Nothing is defined onto the document's navigator any more.
|
|
2061
|
+
// hardwareConcurrency and deviceMemory are the host's own values
|
|
2062
|
+
// (generateHardwareFingerprint), so there is nothing left to override — a
|
|
2063
|
+
// defineProperty that sets a property to the value it already has is a JS
|
|
2064
|
+
// seam bought for nothing. platform is set through CDP
|
|
2065
|
+
// (_applyEmulatedIdentity), and with the persona OS taken from the host it
|
|
2066
|
+
// is usually the truth as well.
|
|
2067
|
+
//
|
|
2068
|
+
// Init scripts never run inside a dedicated Worker, so a detector that
|
|
2069
|
+
// compares navigator in a worker with the main thread saw the real
|
|
2070
|
+
// platform, deviceMemory and hardwareConcurrency beside the spoofed
|
|
2071
|
+
// ones (bot.incolumitas.com "inconsistentWebWorkerNavigatorPropery",
|
|
2072
|
+
// R17 2026-09-04). At the advanced level, classic workers start from a
|
|
2073
|
+
// blob that patches WorkerNavigator and then importScripts() the real
|
|
2074
|
+
// script; relative importScripts/fetch inside such a worker resolve
|
|
2075
|
+
// against the original script URL. Module workers, data:/blob: worker
|
|
2076
|
+
// URLs and a CSP that refuses blob workers fall through to the native
|
|
2077
|
+
// constructor untouched. (camoufox never reaches here — this whole method
|
|
2078
|
+
// returns early for it.)
|
|
2079
|
+
//
|
|
2080
|
+
// MEASURED COVERAGE, 2026-09-16, chromium, comparing a worker's navigator
|
|
2081
|
+
// with the main thread's:
|
|
2082
|
+
//
|
|
2083
|
+
// level new Worker('/w.js') new Worker(blob:)
|
|
2084
|
+
// medium leaks all four leaks all four
|
|
2085
|
+
// advanced matches leaks all four
|
|
2086
|
+
//
|
|
2087
|
+
// The four are platform, hardwareConcurrency, deviceMemory and languages.
|
|
2088
|
+
// So a detector that builds its worker from a Blob — which needs no second
|
|
2089
|
+
// request and is the usual shape — read straight past this at every level.
|
|
2090
|
+
//
|
|
2091
|
+
// THE CALL, taken 2026-09-21: stop drawing what a worker can contradict.
|
|
2092
|
+
// hardwareConcurrency and deviceMemory are the host's own values now, and
|
|
2093
|
+
// languages comes from the context locale, which Playwright applies to the
|
|
2094
|
+
// worker as well as the document — all three agree without a wrapper.
|
|
2095
|
+
// That leaves platform, and only for a caller who pins a customUserAgent
|
|
2096
|
+
// from another OS: the persona's OS is otherwise the host's, so the spoofed
|
|
2097
|
+
// platform IS the real one and the wrapper would rewrite every worker
|
|
2098
|
+
// source on the page to change nothing. Hence the guard — the surface is
|
|
2099
|
+
// paid for only when it buys something. WebGL's unmasked vendor/renderer is
|
|
2100
|
+
// still window-only and still leaks in a worker.
|
|
2101
|
+
const hostPlatform = this.selectRealisticPlatform(this.hostOS());
|
|
2102
|
+
if (config.fingerprinting?.hardwareSpoofing
|
|
2103
|
+
&& config.level === 'advanced'
|
|
2104
|
+
&& fingerprint.hardware.platform !== hostPlatform) {
|
|
1614
2105
|
await context.addInitScript((hardware) => {
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
});
|
|
1618
|
-
|
|
1619
|
-
Object.defineProperty(navigator, 'platform', {
|
|
1620
|
-
get: () => hardware.platform
|
|
1621
|
-
});
|
|
1622
|
-
|
|
1623
|
-
if (navigator.deviceMemory !== undefined) {
|
|
1624
|
-
Object.defineProperty(navigator, 'deviceMemory', {
|
|
1625
|
-
get: () => hardware.deviceMemory
|
|
1626
|
-
});
|
|
1627
|
-
}
|
|
1628
|
-
}, fingerprint.hardware);
|
|
1629
|
-
|
|
1630
|
-
// Init scripts never run inside a dedicated Worker, so a detector that
|
|
1631
|
-
// compares navigator in a worker with the main thread saw the real
|
|
1632
|
-
// platform, deviceMemory and hardwareConcurrency beside the spoofed
|
|
1633
|
-
// ones (bot.incolumitas.com "inconsistentWebWorkerNavigatorPropery",
|
|
1634
|
-
// R17 2026-09-04). At the advanced level, classic workers start from a
|
|
1635
|
-
// blob that patches WorkerNavigator and then importScripts() the real
|
|
1636
|
-
// script; relative importScripts/fetch inside such a worker resolve
|
|
1637
|
-
// against the original script URL. Module workers, data:/blob: worker
|
|
1638
|
-
// URLs and a CSP that refuses blob workers fall through to the native
|
|
1639
|
-
// constructor untouched. camoufox spoofs workers itself.
|
|
1640
|
-
if (config.level === 'advanced' && this._launchedEngine !== 'camoufox') {
|
|
1641
|
-
await context.addInitScript(({ hardware, locale }) => {
|
|
1642
|
-
const NativeWorker = window.Worker;
|
|
1643
|
-
if (typeof NativeWorker !== 'function') return;
|
|
1644
|
-
const primary = locale.split('-')[0];
|
|
1645
|
-
const languages = primary === locale ? [locale] : [locale, primary];
|
|
1646
|
-
const spoof = JSON.stringify({
|
|
1647
|
-
hardwareConcurrency: hardware.hardwareConcurrency,
|
|
1648
|
-
platform: hardware.platform,
|
|
1649
|
-
deviceMemory: hardware.deviceMemory,
|
|
1650
|
-
languages,
|
|
1651
|
-
language: locale
|
|
1652
|
-
});
|
|
1653
|
-
const prelude = (scriptUrl) =>
|
|
2106
|
+
const NativeWorker = window.Worker;
|
|
2107
|
+
if (typeof NativeWorker !== 'function') return;
|
|
2108
|
+
const spoof = JSON.stringify({ platform: hardware.platform });
|
|
2109
|
+
const prelude = (scriptUrl) =>
|
|
1654
2110
|
`(() => {
|
|
1655
2111
|
const spoof = ${spoof};
|
|
1656
2112
|
const proto = self.WorkerNavigator && self.WorkerNavigator.prototype;
|
|
@@ -1658,11 +2114,7 @@ export class StealthBrowserManager {
|
|
|
1658
2114
|
try { Object.defineProperty(proto, key, { get: () => value, configurable: true }); } catch (e) {}
|
|
1659
2115
|
};
|
|
1660
2116
|
if (proto) {
|
|
1661
|
-
define('hardwareConcurrency', spoof.hardwareConcurrency);
|
|
1662
2117
|
define('platform', spoof.platform);
|
|
1663
|
-
if ('deviceMemory' in self.navigator) define('deviceMemory', spoof.deviceMemory);
|
|
1664
|
-
define('languages', spoof.languages);
|
|
1665
|
-
define('language', spoof.language);
|
|
1666
2118
|
}
|
|
1667
2119
|
const base = ${JSON.stringify(scriptUrl)};
|
|
1668
2120
|
const resolve = (u) => { try { return new URL(String(u), base).href; } catch (e) { return u; } };
|
|
@@ -1680,25 +2132,24 @@ export class StealthBrowserManager {
|
|
|
1680
2132
|
}
|
|
1681
2133
|
})();
|
|
1682
2134
|
importScripts(${JSON.stringify(scriptUrl)});`;
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
}
|
|
1691
|
-
} catch (e) {
|
|
1692
|
-
// CSP refused the blob URL or the URL failed to parse: native worker.
|
|
2135
|
+
const Wrapped = function Worker(scriptURL, options) {
|
|
2136
|
+
try {
|
|
2137
|
+
const isModule = options && options.type === 'module';
|
|
2138
|
+
const absolute = new URL(String(scriptURL), location.href).href;
|
|
2139
|
+
if (!isModule && /^https?:/.test(absolute)) {
|
|
2140
|
+
const blob = new Blob([prelude(absolute)], { type: 'text/javascript' });
|
|
2141
|
+
return new NativeWorker(URL.createObjectURL(blob), options);
|
|
1693
2142
|
}
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
2143
|
+
} catch (e) {
|
|
2144
|
+
// CSP refused the blob URL or the URL failed to parse: native worker.
|
|
2145
|
+
}
|
|
2146
|
+
return new NativeWorker(scriptURL, options);
|
|
2147
|
+
};
|
|
2148
|
+
Wrapped.prototype = NativeWorker.prototype;
|
|
2149
|
+
Object.defineProperty(Wrapped, 'name', { value: 'Worker' });
|
|
2150
|
+
Wrapped.toString = () => 'function Worker() { [native code] }';
|
|
2151
|
+
window.Worker = Wrapped;
|
|
2152
|
+
}, fingerprint.hardware);
|
|
1702
2153
|
}
|
|
1703
2154
|
|
|
1704
2155
|
// Font spoofing
|
|
@@ -1780,11 +2231,22 @@ export class StealthBrowserManager {
|
|
|
1780
2231
|
// Battery API spoofing
|
|
1781
2232
|
if (config.antiDetection?.spoofBatteryAPI) {
|
|
1782
2233
|
await context.addInitScript((battery) => {
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
2234
|
+
// On the prototype, where getBattery natively lives. A plain
|
|
2235
|
+
// `navigator.getBattery = …` assignment makes it an OWN property of the
|
|
2236
|
+
// navigator instance, which is the list rebrowser prints (2026-09-21
|
|
2237
|
+
// bench) — a real Chrome's Object.getOwnPropertyNames(navigator) is
|
|
2238
|
+
// empty.
|
|
2239
|
+
if (!('getBattery' in Navigator.prototype)) return;
|
|
2240
|
+
try {
|
|
2241
|
+
Object.defineProperty(Navigator.prototype, 'getBattery', {
|
|
2242
|
+
configurable: true,
|
|
2243
|
+
enumerable: true,
|
|
2244
|
+
writable: true,
|
|
2245
|
+
value: function getBattery() {
|
|
2246
|
+
return Promise.resolve(battery);
|
|
2247
|
+
}
|
|
2248
|
+
});
|
|
2249
|
+
} catch (e) { /* a build that refuses the redefinition keeps the real one */ }
|
|
1788
2250
|
}, fingerprint.battery);
|
|
1789
2251
|
}
|
|
1790
2252
|
}
|
|
@@ -1960,25 +2422,104 @@ export class StealthBrowserManager {
|
|
|
1960
2422
|
}
|
|
1961
2423
|
|
|
1962
2424
|
/**
|
|
1963
|
-
*
|
|
2425
|
+
* Parse one proxyRotation entry into the { server, username, password } shape
|
|
2426
|
+
* Playwright takes.
|
|
2427
|
+
*
|
|
2428
|
+
* Residential proxies — the only kind that helps against Cloudflare's IP
|
|
2429
|
+
* reputation check — are issued as `http://user:pass@host:port`. Those
|
|
2430
|
+
* credentials are the whole point: the previous code pushed the entry into
|
|
2431
|
+
* `--proxy-server=`, a Chromium flag with nowhere to put them, so every
|
|
2432
|
+
* authenticating proxy answered 407 and the request failed. Splitting them out
|
|
2433
|
+
* here is what lets both engines authenticate.
|
|
2434
|
+
*
|
|
2435
|
+
* A malformed entry throws rather than returning null. A proxy that silently
|
|
2436
|
+
* does not apply is the failure mode this whole path is being fixed for: the
|
|
2437
|
+
* caller believes their traffic is proxied and it is not.
|
|
1964
2438
|
*/
|
|
1965
|
-
|
|
1966
|
-
if (
|
|
2439
|
+
parseProxyEntry(entry) {
|
|
2440
|
+
if (typeof entry !== 'string' || !entry.trim()) {
|
|
2441
|
+
throw new Error('proxyRotation.proxies entries must be non-empty strings');
|
|
2442
|
+
}
|
|
2443
|
+
const raw = entry.trim();
|
|
2444
|
+
// `host:port` with no scheme parses as protocol "host:" and an empty host,
|
|
2445
|
+
// so give the bare form the http:// every proxy list assumes.
|
|
2446
|
+
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `http://${raw}`;
|
|
2447
|
+
|
|
2448
|
+
let url;
|
|
2449
|
+
try {
|
|
2450
|
+
url = new URL(withScheme);
|
|
2451
|
+
} catch {
|
|
2452
|
+
throw new Error(`Invalid proxy "${this.redactProxy(raw)}": expected host:port or scheme://user:pass@host:port`);
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
const scheme = url.protocol.replace(':', '').toLowerCase();
|
|
2456
|
+
if (!['http', 'https', 'socks4', 'socks5'].includes(scheme)) {
|
|
2457
|
+
throw new Error(`Invalid proxy scheme "${scheme}": expected http, https, socks4 or socks5`);
|
|
2458
|
+
}
|
|
2459
|
+
if (!url.hostname) {
|
|
2460
|
+
throw new Error(`Invalid proxy "${this.redactProxy(raw)}": no host`);
|
|
2461
|
+
}
|
|
2462
|
+
|
|
2463
|
+
// url.origin is the string "null" for socks4/socks5 — they are not special
|
|
2464
|
+
// schemes — so the server is rebuilt from protocol and host.
|
|
2465
|
+
const proxy = { server: `${url.protocol}//${url.host}` };
|
|
2466
|
+
// A password with a "@" or ":" in it must arrive percent-encoded to parse at
|
|
2467
|
+
// all; the proxy expects the decoded value.
|
|
2468
|
+
if (url.username) proxy.username = decodeURIComponent(url.username);
|
|
2469
|
+
if (url.password) proxy.password = decodeURIComponent(url.password);
|
|
2470
|
+
return proxy;
|
|
2471
|
+
}
|
|
2472
|
+
|
|
2473
|
+
/** A proxy entry with its credentials removed, for logs and get_stats. */
|
|
2474
|
+
redactProxy(entry) {
|
|
2475
|
+
return String(entry).replace(/\/\/[^/@]*@/, '//');
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
/**
|
|
2479
|
+
* Pick the proxy this context should use, advancing the rotation when its
|
|
2480
|
+
* interval has elapsed.
|
|
2481
|
+
*
|
|
2482
|
+
* Called per context rather than per launch. The old call site ran once, at
|
|
2483
|
+
* browser launch, and the browser is cached for the life of the process — so
|
|
2484
|
+
* rotationInterval could never elapse anywhere that mattered and the second
|
|
2485
|
+
* proxy in a list was never reached.
|
|
2486
|
+
*/
|
|
2487
|
+
resolveProxy(config) {
|
|
2488
|
+
// The caller's list always wins: a per-call proxyRotation is a deliberate
|
|
2489
|
+
// choice of exit address, and the server-level CRAWLFORGE_STEALTH_PROXIES
|
|
2490
|
+
// is only what this server goes out from when nobody chose one — the
|
|
2491
|
+
// escalation stage, the agent and browser_session have no caller to ask.
|
|
2492
|
+
// Read through the helper on every call, so an operator's list can be set
|
|
2493
|
+
// for a run without restarting the process.
|
|
2494
|
+
const requested = config.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
|
|
2495
|
+
const proxies = requested.length ? requested : serverStealthProxies();
|
|
2496
|
+
if (!proxies.length) {
|
|
1967
2497
|
return null;
|
|
1968
2498
|
}
|
|
1969
|
-
|
|
2499
|
+
// The server-level list arrives as a bare list with no rotation block of
|
|
2500
|
+
// its own, so it gets the schema's own default interval rather than
|
|
2501
|
+
// comparing against undefined — which is never greater, i.e. never rotates.
|
|
2502
|
+
const rotationInterval = config.proxyRotation?.rotationInterval ?? 300000;
|
|
2503
|
+
|
|
1970
2504
|
const now = Date.now();
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
2505
|
+
if (this.proxyManager.currentProxy === null) {
|
|
2506
|
+
// First use takes proxies[0]. The old code advanced the index before its
|
|
2507
|
+
// first read, so a single-proxy list worked by wrapping to 0 and every
|
|
2508
|
+
// longer list silently started at the second entry.
|
|
2509
|
+
this.proxyManager.proxyIndex = 0;
|
|
2510
|
+
this.proxyManager.lastRotation = now;
|
|
2511
|
+
} else if (now - this.proxyManager.lastRotation > rotationInterval) {
|
|
1974
2512
|
this.proxyManager.proxyIndex = (this.proxyManager.proxyIndex + 1) % proxies.length;
|
|
1975
|
-
this.proxyManager.currentProxy = proxies[this.proxyManager.proxyIndex];
|
|
1976
2513
|
this.proxyManager.lastRotation = now;
|
|
1977
|
-
|
|
1978
|
-
console.error('Rotated to proxy:', this.proxyManager.currentProxy);
|
|
1979
2514
|
}
|
|
1980
|
-
|
|
1981
|
-
|
|
2515
|
+
|
|
2516
|
+
const entry = proxies[this.proxyManager.proxyIndex % proxies.length];
|
|
2517
|
+
const parsed = this.parseProxyEntry(entry);
|
|
2518
|
+
// Only ever hold the redacted form: getStats() returns currentProxy to the
|
|
2519
|
+
// caller, and these strings carry a password.
|
|
2520
|
+
this.proxyManager.currentProxy = this.redactProxy(entry);
|
|
2521
|
+
this.proxyManager.activeProxies = proxies.map((p) => this.redactProxy(p));
|
|
2522
|
+
return parsed;
|
|
1982
2523
|
}
|
|
1983
2524
|
|
|
1984
2525
|
/**
|
|
@@ -2014,18 +2555,27 @@ export class StealthBrowserManager {
|
|
|
2014
2555
|
*
|
|
2015
2556
|
* @param {Object} params
|
|
2016
2557
|
* @param {string} params.url — URL to scrape
|
|
2017
|
-
* @param {string} [params.engine] — browser engine (forwarded to config;
|
|
2558
|
+
* @param {string} [params.engine] — browser engine (forwarded to config; 'auto' by default, i.e. camoufox when installed)
|
|
2018
2559
|
* @param {number} [params.wait_for] — extra wait after load, in ms
|
|
2019
2560
|
* @param {boolean} [params.screenshot] — capture a base64 PNG screenshot
|
|
2020
2561
|
* @param {Object} [params.stealthConfig] — stealth configuration overrides
|
|
2021
|
-
* @returns {Promise<{success:boolean, url:string, title:string, text:string, html:string, screenshot:?string}>}
|
|
2562
|
+
* @returns {Promise<{success:boolean, url:string, title:string, text:string, html:string, screenshot:?string, engine:?string, warnings:string[]}>}
|
|
2563
|
+
* `engine` is the one that actually ran and `warnings` carries the
|
|
2564
|
+
* 'auto'→chromium downgrade, so the caller can report it.
|
|
2022
2565
|
*/
|
|
2023
2566
|
/**
|
|
2024
|
-
* If the document is a self-solving bot-wall interstitial
|
|
2025
|
-
*
|
|
2026
|
-
*
|
|
2027
|
-
*
|
|
2028
|
-
*
|
|
2567
|
+
* If the document is a self-solving bot-wall interstitial, wait up to
|
|
2568
|
+
* timeoutMs for the title to change — the challenge script navigates to the
|
|
2569
|
+
* real page when it passes — and for that navigation to reach
|
|
2570
|
+
* domcontentloaded. A challenge that never passes is left to
|
|
2571
|
+
* challengeDetection to report as blocked.
|
|
2572
|
+
*
|
|
2573
|
+
* The trigger is looksLikeInterstitial, not a known title: nowsecure.nl's
|
|
2574
|
+
* interstitial is titled "nowsecure.nl", so a title-only test skipped the
|
|
2575
|
+
* wait entirely and the block verdict fired on a challenge that had not been
|
|
2576
|
+
* given its chance to solve itself (2026-09-21 benchmark, section 2.2). That
|
|
2577
|
+
* costs one page.content() per call, which is what reading the challenge
|
|
2578
|
+
* bootstrap out of the document takes.
|
|
2029
2579
|
*/
|
|
2030
2580
|
/**
|
|
2031
2581
|
* A document with no title and no text right after domcontentloaded is
|
|
@@ -2056,6 +2606,80 @@ export class StealthBrowserManager {
|
|
|
2056
2606
|
return Date.now() - started;
|
|
2057
2607
|
}
|
|
2058
2608
|
|
|
2609
|
+
/**
|
|
2610
|
+
* Give the page the chance to finish rendering before it is read.
|
|
2611
|
+
*
|
|
2612
|
+
* Navigation returns at DOMContentLoaded, and everything before this only
|
|
2613
|
+
* waits for the document to become non-empty (_waitOutEmptyDocument) or to
|
|
2614
|
+
* stop being an interstitial (_waitOutChallenge). A page that already has
|
|
2615
|
+
* prose and writes the part the caller came for in a load handler passed all
|
|
2616
|
+
* of those as finished, and the read landed mid-render.
|
|
2617
|
+
*
|
|
2618
|
+
* The 6.6.2 bench recorded that as a format bug — "markdown silently dropped
|
|
2619
|
+
* the verdict, text captured it" — but its two calls were two page loads and
|
|
2620
|
+
* only one of them raced. One call asking for markdown and text returns the
|
|
2621
|
+
* same content in both, because both are built from one render. The defect
|
|
2622
|
+
* was the timing.
|
|
2623
|
+
*
|
|
2624
|
+
* Two bounded waits: the page's own load event, then quiet in the DOM. What
|
|
2625
|
+
* this cannot do is predict a payload injected into a still page some
|
|
2626
|
+
* arbitrary time later — that is what the caller's `wait_for` is for.
|
|
2627
|
+
*
|
|
2628
|
+
* @returns {Promise<number>} milliseconds actually waited, 0 for a page that
|
|
2629
|
+
* was already finished.
|
|
2630
|
+
*/
|
|
2631
|
+
async _settleRender(page, { loadTimeoutMs = 3000, quietMs = 400, capMs = 2500 } = {}) {
|
|
2632
|
+
const started = Date.now();
|
|
2633
|
+
// Subresources are often what the render waits on. Bounded, because a page
|
|
2634
|
+
// with a hanging tracker request never fires load at all.
|
|
2635
|
+
await page.waitForLoadState('load', { timeout: loadTimeoutMs }).catch(() => {});
|
|
2636
|
+
const loadMs = Date.now() - started;
|
|
2637
|
+
return loadMs + await this._settleDom(page, { quietMs, capMs });
|
|
2638
|
+
}
|
|
2639
|
+
|
|
2640
|
+
/**
|
|
2641
|
+
* Resolve once the DOM has been unchanged for `quietMs`, or at `capMs`.
|
|
2642
|
+
*
|
|
2643
|
+
* A MutationObserver in the page, so this is one round trip and returns
|
|
2644
|
+
* immediately on a page that was already still. The cap bounds a page that
|
|
2645
|
+
* never stops animating.
|
|
2646
|
+
*
|
|
2647
|
+
* @returns {Promise<number>} milliseconds the DOM went on changing for — 0
|
|
2648
|
+
* when the page was already still, so a settled page reports no extra wait.
|
|
2649
|
+
*/
|
|
2650
|
+
async _settleDom(page, { quietMs = 400, capMs = 2500 } = {}) {
|
|
2651
|
+
try {
|
|
2652
|
+
return await page.evaluate(({ quiet, cap }) => new Promise((resolve) => {
|
|
2653
|
+
if (!document.documentElement) {
|
|
2654
|
+
resolve(0);
|
|
2655
|
+
return;
|
|
2656
|
+
}
|
|
2657
|
+
const started = Date.now();
|
|
2658
|
+
let quietTimer;
|
|
2659
|
+
const finish = () => {
|
|
2660
|
+
observer.disconnect();
|
|
2661
|
+
clearTimeout(quietTimer);
|
|
2662
|
+
clearTimeout(capTimer);
|
|
2663
|
+
// Subtract the quiet window itself: what is worth reporting is how
|
|
2664
|
+
// long the page kept changing, not the time spent confirming it had
|
|
2665
|
+
// stopped.
|
|
2666
|
+
resolve(Math.max(0, Date.now() - started - quiet));
|
|
2667
|
+
};
|
|
2668
|
+
const observer = new MutationObserver(() => {
|
|
2669
|
+
clearTimeout(quietTimer);
|
|
2670
|
+
quietTimer = setTimeout(finish, quiet);
|
|
2671
|
+
});
|
|
2672
|
+
const capTimer = setTimeout(finish, cap);
|
|
2673
|
+
quietTimer = setTimeout(finish, quiet);
|
|
2674
|
+
observer.observe(document.documentElement, { childList: true, subtree: true, characterData: true });
|
|
2675
|
+
}), { quiet: quietMs, cap: capMs });
|
|
2676
|
+
} catch {
|
|
2677
|
+
// Navigated away, closed or crashed mid-settle. The read that follows
|
|
2678
|
+
// reports that properly; there is nothing to add here.
|
|
2679
|
+
return 0;
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2059
2683
|
async _waitOutChallenge(page, { timeoutMs = 8000 } = {}) {
|
|
2060
2684
|
let title;
|
|
2061
2685
|
try {
|
|
@@ -2063,7 +2687,15 @@ export class StealthBrowserManager {
|
|
|
2063
2687
|
} catch {
|
|
2064
2688
|
return;
|
|
2065
2689
|
}
|
|
2066
|
-
|
|
2690
|
+
// A page that is navigating — which a challenge that has just solved itself
|
|
2691
|
+
// is — can answer title() and still refuse content(). Falling back to the
|
|
2692
|
+
// title-only decision keeps a known interstitial waited on; only the marker
|
|
2693
|
+
// branch, and with it the custom-titled wall, is lost for that call.
|
|
2694
|
+
let html = '';
|
|
2695
|
+
try {
|
|
2696
|
+
html = await page.content();
|
|
2697
|
+
} catch { /* title-only decision */ }
|
|
2698
|
+
if (!looksLikeInterstitial({ title, html })) return;
|
|
2067
2699
|
await page
|
|
2068
2700
|
.waitForFunction((t) => document.title !== t, title, { timeout: timeoutMs })
|
|
2069
2701
|
.catch(() => {});
|
|
@@ -2073,7 +2705,12 @@ export class StealthBrowserManager {
|
|
|
2073
2705
|
async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
|
|
2074
2706
|
if (!url) throw new Error('scrapeWithStealth requires a url');
|
|
2075
2707
|
|
|
2076
|
-
const { contextId } = await this.createStealthContext({ ...stealthConfig, engine });
|
|
2708
|
+
const { contextId, engineFallbackWarning = null } = await this.createStealthContext({ ...stealthConfig, engine });
|
|
2709
|
+
// What ran, not what was asked for: 'auto' becomes chromium when camoufox
|
|
2710
|
+
// is absent, and a result that says the stealth browser did not get the
|
|
2711
|
+
// page has to name the browser that tried.
|
|
2712
|
+
const engineUsed = this._launchedEngine ?? null;
|
|
2713
|
+
const warnings = engineFallbackWarning ? [engineFallbackWarning] : [];
|
|
2077
2714
|
try {
|
|
2078
2715
|
const page = await this.createStealthPage(contextId);
|
|
2079
2716
|
let crashed = false;
|
|
@@ -2100,7 +2737,13 @@ export class StealthBrowserManager {
|
|
|
2100
2737
|
// up, so the interstitial came back as success:true (R17, 2026-09-04).
|
|
2101
2738
|
// An auto-solving challenge gets one bounded wait to finish first.
|
|
2102
2739
|
await this._waitOutChallenge(page);
|
|
2103
|
-
const
|
|
2740
|
+
const emptyGraceMs = await this._waitOutEmptyDocument(page);
|
|
2741
|
+
// Last: let whatever is still rendering finish. Without this the read
|
|
2742
|
+
// below can land between "the page has content" and "the page has the
|
|
2743
|
+
// content the caller came for", and a half-rendered page is returned as
|
|
2744
|
+
// a successful scrape.
|
|
2745
|
+
const renderMs = await this._settleRender(page);
|
|
2746
|
+
const gracedMs = emptyGraceMs + renderMs;
|
|
2104
2747
|
|
|
2105
2748
|
// A failure to read the document is a failure. With every read wrapped
|
|
2106
2749
|
// in .catch(() => ''), a renderer that crashed or a page closed during
|
|
@@ -2127,47 +2770,140 @@ export class StealthBrowserManager {
|
|
|
2127
2770
|
? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
|
|
2128
2771
|
: null;
|
|
2129
2772
|
|
|
2130
|
-
return { success: true, url, title, text, html, screenshot: shot, status, gracedMs };
|
|
2773
|
+
return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings };
|
|
2131
2774
|
} finally {
|
|
2132
2775
|
await this.closeContext(contextId).catch(() => {});
|
|
2133
2776
|
}
|
|
2134
2777
|
}
|
|
2135
2778
|
|
|
2136
2779
|
/**
|
|
2137
|
-
* Whether a request is dropped before it leaves the browser.
|
|
2138
|
-
*
|
|
2780
|
+
* Whether a request is dropped before it leaves the browser. Nothing is, at
|
|
2781
|
+
* any level, which is why applyPageStealthMeasures routes no page at all —
|
|
2782
|
+
* this is the one place that answers the question, and the answer is the
|
|
2783
|
+
* reason the interception is gone.
|
|
2139
2784
|
* @param {string} resourceType - Playwright's request.resourceType()
|
|
2140
2785
|
* @param {string} level - stealth level
|
|
2141
2786
|
* @returns {boolean}
|
|
2142
2787
|
*/
|
|
2143
|
-
static shouldAbortRequest(
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2788
|
+
static shouldAbortRequest(_resourceType, _level) {
|
|
2789
|
+
return false;
|
|
2790
|
+
}
|
|
2791
|
+
|
|
2792
|
+
/**
|
|
2793
|
+
* The full user-agent client-hint metadata for a fingerprint: what Playwright
|
|
2794
|
+
* already derives from the user agent, plus the brand lists it leaves to the
|
|
2795
|
+
* binary. Omitting a field means "fill this in with what you would normally
|
|
2796
|
+
* use", and what a headless Chromium normally uses says HeadlessChrome — in
|
|
2797
|
+
* brands, in fullVersionList, and so in every high-entropy hint a detector
|
|
2798
|
+
* asks for.
|
|
2799
|
+
*
|
|
2800
|
+
* The five derived fields are what playwright-core's own
|
|
2801
|
+
* calculateUserAgentMetadata produces for the three desktop personas this
|
|
2802
|
+
* manager draws — so for every user agent it can hand out, this changes the
|
|
2803
|
+
* brands and nothing else. A mobile persona is out of scope here for the same
|
|
2804
|
+
* reason isMobile is pinned false in generateAdvancedFingerprint: the pool has
|
|
2805
|
+
* no mobile user agent to be coherent with.
|
|
2806
|
+
*
|
|
2807
|
+
* @param {Object} fingerprint
|
|
2808
|
+
* @returns {Object} a CDP Emulation.UserAgentMetadata
|
|
2809
|
+
*/
|
|
2810
|
+
generateUserAgentMetadata(fingerprint) {
|
|
2811
|
+
const userAgent = String(fingerprint.userAgent || '');
|
|
2812
|
+
const brands = this.generateUserAgentBrands(userAgent);
|
|
2813
|
+
const brandVersion = brands[brands.length - 1].version;
|
|
2814
|
+
// The four-part build number, but only when it is the binary the brand
|
|
2815
|
+
// version names; a custom UA on another major gets the reduced form rather
|
|
2816
|
+
// than this machine's build under someone else's version.
|
|
2817
|
+
const fullVersion = brandVersion === String(this.chromeMajor)
|
|
2818
|
+
? this.chromeVersion
|
|
2819
|
+
: `${brandVersion}.0.0.0`;
|
|
2820
|
+
const platformVersions = { windows: '10.0', macos: '10_15_7', linux: '' };
|
|
2821
|
+
const platforms = { windows: 'Windows', macos: 'macOS', linux: 'Linux' };
|
|
2822
|
+
const os = this.inferOSFromUserAgent(userAgent);
|
|
2823
|
+
|
|
2824
|
+
return {
|
|
2825
|
+
brands,
|
|
2826
|
+
fullVersionList: brands.map((brand) => ({ brand: brand.brand, version: fullVersion })),
|
|
2827
|
+
fullVersion,
|
|
2828
|
+
platform: platforms[os],
|
|
2829
|
+
platformVersion: platformVersions[os],
|
|
2830
|
+
architecture: 'x86',
|
|
2831
|
+
model: '',
|
|
2832
|
+
mobile: false
|
|
2833
|
+
};
|
|
2834
|
+
}
|
|
2835
|
+
|
|
2836
|
+
/**
|
|
2837
|
+
* Tell the renderer who it is, instead of patching it from script.
|
|
2838
|
+
*
|
|
2839
|
+
* navigator.userAgent, .platform and .userAgentData were being defined on the
|
|
2840
|
+
* document's navigator by init scripts. An init script does not run in a
|
|
2841
|
+
* Worker, so every one of those values was spoofed in the document and
|
|
2842
|
+
* truthful in the worker beside it — a MacIntel under a Windows persona,
|
|
2843
|
+
* which is what bot.incolumitas.com and CreepJS score (2026-09-21 benchmark,
|
|
2844
|
+
* worker-*). Emulation.setUserAgentOverride is applied by the renderer, and
|
|
2845
|
+
* the brands it carries are the ones the binary would otherwise fill in as
|
|
2846
|
+
* HeadlessChrome.
|
|
2847
|
+
*
|
|
2848
|
+
* `acceptLanguage` is the plain locale tag, the same single value Playwright
|
|
2849
|
+
* gives the context. It is deliberately NOT the two-entry "en-US,en" list:
|
|
2850
|
+
* this override reaches the document only, while a worker's
|
|
2851
|
+
* navigator.languages comes from the context locale, so a second entry here
|
|
2852
|
+
* showed up in the document and nowhere else — the very mismatch the init
|
|
2853
|
+
* script used to create (2026-09-21 benchmark, worker-languages). The
|
|
2854
|
+
* Accept-Language *header* still carries the `en;q=0.9` fallback, from
|
|
2855
|
+
* generateAdvancedHeaders, which is what a real Chrome sends for a
|
|
2856
|
+
* single-locale preference. The field cannot simply be omitted: that clears
|
|
2857
|
+
* the override Playwright installed and a de-DE persona would fall back to
|
|
2858
|
+
* the binary's default.
|
|
2859
|
+
*
|
|
2860
|
+
* Chromium only: CDP does not exist on camoufox's Firefox, which spoofs all
|
|
2861
|
+
* of this in its own engine anyway. A failure here is not fatal — Playwright
|
|
2862
|
+
* has already set the user agent through the context, so the persona survives
|
|
2863
|
+
* without the brands.
|
|
2864
|
+
*/
|
|
2865
|
+
async _applyEmulatedIdentity(page, config, fingerprint) {
|
|
2866
|
+
if (!isChromium(page) || !fingerprint.userAgent) return;
|
|
2867
|
+
try {
|
|
2868
|
+
const client = await page.context().newCDPSession(page);
|
|
2869
|
+
await client.send('Emulation.setUserAgentOverride', {
|
|
2870
|
+
userAgent: fingerprint.userAgent,
|
|
2871
|
+
acceptLanguage: fingerprint.locale || config.locale || 'en-US',
|
|
2872
|
+
platform: fingerprint.hardware.platform,
|
|
2873
|
+
userAgentMetadata: this.generateUserAgentMetadata(fingerprint)
|
|
2874
|
+
});
|
|
2875
|
+
} catch (error) {
|
|
2876
|
+
console.warn(`Identity emulation skipped: ${error.message}`);
|
|
2877
|
+
}
|
|
2148
2878
|
}
|
|
2149
2879
|
|
|
2150
2880
|
/**
|
|
2151
2881
|
* Apply page-level stealth measures
|
|
2152
2882
|
*/
|
|
2153
2883
|
async applyPageStealthMeasures(page, config, fingerprint) {
|
|
2154
|
-
//
|
|
2155
|
-
// blocked challenges.cloudflare.com
|
|
2156
|
-
//
|
|
2157
|
-
//
|
|
2158
|
-
//
|
|
2159
|
-
//
|
|
2160
|
-
//
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2884
|
+
// No request routing at all. R15 had already stopped aborting by URL (the
|
|
2885
|
+
// old list blocked challenges.cloudflare.com, so a Cloudflare challenge
|
|
2886
|
+
// could never complete, and killed the navigation to www.selenium.dev);
|
|
2887
|
+
// what was left was the advanced level dropping about a third of images,
|
|
2888
|
+
// fonts and stylesheets at random. Both halves of that are timeable: a page
|
|
2889
|
+
// that renders without the fonts and images it asked for does not look like
|
|
2890
|
+
// a browser reading it, and route('**/*') itself puts every request through
|
|
2891
|
+
// a node round trip, which shows up as latency no network explains
|
|
2892
|
+
// (2026-09-21 review, finding 10). With nothing left to abort there is
|
|
2893
|
+
// nothing for the interception to decide, so the handler is gone rather
|
|
2894
|
+
// than made conditional.
|
|
2895
|
+
|
|
2896
|
+
// Every identity the renderer can be told about directly, rather than
|
|
2897
|
+
// patched into from script.
|
|
2898
|
+
await this._applyEmulatedIdentity(page, config, fingerprint);
|
|
2899
|
+
|
|
2900
|
+
// Add request headers — Chromium only. These carry sec-ch-ua,
|
|
2901
|
+
// sec-ch-ua-mobile and sec-ch-ua-platform, client hints Gecko does not
|
|
2902
|
+
// implement, so setting them here would put them straight back onto a
|
|
2903
|
+
// camoufox page after createStealthContext had taken them off the context.
|
|
2904
|
+
if (isChromium(page)) {
|
|
2905
|
+
await page.setExtraHTTPHeaders(fingerprint.headers);
|
|
2906
|
+
}
|
|
2171
2907
|
|
|
2172
2908
|
// Emulate realistic network conditions.
|
|
2173
2909
|
//
|
|
@@ -2369,6 +3105,7 @@ export class StealthBrowserManager {
|
|
|
2369
3105
|
this._parkedBrowsers.clear();
|
|
2370
3106
|
}
|
|
2371
3107
|
this._launchedEngine = null;
|
|
3108
|
+
this._engineFallbackWarning = null;
|
|
2372
3109
|
for (const browser of browsers) {
|
|
2373
3110
|
const closed = await withDeadline(browser.close(), 5000);
|
|
2374
3111
|
if (!closed) {
|
|
@@ -2541,6 +3278,115 @@ export class CamoufoxAdapter extends BrowserEngine {
|
|
|
2541
3278
|
}
|
|
2542
3279
|
}
|
|
2543
3280
|
|
|
3281
|
+
/**
|
|
3282
|
+
* A browserforge fingerprint pinned to the installed binary's Firefox major.
|
|
3283
|
+
*
|
|
3284
|
+
* camoufox@0.1.19 rewrites the persona's version tokens with
|
|
3285
|
+
*
|
|
3286
|
+
* data.replace(/(?<!\d)(1[0-9]{2})(\.0)(?!\d)/, `${ffVersion}$2`)
|
|
3287
|
+
*
|
|
3288
|
+
* and there is no /g, so only the FIRST match in each string changes. In a
|
|
3289
|
+
* Firefox UA that match is `rv:`, and `Firefox/` keeps whatever version
|
|
3290
|
+
* browserforge drew — the two then agree only by coincidence. Measured on
|
|
3291
|
+
* the installed 135 binary: 4 of 8 launches produced a self-contradicting UA
|
|
3292
|
+
* (`rv:135.0 ... Firefox/150.0`), which is a one-line detection. Generating
|
|
3293
|
+
* the persona ourselves at the binary's own version makes that replace a
|
|
3294
|
+
* no-op and both tokens agree (8/8 measured, end to end through
|
|
3295
|
+
* launchOptions).
|
|
3296
|
+
*
|
|
3297
|
+
* Nothing else is constrained here because camoufox constrains nothing else:
|
|
3298
|
+
* its getScreenCons() returns null on both branches, and its `os` option is
|
|
3299
|
+
* dropped before it reaches the generator (the client sends `os`, the
|
|
3300
|
+
* generator's key is `operatingSystems`). Pinning the OS too would fix that
|
|
3301
|
+
* second bug, but it is Phase 1's documented `persona-os-vs-host` finding and
|
|
3302
|
+
* changing it here would move a baseline this change has no business moving.
|
|
3303
|
+
*
|
|
3304
|
+
* @returns {object|null} null when the version cannot be read, or when
|
|
3305
|
+
* browserforge has no data for it — 152 is outside its set, and silently
|
|
3306
|
+
* drawing some other version would put us back where we started. The
|
|
3307
|
+
* caller then lets camoufox generate as it did before.
|
|
3308
|
+
*/
|
|
3309
|
+
/** process.platform in the vocabulary fingerprint-generator uses, or null. */
|
|
3310
|
+
static _hostOperatingSystem() {
|
|
3311
|
+
if (process.platform === 'darwin') return 'macos';
|
|
3312
|
+
if (process.platform === 'win32') return 'windows';
|
|
3313
|
+
if (process.platform === 'linux') return 'linux';
|
|
3314
|
+
return null;
|
|
3315
|
+
}
|
|
3316
|
+
|
|
3317
|
+
/** The OS a Firefox UA claims, in the same vocabulary, or null. */
|
|
3318
|
+
static _osFromUserAgent(ua) {
|
|
3319
|
+
if (/Macintosh|Mac OS X/.test(ua)) return 'macos';
|
|
3320
|
+
if (/Windows/.test(ua)) return 'windows';
|
|
3321
|
+
if (/Android/.test(ua)) return null; // Linux-derived, but not the host OS we mean
|
|
3322
|
+
if (/X11|Linux/.test(ua)) return 'linux';
|
|
3323
|
+
return null;
|
|
3324
|
+
}
|
|
3325
|
+
|
|
3326
|
+
_pinnedFingerprint(camoufox) {
|
|
3327
|
+
try {
|
|
3328
|
+
const raw = fs.readFileSync(path.join(camoufox.INSTALL_DIR, 'version.json'), 'utf8');
|
|
3329
|
+
const major = parseInt(String(JSON.parse(raw).version).split('.')[0], 10);
|
|
3330
|
+
if (!Number.isInteger(major)) return null;
|
|
3331
|
+
|
|
3332
|
+
// Resolved through camoufox's own module path first: fingerprint-generator
|
|
3333
|
+
// is its dependency, not ours, and this guarantees the same copy it feeds
|
|
3334
|
+
// its data to rather than a second one hoisted elsewhere.
|
|
3335
|
+
const here = createRequire(import.meta.url);
|
|
3336
|
+
let load;
|
|
3337
|
+
try {
|
|
3338
|
+
load = createRequire(here.resolve('camoufox'));
|
|
3339
|
+
} catch {
|
|
3340
|
+
load = here;
|
|
3341
|
+
}
|
|
3342
|
+
const { FingerprintGenerator } = load('fingerprint-generator');
|
|
3343
|
+
|
|
3344
|
+
// Verify rather than assume: a generator with no data for a constraint can
|
|
3345
|
+
// still hand back a persona that ignores it, and an unchecked fingerprint
|
|
3346
|
+
// would reintroduce exactly the mismatch this exists to close.
|
|
3347
|
+
const attempt = (operatingSystems) => {
|
|
3348
|
+
const { fingerprint } = new FingerprintGenerator({
|
|
3349
|
+
browsers: [{ name: 'firefox', minVersion: major, maxVersion: major }],
|
|
3350
|
+
...(operatingSystems ? { operatingSystems } : {})
|
|
3351
|
+
}).getFingerprint();
|
|
3352
|
+
const ua = fingerprint?.navigator?.userAgent ?? '';
|
|
3353
|
+
if (ua.match(/rv:(\d+)/)?.[1] !== String(major)) return null;
|
|
3354
|
+
if (ua.match(/Firefox\/(\d+)/)?.[1] !== String(major)) return null;
|
|
3355
|
+
if (operatingSystems && CamoufoxAdapter._osFromUserAgent(ua) !== operatingSystems[0]) return null;
|
|
3356
|
+
return fingerprint;
|
|
3357
|
+
};
|
|
3358
|
+
|
|
3359
|
+
// `operatingSystems` is the key the client gets wrong — it sends `os`, so
|
|
3360
|
+
// the OS is drawn from market share and a Linux host ships a macOS
|
|
3361
|
+
// persona. Generating here means we can use the right key. Measured on the
|
|
3362
|
+
// hosted box 2026-09-22: persona-os-vs-host failed with a coherent UA
|
|
3363
|
+
// version, which is a contradiction a detector scores on its own.
|
|
3364
|
+
const hostOs = CamoufoxAdapter._hostOperatingSystem();
|
|
3365
|
+
if (hostOs) {
|
|
3366
|
+
try {
|
|
3367
|
+
const matched = attempt([hostOs]);
|
|
3368
|
+
if (matched) return matched;
|
|
3369
|
+
} catch {
|
|
3370
|
+
// An impossible constraint THROWS here rather than returning a
|
|
3371
|
+
// mismatched persona ("No headers based on this input can be
|
|
3372
|
+
// generated"), and browserforge has no Firefox 135 on Linux at all —
|
|
3373
|
+
// measured, and it is the combination the hosted box runs. Letting
|
|
3374
|
+
// that escape would lose the version pin too and put the UA back
|
|
3375
|
+
// where it started, so it falls through to the version-only attempt.
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
|
|
3379
|
+
// Fall back to a version-only pin. browserforge may have no data for this
|
|
3380
|
+
// Firefox version on this OS, and a persona that is coherent about its
|
|
3381
|
+
// version still beats one that is coherent about neither.
|
|
3382
|
+
return attempt(null);
|
|
3383
|
+
} catch {
|
|
3384
|
+
// Best effort: a missing version.json or an absent generator is a reason
|
|
3385
|
+
// to fall back to camoufox's own persona, not to fail a launch.
|
|
3386
|
+
return null;
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3389
|
+
|
|
2544
3390
|
async launch(config = {}) {
|
|
2545
3391
|
let camoufox;
|
|
2546
3392
|
try {
|
|
@@ -2556,14 +3402,80 @@ export class CamoufoxAdapter extends BrowserEngine {
|
|
|
2556
3402
|
|
|
2557
3403
|
await this._ensureMacOSLayout(camoufox);
|
|
2558
3404
|
|
|
3405
|
+
// Before any Firefox page can run: camoufox reports an uncaught page error
|
|
3406
|
+
// with no location, and playwright reads one anyway. See the guard.
|
|
3407
|
+
guardFirefoxPageErrors();
|
|
3408
|
+
|
|
2559
3409
|
// camoufox's launcher is Camoufox(options) — the package has no launch()
|
|
2560
3410
|
// export. It resolves the fetched Firefox binary (npx camoufox fetch) and
|
|
2561
3411
|
// returns a Playwright-compatible Browser. Takes `headless` directly plus
|
|
2562
3412
|
// passthrough Playwright Firefox launch options.
|
|
2563
|
-
|
|
2564
|
-
|
|
3413
|
+
//
|
|
3414
|
+
// The snake_case names below are camoufox's own option names. They are the
|
|
3415
|
+
// reason to run this engine at all: camoufox spoofs at the C++/Juggler
|
|
3416
|
+
// level, where a page cannot see the seam, and every one of these was
|
|
3417
|
+
// simply not being passed — camoufox ran with its own features off.
|
|
3418
|
+
const options = {
|
|
3419
|
+
// A string mode ('virtual' — Xvfb) has to reach camoufox as that string.
|
|
3420
|
+
// `config.headless !== false` collapsed it to `true`, which is plain
|
|
3421
|
+
// headless: the one mode 'virtual' exists to avoid. Booleans keep their
|
|
3422
|
+
// old meaning, so an unset headless is still headless.
|
|
3423
|
+
headless: typeof config.headless === 'string' ? config.headless : config.headless !== false,
|
|
2565
3424
|
...config.launchOptions
|
|
2566
|
-
}
|
|
3425
|
+
};
|
|
3426
|
+
// A bare `{ server }` is fine here; camoufox normalises both shapes.
|
|
3427
|
+
if (config.proxy) options.proxy = config.proxy;
|
|
3428
|
+
// "windows" | "macos" | "linux", as a plain string — a one-element ARRAY is
|
|
3429
|
+
// ignored and falls back to the random draw. Left unset, camoufox picks one
|
|
3430
|
+
// of the three at random, which is how a Mac ended up claiming Windows.
|
|
3431
|
+
if (config.os) options.os = config.os;
|
|
3432
|
+
// The first listed locale is the one used for the Intl API. camoufox
|
|
3433
|
+
// applies it in its own engine, so navigator.language, Accept-Language and
|
|
3434
|
+
// Intl agree in the document and in every worker. Left unset with geoip on,
|
|
3435
|
+
// camoufox derives it from the proxy's exit IP instead.
|
|
3436
|
+
if (config.locale) options.locale = config.locale;
|
|
3437
|
+
// geoip derives longitude, latitude, timezone, country and locale from the
|
|
3438
|
+
// proxy's exit IP, which is the one thing that makes a proxied browser
|
|
3439
|
+
// coherent. It costs a request through the proxy, and the first ever call
|
|
3440
|
+
// downloads MaxMind's city database (~60 MB) into camoufox's install dir.
|
|
3441
|
+
if (config.geoip) options.geoip = true;
|
|
3442
|
+
// Blocking WebRTC is itself a signal. With geoip camoufox instead reports
|
|
3443
|
+
// the proxy's exit IP through WebRTC, which is the coherent answer — so
|
|
3444
|
+
// `blockWebRTC: false` is the stealthier setting behind a proxy.
|
|
3445
|
+
if (config.blockWebRTC) options.block_webrtc = true;
|
|
3446
|
+
// Native cursor humanization: camoufox moves the pointer along a plausible
|
|
3447
|
+
// path rather than teleporting it.
|
|
3448
|
+
if (config.humanize) options.humanize = true;
|
|
3449
|
+
// Pin the persona to the installed binary's Firefox major so the UA's two
|
|
3450
|
+
// version tokens agree. Left unpinned, camoufox's own version rewrite
|
|
3451
|
+
// reaches only `rv:` and about half of all launches announce a Firefox
|
|
3452
|
+
// that is not the one running — see _pinnedFingerprint.
|
|
3453
|
+
const fingerprint = this._pinnedFingerprint(camoufox);
|
|
3454
|
+
if (fingerprint) {
|
|
3455
|
+
options.fingerprint = fingerprint;
|
|
3456
|
+
// Switches off exactly two things: camoufox's "you passed your own
|
|
3457
|
+
// fingerprint" advisory, and its non-Firefox fingerprint check, which is
|
|
3458
|
+
// satisfied by construction — the generator above is Firefox-only. The
|
|
3459
|
+
// other options this flag gates (ff_version, block_images, disable_coop,
|
|
3460
|
+
// block_webgl) are ones this adapter never sets, so nothing else is
|
|
3461
|
+
// silenced.
|
|
3462
|
+
options.i_know_what_im_doing = true;
|
|
3463
|
+
}
|
|
3464
|
+
|
|
3465
|
+
try {
|
|
3466
|
+
return await camoufox.Camoufox(options);
|
|
3467
|
+
} catch (err) {
|
|
3468
|
+
if (options.geoip) {
|
|
3469
|
+
// Do not quietly retry without geoip. A proxied camoufox whose
|
|
3470
|
+
// geolocation says one country while its exit IP says another is a
|
|
3471
|
+
// cleaner detection signal than an unproxied one.
|
|
3472
|
+
throw new Error(
|
|
3473
|
+
`camoufox failed to launch with geoip through the configured proxy: ${err.message}. ` +
|
|
3474
|
+
'Check the proxy credentials and that the proxy can reach the internet.'
|
|
3475
|
+
);
|
|
3476
|
+
}
|
|
3477
|
+
throw err;
|
|
3478
|
+
}
|
|
2567
3479
|
}
|
|
2568
3480
|
|
|
2569
3481
|
/**
|