barebrowse 0.14.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +157 -0
- package/README.md +33 -1
- package/barebrowse.context.md +5 -2
- package/cli.js +10 -2
- package/mcp-server.js +14 -2
- package/package.json +1 -1
- package/src/auth.js +27 -13
- package/src/ax-snapshot.js +287 -0
- package/src/bidi.js +143 -0
- package/src/consent-firefox.js +125 -0
- package/src/consent-patterns.js +154 -0
- package/src/consent.js +5 -144
- package/src/daemon.js +162 -51
- package/src/firefox-page.js +445 -0
- package/src/firefox.js +203 -0
- package/src/index.js +88 -6
- package/src/network-idle.js +66 -28
- package/src/readable.js +32 -20
- package/src/stealth-firefox.js +42 -0
- package/src/stealth.js +107 -69
- package/types/auth.d.ts +18 -1
- package/types/ax-snapshot.d.ts +35 -0
- package/types/bidi.d.ts +10 -0
- package/types/consent-firefox.d.ts +9 -0
- package/types/consent-patterns.d.ts +12 -0
- package/types/daemon.d.ts +34 -0
- package/types/firefox-page.d.ts +21 -0
- package/types/firefox.d.ts +41 -0
- package/types/index.d.ts +13 -2
- package/types/network-idle.d.ts +16 -8
- package/types/readable.d.ts +8 -0
- package/types/stealth-firefox.d.ts +9 -0
- package/types/stealth.d.ts +31 -0
package/src/index.js
CHANGED
|
@@ -9,13 +9,17 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { launch, attach, cleanupBrowser } from './chromium.js';
|
|
12
|
+
import { launchFirefox, cleanupFirefox } from './firefox.js';
|
|
12
13
|
import { createCDP } from './cdp.js';
|
|
14
|
+
import { createBiDi } from './bidi.js';
|
|
15
|
+
import { createFirefoxPage } from './firefox-page.js';
|
|
13
16
|
import { formatTree } from './aria.js';
|
|
14
17
|
import { authenticate } from './auth.js';
|
|
15
18
|
import { prune as pruneTree } from './prune.js';
|
|
16
19
|
import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
|
|
17
20
|
import { dismissConsent } from './consent.js';
|
|
18
21
|
import { applyStealth } from './stealth.js';
|
|
22
|
+
import { applyFirefoxStealth } from './stealth-firefox.js';
|
|
19
23
|
import { DEFAULT_BLOCKLIST } from './blocklist.js';
|
|
20
24
|
import { waitForNetworkIdle } from './network-idle.js';
|
|
21
25
|
import { readable as extractReadable } from './readable.js';
|
|
@@ -31,6 +35,8 @@ import { chmodSync } from 'node:fs';
|
|
|
31
35
|
* @param {object} [opts]
|
|
32
36
|
* @param {'headless'|'headed'|'hybrid'} [opts.mode='headless'] - Browser mode
|
|
33
37
|
* @param {boolean} [opts.cookies=true] - Inject user's cookies (Phase 2)
|
|
38
|
+
* @param {boolean} [opts.incognito=false] - Clean, unauthenticated session:
|
|
39
|
+
* skip cookie/auth injection entirely (equivalent to cookies:false here).
|
|
34
40
|
* @param {boolean} [opts.prune=true] - Apply ARIA pruning (Phase 2)
|
|
35
41
|
* @param {number} [opts.timeout=30000] - Navigation timeout in ms
|
|
36
42
|
* @param {boolean} [opts.blockAds=true] - Block ~120 common ad/tracker
|
|
@@ -54,6 +60,8 @@ import { chmodSync } from 'node:fs';
|
|
|
54
60
|
export async function browse(url, opts = {}) {
|
|
55
61
|
const mode = opts.mode || 'headless';
|
|
56
62
|
const timeout = opts.timeout || 30000;
|
|
63
|
+
// Skip all auth injection when incognito (or the legacy cookies:false).
|
|
64
|
+
const noAuth = !!opts.incognito || opts.cookies === false;
|
|
57
65
|
|
|
58
66
|
// Reject local-resource schemes (and optionally private hosts) before we
|
|
59
67
|
// spend a browser launch on a URL we won't navigate to.
|
|
@@ -84,7 +92,7 @@ export async function browse(url, opts = {}) {
|
|
|
84
92
|
await suppressPermissions(cdp);
|
|
85
93
|
|
|
86
94
|
// Step 3: Cookie injection — extract from user's browser, inject via CDP
|
|
87
|
-
if (
|
|
95
|
+
if (!noAuth) {
|
|
88
96
|
try {
|
|
89
97
|
await authenticate(page.session, url, { browser: opts.browser });
|
|
90
98
|
} catch {
|
|
@@ -114,7 +122,7 @@ export async function browse(url, opts = {}) {
|
|
|
114
122
|
cdp = await createCDP(browser.wsUrl);
|
|
115
123
|
page = await createPage(cdp, false, pageOpts);
|
|
116
124
|
await suppressPermissions(cdp);
|
|
117
|
-
if (
|
|
125
|
+
if (!noAuth) {
|
|
118
126
|
try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
|
|
119
127
|
}
|
|
120
128
|
await navigate(page, url, timeout);
|
|
@@ -188,12 +196,32 @@ export async function browse(url, opts = {}) {
|
|
|
188
196
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
|
|
189
197
|
* @param {string} [opts.storageState] - Path to a storage-state JSON file
|
|
190
198
|
* (cookies + localStorage) to load before navigation.
|
|
199
|
+
* @param {boolean} [opts.incognito=false] - Clean, unauthenticated session:
|
|
200
|
+
* skip storageState loading and make injectCookies() a no-op, so no auth
|
|
201
|
+
* ever enters the session even if a caller injects unconditionally.
|
|
191
202
|
* @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Pruning mode.
|
|
203
|
+
* @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
|
|
204
|
+
* 'firefox' drives over WebDriver BiDi (a separate transport / page object);
|
|
205
|
+
* 'chromium' (default) uses CDP.
|
|
192
206
|
* @returns {Promise<object>} Page handle with goto, snapshot, close
|
|
193
207
|
*/
|
|
194
208
|
export async function connect(opts = {}) {
|
|
209
|
+
// Firefox is driven over WebDriver BiDi (CDP is deprecated there) — a
|
|
210
|
+
// separate transport with its own page object. It reuses prune.js/aria.js/
|
|
211
|
+
// readable.js but none of the CDP page machinery below, so branch early.
|
|
212
|
+
if (opts.engine === 'firefox') {
|
|
213
|
+
return connectFirefox(opts);
|
|
214
|
+
}
|
|
215
|
+
|
|
195
216
|
const mode = opts.mode || 'headless';
|
|
196
217
|
const attachMode = !!opts.port;
|
|
218
|
+
// Incognito: run a clean, unauthenticated session. Skips storageState loading
|
|
219
|
+
// and neuters injectCookies() so no auth ever enters the session — even when
|
|
220
|
+
// a caller (MCP goto, daemon) calls injectCookies() unconditionally. The
|
|
221
|
+
// temp profile is already throwaway; this gates the OTHER auth source: the
|
|
222
|
+
// user's real browser cookies. Not Chrome's --incognito flag (redundant with
|
|
223
|
+
// the temp profile and would break the cookie-injection path when off).
|
|
224
|
+
const incognito = !!opts.incognito;
|
|
197
225
|
let browser = null;
|
|
198
226
|
let cdp;
|
|
199
227
|
// Forward caller-supplied launch knobs into every launch() below,
|
|
@@ -243,8 +271,9 @@ export async function connect(opts = {}) {
|
|
|
243
271
|
await suppressPermissions(cdp);
|
|
244
272
|
}
|
|
245
273
|
|
|
246
|
-
// Load storage state (cookies + localStorage) from file
|
|
247
|
-
|
|
274
|
+
// Load storage state (cookies + localStorage) from file.
|
|
275
|
+
// Skipped under incognito — storageState is an auth source too.
|
|
276
|
+
if (opts.storageState && !incognito) {
|
|
248
277
|
try {
|
|
249
278
|
const { readFileSync } = await import('node:fs');
|
|
250
279
|
const state = JSON.parse(readFileSync(opts.storageState, 'utf8'));
|
|
@@ -439,7 +468,10 @@ export async function connect(opts = {}) {
|
|
|
439
468
|
},
|
|
440
469
|
|
|
441
470
|
async injectCookies(url, cookieOpts) {
|
|
442
|
-
|
|
471
|
+
// No-op under incognito: callers (MCP goto, daemon) inject unconditionally,
|
|
472
|
+
// so the gate has to live here, not just at the call site.
|
|
473
|
+
if (incognito) return 0;
|
|
474
|
+
return authenticate(page.session, url, { browser: cookieOpts?.browser });
|
|
443
475
|
},
|
|
444
476
|
|
|
445
477
|
async snapshot(pruneOpts) {
|
|
@@ -655,7 +687,8 @@ export async function connect(opts = {}) {
|
|
|
655
687
|
},
|
|
656
688
|
get botBlocked() { return tabBotBlocked; },
|
|
657
689
|
async injectCookies(url, cookieOpts) {
|
|
658
|
-
|
|
690
|
+
if (incognito) return 0;
|
|
691
|
+
return authenticate(tab.session, url, { browser: cookieOpts?.browser });
|
|
659
692
|
},
|
|
660
693
|
waitForNetworkIdle(idleOpts = {}) {
|
|
661
694
|
return waitForNetworkIdle(tab.session, idleOpts);
|
|
@@ -708,6 +741,55 @@ async function suppressPermissions(cdp) {
|
|
|
708
741
|
}
|
|
709
742
|
}
|
|
710
743
|
|
|
744
|
+
/**
|
|
745
|
+
* Firefox path for connect(): launch Firefox, open a BiDi session, and return
|
|
746
|
+
* a BiDi-backed page object. Separate from the CDP path because the transport,
|
|
747
|
+
* ref model, and AX-tree source all differ (see firefox-page.js). close() is
|
|
748
|
+
* wrapped to also reap the Firefox process + temp profile.
|
|
749
|
+
*
|
|
750
|
+
* Chromium-only options NOT applied on this path: `blockAds`/`blockUrls` and
|
|
751
|
+
* `hybrid` mode. `saveState`, `waitForNavigation`, and download/dialog capture
|
|
752
|
+
* are CDP-only too (the page object stubs them — see firefox-page.js).
|
|
753
|
+
* `waitForNetworkIdle` and daemon console/network capture DO apply as of Phase
|
|
754
|
+
* 2 (over BiDi `network.*` and `log.entryAdded` events). `consent`
|
|
755
|
+
* (auto-dismiss) and
|
|
756
|
+
* stealth patches DO apply here as of v0.16.0 (stealth headless-only, via BiDi
|
|
757
|
+
* preload script — see stealth-firefox.js / consent-firefox.js), alongside the
|
|
758
|
+
* navigation guard
|
|
759
|
+
* (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir` sandbox, `incognito`,
|
|
760
|
+
* `proxy`, `viewport`, and `pruneMode`.
|
|
761
|
+
* @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito })
|
|
762
|
+
* @returns {Promise<object>} Firefox page object
|
|
763
|
+
*/
|
|
764
|
+
async function connectFirefox(opts) {
|
|
765
|
+
const browser = await launchFirefox({
|
|
766
|
+
headed: opts.mode === 'headed',
|
|
767
|
+
proxy: opts.proxy,
|
|
768
|
+
binary: opts.binary,
|
|
769
|
+
viewport: opts.viewport,
|
|
770
|
+
});
|
|
771
|
+
const bidi = await createBiDi(browser.wsUrl);
|
|
772
|
+
// Stealth patches (headless only, mirroring the CDP path's `mode !== 'headed'`
|
|
773
|
+
// gate). Registered before the first navigation via script.addPreloadScript.
|
|
774
|
+
// Firefox needs a much smaller script than Chromium — see stealth-firefox.js.
|
|
775
|
+
if (opts.mode !== 'headed') {
|
|
776
|
+
await applyFirefoxStealth(bidi);
|
|
777
|
+
}
|
|
778
|
+
const page = await createFirefoxPage(bidi, {
|
|
779
|
+
pruneMode: opts.pruneMode,
|
|
780
|
+
urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
|
|
781
|
+
uploadDir: opts.uploadDir || null,
|
|
782
|
+
incognito: !!opts.incognito,
|
|
783
|
+
consent: opts.consent,
|
|
784
|
+
});
|
|
785
|
+
const closePage = page.close.bind(page);
|
|
786
|
+
page.close = async () => {
|
|
787
|
+
await closePage();
|
|
788
|
+
await cleanupFirefox(browser);
|
|
789
|
+
};
|
|
790
|
+
return page;
|
|
791
|
+
}
|
|
792
|
+
|
|
711
793
|
/**
|
|
712
794
|
* Create a new page target and return a session-scoped handle.
|
|
713
795
|
* @param {object} cdp - CDP client
|
package/src/network-idle.js
CHANGED
|
@@ -2,24 +2,27 @@
|
|
|
2
2
|
* network-idle.js — wait until the page's network has been idle for N ms.
|
|
3
3
|
*
|
|
4
4
|
* Tracks in-flight requests by requestId in a Set, so an orphan
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* finish/fail event (for a request whose start event arrived before our
|
|
6
|
+
* listener attached) is a harmless no-op instead of driving a counter
|
|
7
|
+
* negative and resolving prematurely.
|
|
8
|
+
*
|
|
9
|
+
* Two engines, one core: `waitForNetworkIdle` wires the CDP `Network.*`
|
|
10
|
+
* events, `waitForNetworkIdleBiDi` wires the WebDriver BiDi `network.*`
|
|
11
|
+
* events. Both feed the same orphan-resilient idle waiter (`idleWaiter`).
|
|
8
12
|
*/
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* @param {
|
|
15
|
+
* Shared idle-detection loop. `attach` registers the engine-specific event
|
|
16
|
+
* listeners, calling onStart(id) when a request begins and onEnd(id) when it
|
|
17
|
+
* finishes/fails, and pushing each listener's unsub fn into `unsubs`.
|
|
18
|
+
* @param {object} cfg
|
|
19
|
+
* @param {number} cfg.timeout - Max wait before reject
|
|
20
|
+
* @param {number} cfg.idle - Required quiet duration before resolve
|
|
21
|
+
* @param {(hooks: {onStart: (id:any)=>void, onEnd: (id:any)=>void, unsubs: Array<()=>void>}) => void} cfg.attach
|
|
15
22
|
* @returns {Promise<void>}
|
|
16
23
|
*/
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const idle = opts.idle || 500;
|
|
20
|
-
|
|
21
|
-
/** @type {Promise<void>} */
|
|
22
|
-
const settled = new Promise((resolve, reject) => {
|
|
24
|
+
function idleWaiter({ timeout, idle, attach }) {
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
23
26
|
const pending = new Set();
|
|
24
27
|
let timer = null;
|
|
25
28
|
const unsubs = [];
|
|
@@ -38,20 +41,12 @@ export function waitForNetworkIdle(session, opts = {}) {
|
|
|
38
41
|
}
|
|
39
42
|
};
|
|
40
43
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
// requests started before we attached the listener can't push us negative.
|
|
48
|
-
pending.delete(p.requestId);
|
|
49
|
-
check();
|
|
50
|
-
}));
|
|
51
|
-
unsubs.push(session.on('Network.loadingFailed', (p) => {
|
|
52
|
-
pending.delete(p.requestId);
|
|
53
|
-
check();
|
|
54
|
-
}));
|
|
44
|
+
const onStart = (id) => { pending.add(id); clearTimeout(timer); };
|
|
45
|
+
// delete() on a Set is a no-op for unknown keys — orphan finish/fail
|
|
46
|
+
// events from requests started before we attached can't push us negative.
|
|
47
|
+
const onEnd = (id) => { pending.delete(id); check(); };
|
|
48
|
+
|
|
49
|
+
attach({ onStart, onEnd, unsubs });
|
|
55
50
|
|
|
56
51
|
const deadlineTimer = setTimeout(() => {
|
|
57
52
|
for (const unsub of unsubs) unsub();
|
|
@@ -61,5 +56,48 @@ export function waitForNetworkIdle(session, opts = {}) {
|
|
|
61
56
|
// Start check immediately (might already be idle)
|
|
62
57
|
check();
|
|
63
58
|
});
|
|
64
|
-
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* CDP: wait for network idle over `Network.*` events.
|
|
63
|
+
* @param {object} session - CDP session-scoped handle with .on() returning unsub
|
|
64
|
+
* @param {object} [opts]
|
|
65
|
+
* @param {number} [opts.timeout=30000] - Max wait time before reject
|
|
66
|
+
* @param {number} [opts.idle=500] - Required idle duration before resolve
|
|
67
|
+
* @returns {Promise<void>}
|
|
68
|
+
*/
|
|
69
|
+
export function waitForNetworkIdle(session, opts = {}) {
|
|
70
|
+
return idleWaiter({
|
|
71
|
+
timeout: opts.timeout || 30000,
|
|
72
|
+
idle: opts.idle || 500,
|
|
73
|
+
attach: ({ onStart, onEnd, unsubs }) => {
|
|
74
|
+
unsubs.push(session.on('Network.requestWillBeSent', (p) => onStart(p.requestId)));
|
|
75
|
+
unsubs.push(session.on('Network.loadingFinished', (p) => onEnd(p.requestId)));
|
|
76
|
+
unsubs.push(session.on('Network.loadingFailed', (p) => onEnd(p.requestId)));
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Firefox/BiDi: wait for network idle over `network.*` events. Subscribes the
|
|
83
|
+
* events first (idempotent — safe even if the daemon already subscribed for
|
|
84
|
+
* log capture), then runs the same orphan-resilient idle loop. The in-flight
|
|
85
|
+
* key is `params.request.request` (the BiDi request id).
|
|
86
|
+
* @param {object} bidi - BiDi client with async .subscribe() and .on() returning unsub
|
|
87
|
+
* @param {object} [opts]
|
|
88
|
+
* @param {number} [opts.timeout=30000] - Max wait time before reject
|
|
89
|
+
* @param {number} [opts.idle=500] - Required idle duration before resolve
|
|
90
|
+
* @returns {Promise<void>}
|
|
91
|
+
*/
|
|
92
|
+
export async function waitForNetworkIdleBiDi(bidi, opts = {}) {
|
|
93
|
+
await bidi.subscribe(['network.beforeRequestSent', 'network.responseCompleted', 'network.fetchError']);
|
|
94
|
+
return idleWaiter({
|
|
95
|
+
timeout: opts.timeout || 30000,
|
|
96
|
+
idle: opts.idle || 500,
|
|
97
|
+
attach: ({ onStart, onEnd, unsubs }) => {
|
|
98
|
+
unsubs.push(bidi.on('network.beforeRequestSent', (p) => onStart(p.request.request)));
|
|
99
|
+
unsubs.push(bidi.on('network.responseCompleted', (p) => onEnd(p.request.request)));
|
|
100
|
+
unsubs.push(bidi.on('network.fetchError', (p) => onEnd(p.request.request)));
|
|
101
|
+
},
|
|
102
|
+
});
|
|
65
103
|
}
|
package/src/readable.js
CHANGED
|
@@ -72,21 +72,21 @@ export function formatReadable(r) {
|
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
* { ok: true, title, byline, text, length,
|
|
80
|
-
* confidence: 'high'|'low', readerable, hint? } — extracted article
|
|
75
|
+
* The in-page extraction expression (Readability injected + run). Exported so
|
|
76
|
+
* non-CDP transports (BiDi/Firefox) can evaluate the same source and feed the
|
|
77
|
+
* raw result through finalizeReadable() — keeping every engine's output
|
|
78
|
+
* identical.
|
|
81
79
|
*/
|
|
82
|
-
export
|
|
83
|
-
const { result } = await session.send('Runtime.evaluate', {
|
|
84
|
-
expression: EXTRACT_EXPRESSION,
|
|
85
|
-
returnByValue: true,
|
|
86
|
-
awaitPromise: true,
|
|
87
|
-
});
|
|
88
|
-
const r = result.value || {};
|
|
80
|
+
export { EXTRACT_EXPRESSION };
|
|
89
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Turn the raw in-page extraction result into the public readable() shape,
|
|
84
|
+
* applying the advisory confidence rule. Transport-agnostic.
|
|
85
|
+
* @param {object} r - raw result from EXTRACT_EXPRESSION
|
|
86
|
+
* @returns {object} public readable() result
|
|
87
|
+
*/
|
|
88
|
+
export function finalizeReadable(r) {
|
|
89
|
+
r = r || {};
|
|
90
90
|
if (!r.ok) {
|
|
91
91
|
return {
|
|
92
92
|
ok: false,
|
|
@@ -95,22 +95,34 @@ export async function readable(session) {
|
|
|
95
95
|
: 'no article content found on this page; use snapshot() instead',
|
|
96
96
|
};
|
|
97
97
|
}
|
|
98
|
-
|
|
99
98
|
// Advisory confidence: high only when the reader-view heuristic agrees AND
|
|
100
99
|
// there is a substantial amount of text. Low is not an error — the text is
|
|
101
100
|
// still returned; it just means "verify, or prefer snapshot()".
|
|
102
101
|
const confidence = r.readerable && r.length >= MIN_ARTICLE_CHARS ? 'high' : 'low';
|
|
103
102
|
const out = {
|
|
104
103
|
ok: true,
|
|
105
|
-
title: r.title,
|
|
106
|
-
|
|
107
|
-
text: r.text,
|
|
108
|
-
length: r.length,
|
|
109
|
-
readerable: r.readerable,
|
|
110
|
-
confidence,
|
|
104
|
+
title: r.title, byline: r.byline, text: r.text, length: r.length,
|
|
105
|
+
readerable: r.readerable, confidence,
|
|
111
106
|
};
|
|
112
107
|
if (confidence === 'low') {
|
|
113
108
|
out.hint = 'low article confidence — this may not be an article; consider snapshot()';
|
|
114
109
|
}
|
|
115
110
|
return out;
|
|
116
111
|
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Extract the main article from the current page.
|
|
115
|
+
* @param {object} session - CDP session-scoped handle (.send()).
|
|
116
|
+
* @returns {Promise<object>} One of:
|
|
117
|
+
* { ok: false, hint } — no article content found
|
|
118
|
+
* { ok: true, title, byline, text, length,
|
|
119
|
+
* confidence: 'high'|'low', readerable, hint? } — extracted article
|
|
120
|
+
*/
|
|
121
|
+
export async function readable(session) {
|
|
122
|
+
const { result } = await session.send('Runtime.evaluate', {
|
|
123
|
+
expression: EXTRACT_EXPRESSION,
|
|
124
|
+
returnByValue: true,
|
|
125
|
+
awaitPromise: true,
|
|
126
|
+
});
|
|
127
|
+
return finalizeReadable(result.value || {});
|
|
128
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stealth-firefox.js — Anti-detection patches for headless Firefox (BiDi).
|
|
3
|
+
*
|
|
4
|
+
* Deliberately MUCH smaller than the Chromium stealth.js. That isn't laziness —
|
|
5
|
+
* it's what a POC measured stock Firefox-under-BiDi to actually expose:
|
|
6
|
+
*
|
|
7
|
+
* navigator.webdriver → true (the tell — we hide it)
|
|
8
|
+
* window.chrome → absent (correct; ADDING it would be a spoof tell)
|
|
9
|
+
* navigator.plugins → 5 (realistic PDF set; already normal)
|
|
10
|
+
* languages / hardwareConcurrency → normal (en-US,en / 8)
|
|
11
|
+
* User-Agent → real Firefox, NO "Headless" marker (no rewrite needed)
|
|
12
|
+
* WebGL vendor/renderer → the REAL GPU, not SwiftShader (spoofing would
|
|
13
|
+
* replace a real value with a fake one)
|
|
14
|
+
*
|
|
15
|
+
* So porting Chromium's STEALTH_SCRIPT verbatim would have made Firefox look
|
|
16
|
+
* like a spoofed browser (window.chrome + Chrome plugins on Firefox = obvious
|
|
17
|
+
* tell) — a worse fingerprint than the one removed. The only genuinely
|
|
18
|
+
* engine-agnostic pieces are webdriver-hiding and canvas noise, both reused
|
|
19
|
+
* verbatim from stealth.js so their fixes stay single-sourced.
|
|
20
|
+
*
|
|
21
|
+
* BiDi's script.addPreloadScript is the equivalent of CDP's
|
|
22
|
+
* Page.addScriptToEvaluateOnNewDocument — the POC confirmed it runs BEFORE any
|
|
23
|
+
* page script (navigator.webdriver read at parse time already saw `undefined`).
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { WEBDRIVER_PATCH, CANVAS_NOISE_PATCH } from './stealth.js';
|
|
27
|
+
|
|
28
|
+
const FIREFOX_STEALTH_SCRIPT = `${WEBDRIVER_PATCH}\n${CANVAS_NOISE_PATCH}`;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Register the Firefox stealth patches so they run before any page script.
|
|
32
|
+
* Must be called before the first navigation. addPreloadScript is global
|
|
33
|
+
* (applies to every browsing context / navigation), so a single registration
|
|
34
|
+
* covers the whole session.
|
|
35
|
+
*
|
|
36
|
+
* @param {object} bidi - BiDi client from createBiDi()
|
|
37
|
+
*/
|
|
38
|
+
export async function applyFirefoxStealth(bidi) {
|
|
39
|
+
await bidi.send('script.addPreloadScript', {
|
|
40
|
+
functionDeclaration: `() => { ${FIREFOX_STEALTH_SCRIPT} }`,
|
|
41
|
+
});
|
|
42
|
+
}
|
package/src/stealth.js
CHANGED
|
@@ -5,9 +5,114 @@
|
|
|
5
5
|
* any page scripts (unlike Runtime.evaluate which runs after).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* navigator.webdriver hiding — the one automation tell shared by every engine
|
|
10
|
+
* (Chromium headless AND Firefox-under-BiDi both report `true`). Split out so
|
|
11
|
+
* the Firefox stealth path (stealth-firefox.js) reuses the exact same patch
|
|
12
|
+
* instead of duplicating it. Measured baseline on both engines: `true`.
|
|
13
|
+
*
|
|
14
|
+
* A naive `Object.defineProperty(navigator, 'webdriver', …)` hides the *value*
|
|
15
|
+
* but leaves an advanced tell: it creates an OWN property on the instance, so
|
|
16
|
+
* `navigator.hasOwnProperty('webdriver')` becomes `true` where a real browser
|
|
17
|
+
* (property lives on Navigator.prototype) reports `false` — sophisticated
|
|
18
|
+
* anti-bot checks (e.g. sannysoft's "WebDriver New") catch that inconsistency.
|
|
19
|
+
* So we DELETE the getter off the prototype instead: then `navigator.webdriver`
|
|
20
|
+
* is `undefined` AND `'webdriver' in navigator` AND `hasOwnProperty` are both
|
|
21
|
+
* `false`, matching a stock browser. POC-verified on Chromium and Firefox
|
|
22
|
+
* (own/in both false, prototype descriptor gone). The defineProperty fallback
|
|
23
|
+
* only fires if the descriptor is somehow non-configurable (delete a no-op).
|
|
24
|
+
*/
|
|
25
|
+
export const WEBDRIVER_PATCH = `
|
|
26
|
+
try {
|
|
27
|
+
const proto = Object.getPrototypeOf(navigator);
|
|
28
|
+
const desc = proto && Object.getOwnPropertyDescriptor(proto, 'webdriver');
|
|
29
|
+
if (desc && desc.configurable) delete proto.webdriver;
|
|
30
|
+
if (navigator.webdriver) {
|
|
31
|
+
Object.defineProperty(navigator, 'webdriver', { get: () => undefined, configurable: true });
|
|
32
|
+
}
|
|
33
|
+
} catch {}
|
|
34
|
+
`;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Canvas-fingerprint noise — browser-agnostic (standard Canvas2D API), so both
|
|
38
|
+
* the Chromium and Firefox stealth scripts compose it. This block carries the
|
|
39
|
+
* subtle double-XOR-cancellation fixes (see git history / project memory); keep
|
|
40
|
+
* it SINGLE-SOURCED here so a fix never has to be made in two places.
|
|
41
|
+
*/
|
|
42
|
+
export const CANVAS_NOISE_PATCH = `
|
|
43
|
+
// Canvas fingerprinting — sites render standard text/shapes, then read
|
|
44
|
+
// pixels via toDataURL or getImageData. The output is stable per machine
|
|
45
|
+
// (GPU, font rasterizer, anti-aliasing) but unique across machines, which
|
|
46
|
+
// makes it the second-most-common fingerprint after WebGL. Defense: nudge
|
|
47
|
+
// a few RGB channels by ±1 per session so the hash changes each visit
|
|
48
|
+
// while the canvas still looks identical to the human eye. The per-tab
|
|
49
|
+
// seed keeps reads stable within a session so legitimate canvas use
|
|
50
|
+
// (image processing, games) doesn't flicker.
|
|
51
|
+
// crypto.getRandomValues is guaranteed unique per browsing context; using
|
|
52
|
+
// Math.random alone can collide when two fresh V8 contexts start within
|
|
53
|
+
// microseconds of each other (real-world: parallel tests, real-world hit:
|
|
54
|
+
// we observed it). performance.now adds a wall-clock anchor as a belt-and-
|
|
55
|
+
// braces guard against contexts where crypto is somehow stubbed.
|
|
56
|
+
const _seedBuf = new Uint32Array(1);
|
|
57
|
+
crypto.getRandomValues(_seedBuf);
|
|
58
|
+
const CANVAS_SEED = (_seedBuf[0] ^ ((performance.now() * 1e6) | 0)) >>> 0;
|
|
59
|
+
function shiftPixels(data) {
|
|
60
|
+
// Touch ~1 byte per 64-byte stride. The bit we XOR with is taken from a
|
|
61
|
+
// position-dependent SLICE of a seed-mixed hash, not its low bit — a
|
|
62
|
+
// naive 'mix & 1' collapses to only two possible outputs per seed
|
|
63
|
+
// parity because every stride index is even (the position multiplier
|
|
64
|
+
// is odd, so the low bit only depends on seed parity). Indexing the
|
|
65
|
+
// hash by (i/64) mod 32 makes every stride position sample a different
|
|
66
|
+
// bit, so two distinct seeds produce different mask patterns.
|
|
67
|
+
for (let i = 0; i < data.length; i += 64) {
|
|
68
|
+
const h = ((CANVAS_SEED * 2654435761) ^ (i * 1597334677)) >>> 0;
|
|
69
|
+
const bit = (h >>> ((i >>> 6) & 31)) & 1;
|
|
70
|
+
data[i] = (data[i] ^ bit) & 0xff;
|
|
71
|
+
}
|
|
72
|
+
return data;
|
|
73
|
+
}
|
|
74
|
+
// Capture originals BEFORE replacing — toDataURL must read pixels via the
|
|
75
|
+
// native getImageData (not the patched one), otherwise the patch double-
|
|
76
|
+
// applies and the second XOR cancels the first, leaving output unchanged.
|
|
77
|
+
const origGetImageData = CanvasRenderingContext2D.prototype.getImageData;
|
|
78
|
+
const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
79
|
+
HTMLCanvasElement.prototype.toDataURL = function() {
|
|
80
|
+
const ctx = this.getContext('2d');
|
|
81
|
+
if (ctx && this.width > 0 && this.height > 0) {
|
|
82
|
+
try {
|
|
83
|
+
const img = origGetImageData.call(ctx, 0, 0, this.width, this.height);
|
|
84
|
+
// Snapshot the original bytes so we can restore them after encoding.
|
|
85
|
+
// Without this, repeated toDataURL() alternates noisy/clean: call 1
|
|
86
|
+
// XORs the canvas in place, call 2 reads the noisy canvas and XORs
|
|
87
|
+
// again (self-inverse), call 3 again, etc. Same XOR-cancellation
|
|
88
|
+
// class as the earlier double-apply bug, just through canvas state
|
|
89
|
+
// rather than method composition. The restore also keeps the
|
|
90
|
+
// bitmap idempotent for any downstream legitimate canvas reads.
|
|
91
|
+
const original = new Uint8ClampedArray(img.data);
|
|
92
|
+
shiftPixels(img.data);
|
|
93
|
+
ctx.putImageData(img, 0, 0);
|
|
94
|
+
const result = origToDataURL.apply(this, arguments);
|
|
95
|
+
img.data.set(original);
|
|
96
|
+
ctx.putImageData(img, 0, 0);
|
|
97
|
+
return result;
|
|
98
|
+
} catch {
|
|
99
|
+
// Tainted canvas (cross-origin image) — can't read; skip the nudge
|
|
100
|
+
// and fall through to the original call so the page sees the
|
|
101
|
+
// expected SecurityError instead of silent corruption.
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return origToDataURL.apply(this, arguments);
|
|
105
|
+
};
|
|
106
|
+
CanvasRenderingContext2D.prototype.getImageData = function() {
|
|
107
|
+
const img = origGetImageData.apply(this, arguments);
|
|
108
|
+
shiftPixels(img.data);
|
|
109
|
+
return img;
|
|
110
|
+
};
|
|
111
|
+
`;
|
|
112
|
+
|
|
8
113
|
const STEALTH_SCRIPT = `
|
|
9
114
|
// Hide webdriver flag
|
|
10
|
-
|
|
115
|
+
${WEBDRIVER_PATCH}
|
|
11
116
|
|
|
12
117
|
// Fake plugins (headless has 0)
|
|
13
118
|
Object.defineProperty(navigator, 'plugins', {
|
|
@@ -93,74 +198,7 @@ const STEALTH_SCRIPT = `
|
|
|
93
198
|
};
|
|
94
199
|
}
|
|
95
200
|
|
|
96
|
-
|
|
97
|
-
// pixels via toDataURL or getImageData. The output is stable per machine
|
|
98
|
-
// (GPU, font rasterizer, anti-aliasing) but unique across machines, which
|
|
99
|
-
// makes it the second-most-common fingerprint after WebGL. Defense: nudge
|
|
100
|
-
// a few RGB channels by ±1 per session so the hash changes each visit
|
|
101
|
-
// while the canvas still looks identical to the human eye. The per-tab
|
|
102
|
-
// seed keeps reads stable within a session so legitimate canvas use
|
|
103
|
-
// (image processing, games) doesn't flicker.
|
|
104
|
-
// crypto.getRandomValues is guaranteed unique per browsing context; using
|
|
105
|
-
// Math.random alone can collide when two fresh V8 contexts start within
|
|
106
|
-
// microseconds of each other (real-world: parallel tests, real-world hit:
|
|
107
|
-
// we observed it). performance.now adds a wall-clock anchor as a belt-and-
|
|
108
|
-
// braces guard against contexts where crypto is somehow stubbed.
|
|
109
|
-
const _seedBuf = new Uint32Array(1);
|
|
110
|
-
crypto.getRandomValues(_seedBuf);
|
|
111
|
-
const CANVAS_SEED = (_seedBuf[0] ^ ((performance.now() * 1e6) | 0)) >>> 0;
|
|
112
|
-
function shiftPixels(data) {
|
|
113
|
-
// Touch ~1 byte per 64-byte stride. The bit we XOR with is taken from a
|
|
114
|
-
// position-dependent SLICE of a seed-mixed hash, not its low bit — a
|
|
115
|
-
// naive 'mix & 1' collapses to only two possible outputs per seed
|
|
116
|
-
// parity because every stride index is even (the position multiplier
|
|
117
|
-
// is odd, so the low bit only depends on seed parity). Indexing the
|
|
118
|
-
// hash by (i/64) mod 32 makes every stride position sample a different
|
|
119
|
-
// bit, so two distinct seeds produce different mask patterns.
|
|
120
|
-
for (let i = 0; i < data.length; i += 64) {
|
|
121
|
-
const h = ((CANVAS_SEED * 2654435761) ^ (i * 1597334677)) >>> 0;
|
|
122
|
-
const bit = (h >>> ((i >>> 6) & 31)) & 1;
|
|
123
|
-
data[i] = (data[i] ^ bit) & 0xff;
|
|
124
|
-
}
|
|
125
|
-
return data;
|
|
126
|
-
}
|
|
127
|
-
// Capture originals BEFORE replacing — toDataURL must read pixels via the
|
|
128
|
-
// native getImageData (not the patched one), otherwise the patch double-
|
|
129
|
-
// applies and the second XOR cancels the first, leaving output unchanged.
|
|
130
|
-
const origGetImageData = CanvasRenderingContext2D.prototype.getImageData;
|
|
131
|
-
const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
132
|
-
HTMLCanvasElement.prototype.toDataURL = function() {
|
|
133
|
-
const ctx = this.getContext('2d');
|
|
134
|
-
if (ctx && this.width > 0 && this.height > 0) {
|
|
135
|
-
try {
|
|
136
|
-
const img = origGetImageData.call(ctx, 0, 0, this.width, this.height);
|
|
137
|
-
// Snapshot the original bytes so we can restore them after encoding.
|
|
138
|
-
// Without this, repeated toDataURL() alternates noisy/clean: call 1
|
|
139
|
-
// XORs the canvas in place, call 2 reads the noisy canvas and XORs
|
|
140
|
-
// again (self-inverse), call 3 again, etc. Same XOR-cancellation
|
|
141
|
-
// class as the earlier double-apply bug, just through canvas state
|
|
142
|
-
// rather than method composition. The restore also keeps the
|
|
143
|
-
// bitmap idempotent for any downstream legitimate canvas reads.
|
|
144
|
-
const original = new Uint8ClampedArray(img.data);
|
|
145
|
-
shiftPixels(img.data);
|
|
146
|
-
ctx.putImageData(img, 0, 0);
|
|
147
|
-
const result = origToDataURL.apply(this, arguments);
|
|
148
|
-
img.data.set(original);
|
|
149
|
-
ctx.putImageData(img, 0, 0);
|
|
150
|
-
return result;
|
|
151
|
-
} catch {
|
|
152
|
-
// Tainted canvas (cross-origin image) — can't read; skip the nudge
|
|
153
|
-
// and fall through to the original call so the page sees the
|
|
154
|
-
// expected SecurityError instead of silent corruption.
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
return origToDataURL.apply(this, arguments);
|
|
158
|
-
};
|
|
159
|
-
CanvasRenderingContext2D.prototype.getImageData = function() {
|
|
160
|
-
const img = origGetImageData.apply(this, arguments);
|
|
161
|
-
shiftPixels(img.data);
|
|
162
|
-
return img;
|
|
163
|
-
};
|
|
201
|
+
${CANVAS_NOISE_PATCH}
|
|
164
202
|
`;
|
|
165
203
|
|
|
166
204
|
/**
|
package/types/auth.d.ts
CHANGED
|
@@ -25,9 +25,26 @@ export function injectCookies(session: object, cookies: Array<object>): Promise<
|
|
|
25
25
|
* @returns {boolean}
|
|
26
26
|
*/
|
|
27
27
|
export function cookieDomainMatch(host: string, cookieDomain: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Select the user's cookies that are in-scope for `url`. Shared by BOTH engines
|
|
30
|
+
* (CDP `authenticate` and Firefox/BiDi `injectCookies`) so they can't drift on
|
|
31
|
+
* scoping — a divergence here re-opens loading the entire cookie jar into an
|
|
32
|
+
* agent session. Returns the filtered cookie array; never injects.
|
|
33
|
+
*
|
|
34
|
+
* Coarse SQL pre-filter strips to a registrable-ish domain so the LIKE query
|
|
35
|
+
* returns a superset (incl. parent-domain cookies). slice(-2) is a cheap
|
|
36
|
+
* heuristic — it over-selects for multi-part eTLDs (co.uk) and as a substring
|
|
37
|
+
* match, so the precise RFC-6265 domain-match is what actually decides which
|
|
38
|
+
* cookies pass. Without it, browsing apple.com would match apple.com.evil.org
|
|
39
|
+
* and every *.co.uk site (verified).
|
|
40
|
+
* @param {string} url - URL whose host the cookies must match
|
|
41
|
+
* @param {object} [opts] - passed to extractCookies (e.g. { browser })
|
|
42
|
+
* @returns {Array<object>} cookies whose domain matches the URL host
|
|
43
|
+
*/
|
|
44
|
+
export function scopedCookiesForUrl(url: string, opts?: object): Array<object>;
|
|
28
45
|
/**
|
|
29
46
|
* Extract cookies for a URL and inject them into a CDP session.
|
|
30
|
-
* Convenience function combining
|
|
47
|
+
* Convenience function combining scopedCookiesForUrl + injectCookies.
|
|
31
48
|
* @param {object} session - CDP session handle
|
|
32
49
|
* @param {string} url - URL to extract cookies for
|
|
33
50
|
* @param {object} [opts] - Options passed to extractCookies
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the script.evaluate expression that reconstructs the AX tree in a
|
|
3
|
+
* context, assigning refs from `base`.
|
|
4
|
+
* @param {number} base - Starting ref offset (exclusive; first ref is base+1)
|
|
5
|
+
* @returns {string} JS expression returning JSON { tree, count }
|
|
6
|
+
*/
|
|
7
|
+
export function axSnapshotExpression(base: number): string;
|
|
8
|
+
/**
|
|
9
|
+
* ax-snapshot.js — Reconstruct a CDP-shaped accessibility tree in-page.
|
|
10
|
+
*
|
|
11
|
+
* CDP hands you Accessibility.getFullAXTree; BiDi has no equivalent, so on
|
|
12
|
+
* Firefox we rebuild an equivalent tree inside the page via script.evaluate.
|
|
13
|
+
* The output tree uses the SAME node shape and role vocabulary as
|
|
14
|
+
* buildTree()'s CDP output, so prune.js and aria.js consume it unchanged:
|
|
15
|
+
*
|
|
16
|
+
* { nodeId, role, name, properties, children, ignored }
|
|
17
|
+
*
|
|
18
|
+
* where `role` is CDP/ARIA vocabulary ('RootWebArea', 'StaticText', 'heading',
|
|
19
|
+
* 'link', 'button', 'img', 'paragraph', 'list', 'navigation', …), NOT tag
|
|
20
|
+
* names. The three things getFullAXTree gave us for free and we reimplement:
|
|
21
|
+
* 1. implicit ARIA role mapping (HTML element → role)
|
|
22
|
+
* 2. accessible-name computation (aria-labelledby → aria-label → native
|
|
23
|
+
* label/alt/legend/title → text content) — the POC proved textContent
|
|
24
|
+
* alone is NOT enough (img alt, <label>, aria-labelledby all missed).
|
|
25
|
+
* 3. hidden-subtree filtering (aria-hidden, display:none, visibility:hidden,
|
|
26
|
+
* the hidden attribute).
|
|
27
|
+
*
|
|
28
|
+
* Each kept element is tagged with a data-bb-ref attribute carrying its ref,
|
|
29
|
+
* so firefox-page.js (resolveRef) can resolve a ref back to its element via
|
|
30
|
+
* querySelector.
|
|
31
|
+
* Refs are assigned from a caller-supplied `base` so they stay globally unique
|
|
32
|
+
* across browsing contexts (iframes), matching CDP's flat integer refs.
|
|
33
|
+
*/
|
|
34
|
+
/** Attribute used to tag elements for ref → element resolution. */
|
|
35
|
+
export const REF_ATTR: "data-bb-ref";
|