@speedkit/cli 4.28.0 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +5 -2
- package/dist/commands/onboarding.js +2 -2
- package/dist/helpers/origin-proxy.d.ts +30 -0
- package/dist/helpers/origin-proxy.js +29 -0
- package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
- package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
- package/dist/services/browser-fetch/browser-response.d.ts +21 -0
- package/dist/services/browser-fetch/browser-response.js +86 -0
- package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-response.spec.js +90 -0
- package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
- package/dist/services/browser-fetch/device-emulation.js +187 -0
- package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
- package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
- package/dist/services/browser-fetch/index.d.ts +4 -0
- package/dist/services/browser-fetch/index.js +4 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
- package/dist/services/onboarding/onboarding-model.d.ts +3 -1
- package/dist/services/onboarding/onboarding-model.js +8 -2
- package/dist/services/onboarding/onboarding-service-factory.js +12 -1
- package/dist/services/onboarding/onboarding-service.d.ts +3 -1
- package/dist/services/onboarding/onboarding-service.js +7 -1
- package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
- package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
- package/dist/services/origin-request/asset-variations.d.ts +2 -0
- package/dist/services/origin-request/asset-variations.js +2 -2
- package/oclif.manifest.json +7 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
# [4.29.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.28.0...v4.29.0) (2026-09-17)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **onboarding:** add tablet and mobile support for browserFetch ([19bcd29](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/19bcd2946ca92517f2ff27081c34e598905cf493))
|
|
7
|
+
* **onboarding:** implements browserFetch as alternative to nodeFetch ([2f5abdc](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/2f5abdca38911c9af0be001586fabf30a61b5115))
|
|
8
|
+
|
|
1
9
|
# [4.28.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.27.2...v4.28.0) (2026-09-14)
|
|
2
10
|
|
|
3
11
|
|
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
|
|
|
21
21
|
$ sk COMMAND
|
|
22
22
|
running command...
|
|
23
23
|
$ sk (--version)
|
|
24
|
-
@speedkit/cli/4.
|
|
24
|
+
@speedkit/cli/4.29.0 linux-x64 node-v22.23.2
|
|
25
25
|
$ sk --help [COMMAND]
|
|
26
26
|
USAGE
|
|
27
27
|
$ sk COMMAND
|
|
@@ -688,7 +688,7 @@ Start a local dev environment for building and testing the Speed Kit configurati
|
|
|
688
688
|
```
|
|
689
689
|
USAGE
|
|
690
690
|
$ sk onboarding CUSTOMERPATH [-c <value>] [-t] [-d <value>] [-l] [-p] [--debugPort <value>] [-v] [-s]
|
|
691
|
-
[--headless]
|
|
691
|
+
[--headless] [--browserFetch]
|
|
692
692
|
|
|
693
693
|
ARGUMENTS
|
|
694
694
|
CUSTOMERPATH The customer config path
|
|
@@ -701,6 +701,9 @@ FLAGS
|
|
|
701
701
|
-s, --ignoreContentSecurityPolicy Removes header/meta "content-security-policy" for origin-responses
|
|
702
702
|
-t, --useTestConfig Override app with configured testApp
|
|
703
703
|
-v, --verboseLevel Show all messages
|
|
704
|
+
--browserFetch Fetch every origin document with a second headless Chrome instead of a direct
|
|
705
|
+
request. Only needed when bot protection refuses the CLI outright — a refusal is
|
|
706
|
+
otherwise detected and rerouted on its own.
|
|
704
707
|
--debugPort=<value> [default: 9222] Chrome remote debugging port (only used with -p/--headless). Use
|
|
705
708
|
distinct ports to run multiple onboarding processes at once.
|
|
706
709
|
--headless Run Chrome in headless mode (for CI environments). Implicitly enables remote
|
|
@@ -11,8 +11,8 @@ export default class Onboarding extends Command {
|
|
|
11
11
|
`$ sk onboarding ${CLIParametersExample.CustomerPath} -${CLIParametersChar.ConfigName} ${CLIParametersExample.ConfigName} -${CLIParametersChar.Domain} ${CLIParametersExample.Domain}`,
|
|
12
12
|
];
|
|
13
13
|
async run() {
|
|
14
|
-
const { args: { customerPath }, flags: { configName, domain, local, verboseLevel, ignoreContentSecurityPolicy, useTestConfig, debuggingPort, headless, debugPort, }, } = await this.parse(Onboarding);
|
|
15
|
-
const service = await new OnboardingServiceFactory(new OnboardingContext(customerPath, configName, domain, local, ignoreContentSecurityPolicy, this.config.root, verboseLevel, useTestConfig, debuggingPort, headless, debugPort), new CliConfig(this.config).load(customerPath)).getService();
|
|
14
|
+
const { args: { customerPath }, flags: { configName, domain, local, verboseLevel, ignoreContentSecurityPolicy, useTestConfig, debuggingPort, headless, debugPort, browserFetch, }, } = await this.parse(Onboarding);
|
|
15
|
+
const service = await new OnboardingServiceFactory(new OnboardingContext(customerPath, configName, domain, local, ignoreContentSecurityPolicy, this.config.root, verboseLevel, useTestConfig, debuggingPort, headless, debugPort, browserFetch), new CliConfig(this.config).load(customerPath)).getService();
|
|
16
16
|
const shutdown = new AbortController();
|
|
17
17
|
const onSignal = () => shutdown.abort();
|
|
18
18
|
// `once`: a second Ctrl+C falls through to Node's default handler and
|
|
@@ -22,6 +22,14 @@ export interface OriginFetchOptions {
|
|
|
22
22
|
*/
|
|
23
23
|
tolerateLegacyTls?: boolean;
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Replacement transport for `originFetch`.
|
|
27
|
+
*
|
|
28
|
+
* A transport receives the same arguments as `originFetch` and answers the same `Response`, so a
|
|
29
|
+
* call site never learns which one served it. `directOriginFetch` stays available to a transport
|
|
30
|
+
* that wants to try the network first.
|
|
31
|
+
*/
|
|
32
|
+
export type OriginFetchTransport = (url: string, init?: RequestInit, options?: OriginFetchOptions) => Promise<Response>;
|
|
25
33
|
/**
|
|
26
34
|
* Parses a Chrome `--proxy-server` value into a SOCKS proxy.
|
|
27
35
|
*
|
|
@@ -68,6 +76,21 @@ export declare function applyOriginProxy(chromeFlags?: string[], logger?: ProxyL
|
|
|
68
76
|
export declare function setOriginProxy(proxy?: OriginProxy, fromEnvironment?: boolean): void;
|
|
69
77
|
/** Drops the installed proxy, so the next call resolves it again. */
|
|
70
78
|
export declare function resetOriginProxy(): void;
|
|
79
|
+
/**
|
|
80
|
+
* The proxy every origin request currently goes through, whatever installed it.
|
|
81
|
+
*
|
|
82
|
+
* `withOriginProxyFlag` only hands Chrome a proxy the environment set, because a customer config
|
|
83
|
+
* already carries its own flag. A launch the CLI assembles itself has no such flag to inherit, so
|
|
84
|
+
* it needs the effective proxy instead.
|
|
85
|
+
*/
|
|
86
|
+
export declare function currentOriginProxy(): OriginProxy | undefined;
|
|
87
|
+
/**
|
|
88
|
+
* Routes every later `originFetch` through `next` instead of the network.
|
|
89
|
+
*
|
|
90
|
+
* Pass nothing to go back to the direct transport. Installing a transport does not touch the
|
|
91
|
+
* proxy: a transport that reaches the network itself is expected to honour it.
|
|
92
|
+
*/
|
|
93
|
+
export declare function setOriginFetchTransport(next?: OriginFetchTransport): void;
|
|
71
94
|
/**
|
|
72
95
|
* Appends the enforced proxy to the Chrome flags of a launch.
|
|
73
96
|
*
|
|
@@ -87,3 +110,10 @@ export declare function withOriginProxyFlag(chromeFlags?: string[]): string[];
|
|
|
87
110
|
* needs one runs through undici's own `fetch`. A plain direct request keeps the global `fetch`.
|
|
88
111
|
*/
|
|
89
112
|
export declare function originFetch(url: string, init?: RequestInit, options?: OriginFetchOptions): Promise<Response>;
|
|
113
|
+
/**
|
|
114
|
+
* Fetches a customer origin over the network, through the enforced SOCKS proxy.
|
|
115
|
+
*
|
|
116
|
+
* This is what `originFetch` does with no transport installed. Call it directly only from a
|
|
117
|
+
* transport that needs the network leg it replaces.
|
|
118
|
+
*/
|
|
119
|
+
export declare function directOriginFetch(url: string, init?: RequestInit, options?: OriginFetchOptions): Promise<Response>;
|
|
@@ -17,6 +17,7 @@ const CHROME_PROXY_FLAG = "--proxy-server=";
|
|
|
17
17
|
/** Port Chrome assumes for a SOCKS proxy that names none. */
|
|
18
18
|
const DEFAULT_SOCKS_PORT = 1080;
|
|
19
19
|
let state;
|
|
20
|
+
let transport;
|
|
20
21
|
/**
|
|
21
22
|
* Parses a Chrome `--proxy-server` value into a SOCKS proxy.
|
|
22
23
|
*
|
|
@@ -139,6 +140,25 @@ export function setOriginProxy(proxy, fromEnvironment = false) {
|
|
|
139
140
|
export function resetOriginProxy() {
|
|
140
141
|
state = undefined;
|
|
141
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* The proxy every origin request currently goes through, whatever installed it.
|
|
145
|
+
*
|
|
146
|
+
* `withOriginProxyFlag` only hands Chrome a proxy the environment set, because a customer config
|
|
147
|
+
* already carries its own flag. A launch the CLI assembles itself has no such flag to inherit, so
|
|
148
|
+
* it needs the effective proxy instead.
|
|
149
|
+
*/
|
|
150
|
+
export function currentOriginProxy() {
|
|
151
|
+
return currentState().proxy;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Routes every later `originFetch` through `next` instead of the network.
|
|
155
|
+
*
|
|
156
|
+
* Pass nothing to go back to the direct transport. Installing a transport does not touch the
|
|
157
|
+
* proxy: a transport that reaches the network itself is expected to honour it.
|
|
158
|
+
*/
|
|
159
|
+
export function setOriginFetchTransport(next) {
|
|
160
|
+
transport = next;
|
|
161
|
+
}
|
|
142
162
|
/**
|
|
143
163
|
* Appends the enforced proxy to the Chrome flags of a launch.
|
|
144
164
|
*
|
|
@@ -164,6 +184,15 @@ export function withOriginProxyFlag(chromeFlags = []) {
|
|
|
164
184
|
* needs one runs through undici's own `fetch`. A plain direct request keeps the global `fetch`.
|
|
165
185
|
*/
|
|
166
186
|
export function originFetch(url, init, options = {}) {
|
|
187
|
+
return (transport ?? directOriginFetch)(url, init, options);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Fetches a customer origin over the network, through the enforced SOCKS proxy.
|
|
191
|
+
*
|
|
192
|
+
* This is what `originFetch` does with no transport installed. Call it directly only from a
|
|
193
|
+
* transport that needs the network leg it replaces.
|
|
194
|
+
*/
|
|
195
|
+
export function directOriginFetch(url, init, options = {}) {
|
|
167
196
|
const dispatcher = originDispatcher(options.tolerateLegacyTls === true);
|
|
168
197
|
if (!dispatcher) {
|
|
169
198
|
return fetch(url, init);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
2
|
+
import { directOriginFetch } from "../../helpers/origin-proxy.js";
|
|
3
|
+
import { BrowserFetchLogger, BrowserOriginFetcher } from "./browser-origin-fetcher.js";
|
|
4
|
+
/**
|
|
5
|
+
* When the browser is used.
|
|
6
|
+
*
|
|
7
|
+
* `auto` keeps the network on the fast path and only reaches for the browser once an origin has
|
|
8
|
+
* refused a request; `always` sends everything through the browser from the first request.
|
|
9
|
+
*/
|
|
10
|
+
export type BrowserFetchMode = "auto" | "always";
|
|
11
|
+
/** The part of `BrowserOriginFetcher` the transport uses. */
|
|
12
|
+
export type OriginFetcherLike = Pick<BrowserOriginFetcher, "fetch" | "close">;
|
|
13
|
+
/** What an installed browser transport hands back, so the caller can shut it down again. */
|
|
14
|
+
export interface BrowserOriginFetchHandle {
|
|
15
|
+
readonly fetcher: OriginFetcherLike;
|
|
16
|
+
/** Restores the direct transport and closes the browser. */
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
export interface BrowserOriginFetchOptions {
|
|
20
|
+
userConfig: UserCliConfig;
|
|
21
|
+
logger?: BrowserFetchLogger;
|
|
22
|
+
mode?: BrowserFetchMode;
|
|
23
|
+
/**
|
|
24
|
+
* The browser to fetch with. Defaults to one this module owns; pass one to share a browser
|
|
25
|
+
* across installs, or to drive the policy without launching Chrome.
|
|
26
|
+
*/
|
|
27
|
+
fetcher?: OriginFetcherLike;
|
|
28
|
+
/** The network leg. Defaults to `directOriginFetch`, which is what production always wants. */
|
|
29
|
+
direct?: typeof directOriginFetch;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Routes `originFetch` through a real browser, for origins that refuse a server-side fetcher.
|
|
33
|
+
*
|
|
34
|
+
* In `auto` mode a host is fetched over the network until it answers with a blocked status, and
|
|
35
|
+
* from then on through the browser — the first refusal is what identifies the host, and repeating
|
|
36
|
+
* the network leg afterwards would only cost a request per fetch. A host that the browser cannot
|
|
37
|
+
* reach either falls back to whatever the network said, so the response still carries the status
|
|
38
|
+
* and headers the caller needs to report a cause.
|
|
39
|
+
*
|
|
40
|
+
* Installing replaces any transport installed before it. The returned handle is the only way back
|
|
41
|
+
* to the direct transport, so a caller that installs one has to close it.
|
|
42
|
+
*/
|
|
43
|
+
export declare function installBrowserOriginFetch(options: BrowserOriginFetchOptions): BrowserOriginFetchHandle;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { directOriginFetch, setOriginFetchTransport, } from "../../helpers/origin-proxy.js";
|
|
2
|
+
import { safe } from "../../helpers/safe.js";
|
|
3
|
+
import { isBlockedOriginStatus } from "../onboarding/virtual-orestes-app/origin-error-cause.js";
|
|
4
|
+
import { BrowserOriginFetcher, } from "./browser-origin-fetcher.js";
|
|
5
|
+
/**
|
|
6
|
+
* Routes `originFetch` through a real browser, for origins that refuse a server-side fetcher.
|
|
7
|
+
*
|
|
8
|
+
* In `auto` mode a host is fetched over the network until it answers with a blocked status, and
|
|
9
|
+
* from then on through the browser — the first refusal is what identifies the host, and repeating
|
|
10
|
+
* the network leg afterwards would only cost a request per fetch. A host that the browser cannot
|
|
11
|
+
* reach either falls back to whatever the network said, so the response still carries the status
|
|
12
|
+
* and headers the caller needs to report a cause.
|
|
13
|
+
*
|
|
14
|
+
* Installing replaces any transport installed before it. The returned handle is the only way back
|
|
15
|
+
* to the direct transport, so a caller that installs one has to close it.
|
|
16
|
+
*/
|
|
17
|
+
export function installBrowserOriginFetch(options) {
|
|
18
|
+
const mode = options.mode ?? "auto";
|
|
19
|
+
const direct = options.direct ?? directOriginFetch;
|
|
20
|
+
const fetcher = options.fetcher ??
|
|
21
|
+
new BrowserOriginFetcher(options.userConfig, options.logger);
|
|
22
|
+
/** Hosts a network fetch was refused by, which the browser serves from then on. */
|
|
23
|
+
const browserHosts = new Set();
|
|
24
|
+
const transport = async (url, init, fetchOptions) => {
|
|
25
|
+
const host = hostOf(url);
|
|
26
|
+
if (mode === "always" || (host && browserHosts.has(host))) {
|
|
27
|
+
const browserResponse = await safe(fetcher.fetch(url, init, fetchOptions));
|
|
28
|
+
if (browserResponse.success === true)
|
|
29
|
+
return browserResponse.data;
|
|
30
|
+
options.logger?.writeWarning(`[browserFetch]: the browser could not fetch ${url} (${browserResponse.error}), falling back to a direct request.`);
|
|
31
|
+
return direct(url, init, fetchOptions);
|
|
32
|
+
}
|
|
33
|
+
const networkResponse = await direct(url, init, fetchOptions);
|
|
34
|
+
if (!isBlockedOriginStatus(networkResponse.status))
|
|
35
|
+
return networkResponse;
|
|
36
|
+
options.logger?.writeWarning(`[browserFetch]: ${host} answered ${networkResponse.status} to a direct request — retrying through the browser, and routing this host through it from now on.`);
|
|
37
|
+
if (host)
|
|
38
|
+
browserHosts.add(host);
|
|
39
|
+
const retried = await safe(fetcher.fetch(url, init, fetchOptions));
|
|
40
|
+
if (retried.success === true)
|
|
41
|
+
return retried.data;
|
|
42
|
+
options.logger?.writeWarning(`[browserFetch]: the browser could not fetch ${url} either (${retried.error}).`);
|
|
43
|
+
// The refusal is the honest answer at this point, and the caller classifies it into a cause.
|
|
44
|
+
return networkResponse;
|
|
45
|
+
};
|
|
46
|
+
setOriginFetchTransport(transport);
|
|
47
|
+
return {
|
|
48
|
+
fetcher,
|
|
49
|
+
async close() {
|
|
50
|
+
setOriginFetchTransport();
|
|
51
|
+
await fetcher.close();
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/** Host of an absolute URL, or undefined when it is not one. A bad URL is the transport's problem. */
|
|
56
|
+
function hostOf(url) {
|
|
57
|
+
try {
|
|
58
|
+
return new URL(url).host;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { afterEach, beforeEach, describe, it } from "mocha";
|
|
3
|
+
import { originFetch, resetOriginProxy, setOriginFetchTransport, } from "../../helpers/origin-proxy.js";
|
|
4
|
+
import { installBrowserOriginFetch, } from "./browser-origin-fetch.js";
|
|
5
|
+
/** Records what each leg was asked for, so the policy can be asserted on the calls it made. */
|
|
6
|
+
class FetchLog {
|
|
7
|
+
direct = [];
|
|
8
|
+
browser = [];
|
|
9
|
+
}
|
|
10
|
+
function stubFetcher(log, answer) {
|
|
11
|
+
return {
|
|
12
|
+
fetch: (url) => {
|
|
13
|
+
log.browser.push(url);
|
|
14
|
+
return answer(url);
|
|
15
|
+
},
|
|
16
|
+
close: () => Promise.resolve(),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
const warnings = [];
|
|
20
|
+
const logger = {
|
|
21
|
+
writeWarning: (message) => warnings.push(message),
|
|
22
|
+
comment: () => undefined,
|
|
23
|
+
};
|
|
24
|
+
/** The fetcher is always stubbed in these specs, so no Chrome is ever launched from them. */
|
|
25
|
+
const userConfig = {};
|
|
26
|
+
describe("installBrowserOriginFetch", () => {
|
|
27
|
+
let handle;
|
|
28
|
+
let log;
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
log = new FetchLog();
|
|
31
|
+
warnings.length = 0;
|
|
32
|
+
resetOriginProxy();
|
|
33
|
+
});
|
|
34
|
+
afterEach(async () => {
|
|
35
|
+
await handle?.close();
|
|
36
|
+
handle = undefined;
|
|
37
|
+
setOriginFetchTransport();
|
|
38
|
+
});
|
|
39
|
+
describe("auto mode", () => {
|
|
40
|
+
it("leaves an origin that answers on the network alone", async () => {
|
|
41
|
+
handle = installBrowserOriginFetch({
|
|
42
|
+
userConfig,
|
|
43
|
+
logger,
|
|
44
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
45
|
+
direct: (url) => {
|
|
46
|
+
log.direct.push(url);
|
|
47
|
+
return Promise.resolve(new Response("network", { status: 200 }));
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
const response = await originFetch("https://example.com/a");
|
|
51
|
+
expect(await response.text()).to.equal("network");
|
|
52
|
+
expect(log.browser).to.deep.equal([]);
|
|
53
|
+
});
|
|
54
|
+
it("retries through the browser once the origin refuses the request", async () => {
|
|
55
|
+
handle = installBrowserOriginFetch({
|
|
56
|
+
userConfig,
|
|
57
|
+
logger,
|
|
58
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
59
|
+
direct: (url) => {
|
|
60
|
+
log.direct.push(url);
|
|
61
|
+
return Promise.resolve(new Response("blocked", { status: 403 }));
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
const response = await originFetch("https://example.com/a");
|
|
65
|
+
expect(await response.text()).to.equal("browser");
|
|
66
|
+
expect(log.browser).to.deep.equal(["https://example.com/a"]);
|
|
67
|
+
});
|
|
68
|
+
it("keeps a refused host on the browser, instead of paying for the network leg again", async () => {
|
|
69
|
+
handle = installBrowserOriginFetch({
|
|
70
|
+
userConfig,
|
|
71
|
+
logger,
|
|
72
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
73
|
+
direct: (url) => {
|
|
74
|
+
log.direct.push(url);
|
|
75
|
+
return Promise.resolve(new Response("blocked", { status: 403 }));
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
await originFetch("https://example.com/a");
|
|
79
|
+
await originFetch("https://example.com/b");
|
|
80
|
+
expect(log.direct).to.deep.equal(["https://example.com/a"]);
|
|
81
|
+
expect(log.browser).to.deep.equal([
|
|
82
|
+
"https://example.com/a",
|
|
83
|
+
"https://example.com/b",
|
|
84
|
+
]);
|
|
85
|
+
});
|
|
86
|
+
it("leaves another host on the network, because only the refused one is rerouted", async () => {
|
|
87
|
+
handle = installBrowserOriginFetch({
|
|
88
|
+
userConfig,
|
|
89
|
+
logger,
|
|
90
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
91
|
+
direct: (url) => {
|
|
92
|
+
log.direct.push(url);
|
|
93
|
+
return Promise.resolve(new Response("network", {
|
|
94
|
+
status: url.includes("blocked.example") ? 403 : 200,
|
|
95
|
+
}));
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
await originFetch("https://blocked.example/a");
|
|
99
|
+
await originFetch("https://open.example/a");
|
|
100
|
+
expect(log.browser).to.deep.equal(["https://blocked.example/a"]);
|
|
101
|
+
});
|
|
102
|
+
it("does not reroute an ordinary origin error", async () => {
|
|
103
|
+
handle = installBrowserOriginFetch({
|
|
104
|
+
userConfig,
|
|
105
|
+
logger,
|
|
106
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
107
|
+
direct: () => Promise.resolve(new Response("gone", { status: 404 })),
|
|
108
|
+
});
|
|
109
|
+
const response = await originFetch("https://example.com/a");
|
|
110
|
+
expect(response.status).to.equal(404);
|
|
111
|
+
expect(log.browser).to.deep.equal([]);
|
|
112
|
+
});
|
|
113
|
+
it("answers the refusal when the browser cannot reach the origin either", async () => {
|
|
114
|
+
handle = installBrowserOriginFetch({
|
|
115
|
+
userConfig,
|
|
116
|
+
logger,
|
|
117
|
+
fetcher: stubFetcher(log, () => Promise.reject(new Error("navigation timeout"))),
|
|
118
|
+
direct: () => Promise.resolve(new Response("blocked", { status: 403 })),
|
|
119
|
+
});
|
|
120
|
+
const response = await originFetch("https://example.com/a");
|
|
121
|
+
// The status is what the caller classifies into a cause, so it has to survive the retry.
|
|
122
|
+
expect(response.status).to.equal(403);
|
|
123
|
+
expect(warnings.join(" ")).to.contain("could not fetch");
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
describe("always mode", () => {
|
|
127
|
+
it("sends the first request through the browser", async () => {
|
|
128
|
+
handle = installBrowserOriginFetch({
|
|
129
|
+
userConfig,
|
|
130
|
+
logger,
|
|
131
|
+
mode: "always",
|
|
132
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
133
|
+
direct: (url) => {
|
|
134
|
+
log.direct.push(url);
|
|
135
|
+
return Promise.resolve(new Response("network"));
|
|
136
|
+
},
|
|
137
|
+
});
|
|
138
|
+
const response = await originFetch("https://example.com/a");
|
|
139
|
+
expect(await response.text()).to.equal("browser");
|
|
140
|
+
expect(log.direct).to.deep.equal([]);
|
|
141
|
+
});
|
|
142
|
+
it("falls back to the network when the browser fails", async () => {
|
|
143
|
+
handle = installBrowserOriginFetch({
|
|
144
|
+
userConfig,
|
|
145
|
+
logger,
|
|
146
|
+
mode: "always",
|
|
147
|
+
fetcher: stubFetcher(log, () => Promise.reject(new Error("no chrome"))),
|
|
148
|
+
direct: (url) => {
|
|
149
|
+
log.direct.push(url);
|
|
150
|
+
return Promise.resolve(new Response("network"));
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
const response = await originFetch("https://example.com/a");
|
|
154
|
+
expect(await response.text()).to.equal("network");
|
|
155
|
+
expect(warnings.join(" ")).to.contain("falling back to a direct request");
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
it("restores the direct transport when the handle is closed", async () => {
|
|
159
|
+
const installed = installBrowserOriginFetch({
|
|
160
|
+
userConfig,
|
|
161
|
+
logger,
|
|
162
|
+
mode: "always",
|
|
163
|
+
fetcher: stubFetcher(log, () => Promise.resolve(new Response("browser"))),
|
|
164
|
+
direct: () => Promise.resolve(new Response("network")),
|
|
165
|
+
});
|
|
166
|
+
await installed.close();
|
|
167
|
+
setOriginFetchTransport((url) => {
|
|
168
|
+
log.direct.push(url);
|
|
169
|
+
return Promise.resolve(new Response("after-close"));
|
|
170
|
+
});
|
|
171
|
+
expect(await (await originFetch("https://example.com/a")).text()).to.equal("after-close");
|
|
172
|
+
expect(log.browser).to.deep.equal([]);
|
|
173
|
+
});
|
|
174
|
+
});
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { Frame, HTTPRequest } from "puppeteer";
|
|
2
|
+
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
3
|
+
import { OriginFetchOptions } from "../../helpers/origin-proxy.js";
|
|
4
|
+
/** Only the writers this fetcher uses, so it does not depend on the whole CLI service. */
|
|
5
|
+
export interface BrowserFetchLogger {
|
|
6
|
+
writeWarning(message: string): void;
|
|
7
|
+
comment(message: string): void;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Whether a request is the document the fetch was for, rather than something the page went on to
|
|
11
|
+
* want.
|
|
12
|
+
*
|
|
13
|
+
* A redirect hop is a navigation request too, so a redirected fetch still reaches its destination.
|
|
14
|
+
* The main-frame check is what keeps an iframe's document out: an ad frame's document is a
|
|
15
|
+
* subresource of this fetch, whatever it is to the frame.
|
|
16
|
+
*
|
|
17
|
+
* @param request - Request Chrome paused.
|
|
18
|
+
* @param mainFrame - The page's main frame.
|
|
19
|
+
*/
|
|
20
|
+
export declare function isDocumentRequest(request: Pick<HTTPRequest, "isNavigationRequest" | "frame">, mainFrame: Frame): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Fetches customer origins with a real browser instead of a socket.
|
|
23
|
+
*
|
|
24
|
+
* Bot protection scores the client, not the request: a managed challenge answers Chrome and
|
|
25
|
+
* refuses every server-side fetcher, whatever library, HTTP version or header set it uses — a
|
|
26
|
+
* `cf_clearance` cookie lifted out of a browser does not travel either, because it is bound to the
|
|
27
|
+
* client that earned it. A browser is therefore the only transport that reaches such an origin, so
|
|
28
|
+
* this fetcher drives one and adapts its answers back into `Response`.
|
|
29
|
+
*
|
|
30
|
+
* What it costs, and why it is not the default:
|
|
31
|
+
* - a second Chrome, launched lazily on the first request and held until `close()`;
|
|
32
|
+
* - one page per origin host and device, kept open so the clearance cookie stays warm —
|
|
33
|
+
* cross-origin requests from a parked page are refused by CORS, so each host needs its own;
|
|
34
|
+
* - requests to one host are serialised, because a page navigates to one document at a time.
|
|
35
|
+
*
|
|
36
|
+
* Headless is safe here only because of `HEADLESS_PARITY_ARGS` and the headed user agent: measured
|
|
37
|
+
* against a Cloudflare-protected origin, plain headless Chrome is hard-blocked on the first
|
|
38
|
+
* response and never recovers, while the same launch with the parity treatment is answered 200
|
|
39
|
+
* without ever seeing an interstitial. The user agent is the discriminator, so both are load-bearing.
|
|
40
|
+
*
|
|
41
|
+
* That is also why no caller's user agent reaches the origin. The Asset API's device variations are
|
|
42
|
+
* reproduced by emulating the device instead — viewport, touch and client hints included — with a
|
|
43
|
+
* user agent built from this very Chrome. A `(compatible; SpeedKit/1.0)` suffix, or any string that
|
|
44
|
+
* disagrees with what the engine answers to a fingerprint, gives the browser away and costs exactly
|
|
45
|
+
* the access this fetcher exists to get. See `device-emulation.ts`.
|
|
46
|
+
*/
|
|
47
|
+
export declare class BrowserOriginFetcher {
|
|
48
|
+
private readonly userConfig;
|
|
49
|
+
private readonly logger?;
|
|
50
|
+
private readonly navigationTimeout;
|
|
51
|
+
private browser?;
|
|
52
|
+
/** In flight during the launch, so concurrent first requests share one browser. */
|
|
53
|
+
private launch?;
|
|
54
|
+
/** Keyed by device and host, because emulation belongs to a page and outlives a request. */
|
|
55
|
+
private readonly pages;
|
|
56
|
+
/** Tail of the request chain per host, which is how requests to one host are serialised. */
|
|
57
|
+
private readonly queues;
|
|
58
|
+
/** Version of the running Chrome, which every emulated user agent is built from. */
|
|
59
|
+
private browserVersion?;
|
|
60
|
+
private readonly warned;
|
|
61
|
+
/**
|
|
62
|
+
* Pages currently allowed to load anything, which is only ever true while a challenge is being
|
|
63
|
+
* waited out. A protection layer other than Cloudflare serves its interstitial from paths this
|
|
64
|
+
* fetcher cannot know, so waiting for one has to lift the document-only rule.
|
|
65
|
+
*/
|
|
66
|
+
private readonly permissive;
|
|
67
|
+
private closed;
|
|
68
|
+
/** Profile directory of the running browser, removed with it. */
|
|
69
|
+
private profile?;
|
|
70
|
+
/** When the session was last thrown away, so recovery cannot run per request. */
|
|
71
|
+
private lastRecycle;
|
|
72
|
+
constructor(userConfig: UserCliConfig, logger?: BrowserFetchLogger, navigationTimeout?: number);
|
|
73
|
+
/**
|
|
74
|
+
* Fetches one origin URL through the browser.
|
|
75
|
+
*
|
|
76
|
+
* `options.tolerateLegacyTls` has no counterpart here: Chrome decides for itself what handshake
|
|
77
|
+
* it accepts, and it is already told to ignore certificate errors.
|
|
78
|
+
*
|
|
79
|
+
* @param url - Absolute origin URL.
|
|
80
|
+
* @param init - Request headers and redirect mode. Other `RequestInit` fields do not apply: a
|
|
81
|
+
* navigation is always a GET.
|
|
82
|
+
*/
|
|
83
|
+
fetch(url: string, init?: RequestInit, options?: OriginFetchOptions): Promise<Response>;
|
|
84
|
+
/** Closes the browser and removes its profile. A later `fetch` launches a new one. */
|
|
85
|
+
close(): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* Throws the running browser and its profile away, leaving the fetcher usable.
|
|
88
|
+
*
|
|
89
|
+
* Everything derived from that browser goes with it: its pages, the permissive marks on them,
|
|
90
|
+
* and the user agent read from its version.
|
|
91
|
+
*/
|
|
92
|
+
private discardBrowser;
|
|
93
|
+
/**
|
|
94
|
+
* Starts a new session after the origin refused this one, and reports whether it did.
|
|
95
|
+
*
|
|
96
|
+
* The refusal is recorded against the session's cookies, so a new profile is the only way back.
|
|
97
|
+
* Rate-limited by `RECYCLE_COOLDOWN_MS`: against an origin that refuses us however we come, the
|
|
98
|
+
* answer has to settle on "refused" rather than launch a browser per request.
|
|
99
|
+
*/
|
|
100
|
+
private recycleSession;
|
|
101
|
+
/**
|
|
102
|
+
* Runs `task` after every request already queued for `host`.
|
|
103
|
+
*
|
|
104
|
+
* The chain is kept whatever a task does, so one failed request does not strand the ones behind
|
|
105
|
+
* it, and the tail is dropped once it is the last link — otherwise the map would grow a rejected
|
|
106
|
+
* promise per host for the lifetime of the process.
|
|
107
|
+
*/
|
|
108
|
+
private enqueue;
|
|
109
|
+
private fetchOnPage;
|
|
110
|
+
/**
|
|
111
|
+
* Repeats the fetch on a browser started after the refusal, as the same device.
|
|
112
|
+
*
|
|
113
|
+
* Throws when the replacement is refused too, so the caller falls back to reporting what the
|
|
114
|
+
* first attempt said.
|
|
115
|
+
*/
|
|
116
|
+
private fetchFresh;
|
|
117
|
+
/**
|
|
118
|
+
* Navigates the page, waiting for the document rather than for the whole page.
|
|
119
|
+
*
|
|
120
|
+
* `domcontentloaded` rather than `commit`: a challenge only clears itself once its script has
|
|
121
|
+
* run, and a response that needs no challenge is already complete by then anyway.
|
|
122
|
+
*
|
|
123
|
+
* The page's identity is not touched here — it was emulated once when the page was opened, and
|
|
124
|
+
* changing a user agent between two navigations of one page is itself a signal.
|
|
125
|
+
*/
|
|
126
|
+
private navigate;
|
|
127
|
+
/**
|
|
128
|
+
* Waits out a bot-protection interstitial and reports what the origin answered afterwards.
|
|
129
|
+
*
|
|
130
|
+
* An interstitial replaces itself with the real document once its scripts have run, so the page
|
|
131
|
+
* is polled rather than reloaded — a reload would start the challenge over. The document-only
|
|
132
|
+
* rule is lifted for the wait, because those scripts are the whole mechanism. When the wait runs
|
|
133
|
+
* out, the blocked response is returned as it is: the caller has a status to report and a cause
|
|
134
|
+
* to name, which is more useful than an exception.
|
|
135
|
+
*/
|
|
136
|
+
private afterChallenge;
|
|
137
|
+
private waitOutChallenge;
|
|
138
|
+
/**
|
|
139
|
+
* Reads the response body and hands back a `Response`.
|
|
140
|
+
*
|
|
141
|
+
* With `redirect: "manual"` the first hop is answered instead of the destination: Chrome always
|
|
142
|
+
* follows a redirect, so the hop is recovered from the request's redirect chain. A body is never
|
|
143
|
+
* available for it, which matches what a manual redirect carries anyway.
|
|
144
|
+
*/
|
|
145
|
+
private adapt;
|
|
146
|
+
/**
|
|
147
|
+
* Whether the origin answered, or its protection layer did — and if so, whether waiting helps.
|
|
148
|
+
*
|
|
149
|
+
* The status alone decides nothing: an origin may answer 403 for its own reasons, and a
|
|
150
|
+
* protection layer may answer 200 with an interstitial. So Cloudflare's own marker and the
|
|
151
|
+
* document's title are what separate the three outcomes.
|
|
152
|
+
*
|
|
153
|
+
* Read out of the response body rather than with `page.title()`, which would evaluate script in
|
|
154
|
+
* a page a bot detector is watching. Puppeteer memoises the body, so the read `adapt` does later
|
|
155
|
+
* costs nothing. The body is only searched as far as a `<title>` can reach.
|
|
156
|
+
*/
|
|
157
|
+
private verdictOn;
|
|
158
|
+
/**
|
|
159
|
+
* The start of the response body, which is as far as a `<title>` can be.
|
|
160
|
+
*
|
|
161
|
+
* A body that cannot be read is not a verdict either way, so it reads as empty and the status
|
|
162
|
+
* and headers decide alone.
|
|
163
|
+
*/
|
|
164
|
+
private headOf;
|
|
165
|
+
/** The tab's title, or nothing when the page is in a state that cannot report one. */
|
|
166
|
+
private titleOf;
|
|
167
|
+
/**
|
|
168
|
+
* The page this host is fetched on as this device.
|
|
169
|
+
*
|
|
170
|
+
* A device gets its own page rather than a re-emulated one: emulation is page state, and a page
|
|
171
|
+
* that changes device between two navigations tells the origin more about us than either device
|
|
172
|
+
* would have.
|
|
173
|
+
*/
|
|
174
|
+
private pageFor;
|
|
175
|
+
/**
|
|
176
|
+
* Opens a page that loads documents and nothing else.
|
|
177
|
+
*
|
|
178
|
+
* Without this a fetch costs the origin a whole page load — measured against one customer, 87
|
|
179
|
+
* subresource requests for the home page and 182 for a category page. Three such fetches in a
|
|
180
|
+
* row were enough for that origin to blackhole the session, and every request after that was
|
|
181
|
+
* refused. Aborting subresources keeps one fetch to one origin request, which is both what the
|
|
182
|
+
* callers want and what keeps the fetcher from behaving like an attack. It also makes a fetch
|
|
183
|
+
* 3-5x faster, because nothing waits for images or scripts.
|
|
184
|
+
*
|
|
185
|
+
* The rule also keeps the browser out of Cloudflare's JS fingerprinting, which is itself a
|
|
186
|
+
* subresource: measured against the same customer, exempting `/cdn-cgi/` let the detection run,
|
|
187
|
+
* and it hard-blocked the session from the third request onwards — where aborting it left eight
|
|
188
|
+
* consecutive fetches answered. `afterChallenge` lifts the rule only when an interstitial has
|
|
189
|
+
* actually appeared, which is the one case those requests are needed for.
|
|
190
|
+
*/
|
|
191
|
+
private openPage;
|
|
192
|
+
/**
|
|
193
|
+
* Makes the page present one device, for as long as it lives.
|
|
194
|
+
*
|
|
195
|
+
* The viewport is left alone for a desktop fetch, where the launch's `defaultViewport: null`
|
|
196
|
+
* already hands the page the window's own metrics. `setUserAgent` carries the client hints with
|
|
197
|
+
* it, which `page.emulate()` would not.
|
|
198
|
+
*
|
|
199
|
+
* One difference to a real phone survives this: `HEADLESS_PARITY_ARGS` pins Blink's hover and
|
|
200
|
+
* pointer types to the desktop values browser-wide, so a mobile page still answers
|
|
201
|
+
* `hover: hover` / `pointer: fine`. That decides CSS, not the bytes of the document being
|
|
202
|
+
* fetched, and the parity args are what gets the browser answered at all.
|
|
203
|
+
*/
|
|
204
|
+
private emulate;
|
|
205
|
+
private browserInstance;
|
|
206
|
+
private launchBrowser;
|
|
207
|
+
/**
|
|
208
|
+
* Creates this browser's own profile directory, pruning anything a crashed run left behind.
|
|
209
|
+
*
|
|
210
|
+
* Pruning is by age rather than by wiping the parent: two `sk onboarding` sessions can run at
|
|
211
|
+
* once, and one must not delete the profile the other is holding open.
|
|
212
|
+
*/
|
|
213
|
+
private createProfile;
|
|
214
|
+
private pruneProfiles;
|
|
215
|
+
private warnOnce;
|
|
216
|
+
}
|