@speedkit/cli 4.27.2 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +5 -2
  3. package/dist/commands/onboarding.js +2 -2
  4. package/dist/helpers/origin-proxy.d.ts +30 -0
  5. package/dist/helpers/origin-proxy.js +29 -0
  6. package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
  7. package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
  8. package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
  9. package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
  10. package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
  11. package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
  12. package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
  13. package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
  14. package/dist/services/browser-fetch/browser-response.d.ts +21 -0
  15. package/dist/services/browser-fetch/browser-response.js +86 -0
  16. package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
  17. package/dist/services/browser-fetch/browser-response.spec.js +90 -0
  18. package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
  19. package/dist/services/browser-fetch/device-emulation.js +187 -0
  20. package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
  21. package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
  22. package/dist/services/browser-fetch/index.d.ts +4 -0
  23. package/dist/services/browser-fetch/index.js +4 -0
  24. package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
  25. package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
  26. package/dist/services/onboarding/dashboard/index.js +40 -1
  27. package/dist/services/onboarding/onboarding-model.d.ts +3 -1
  28. package/dist/services/onboarding/onboarding-model.js +8 -2
  29. package/dist/services/onboarding/onboarding-service-factory.js +12 -1
  30. package/dist/services/onboarding/onboarding-service.d.ts +3 -1
  31. package/dist/services/onboarding/onboarding-service.js +7 -1
  32. package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
  33. package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
  34. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
  35. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
  36. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
  37. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
  38. package/dist/services/origin-request/asset-variations.d.ts +2 -0
  39. package/dist/services/origin-request/asset-variations.js +2 -2
  40. package/dist/services/query-builder/query-builder-service.d.ts +18 -2
  41. package/dist/services/query-builder/query-builder-service.js +26 -10
  42. package/oclif.manifest.json +7 -1
  43. package/package.json +1 -1
@@ -0,0 +1,90 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import { toFetchResponse, toPlainHeaders } from "./browser-response.js";
4
+ /**
5
+ * The four accessors `toFetchResponse` reads. Chrome reports repeated headers newline-joined in a
6
+ * flat map, which is the shape this stands in for.
7
+ */
8
+ function chromeResponse(overrides) {
9
+ return {
10
+ status: () => overrides.status ?? 200,
11
+ statusText: () => overrides.statusText ?? "OK",
12
+ url: () => overrides.url ?? "https://example.com/",
13
+ headers: () => overrides.headers ?? {},
14
+ };
15
+ }
16
+ describe("toFetchResponse", () => {
17
+ it("carries status, body and url over", async () => {
18
+ const response = toFetchResponse(chromeResponse({
19
+ status: 201,
20
+ statusText: "Created",
21
+ url: "https://example.com/a",
22
+ }), Buffer.from("<html>hi</html>"), false);
23
+ expect(response.status).to.equal(201);
24
+ expect(response.statusText).to.equal("Created");
25
+ expect(await response.text()).to.equal("<html>hi</html>");
26
+ });
27
+ it("reports the url, which is what the CDN-redirect check compares against", () => {
28
+ // A constructed Response has an empty url, and VirtualOrestesApp reads
29
+ // `fetchedUrl !== response.url` — left empty, every request would look like a redirect.
30
+ const response = toFetchResponse(chromeResponse({ url: "https://example.com/final" }), Buffer.alloc(0), false);
31
+ expect(response.url).to.equal("https://example.com/final");
32
+ });
33
+ it("reports whether the request was redirected", () => {
34
+ expect(toFetchResponse(chromeResponse({}), Buffer.alloc(0), true).redirected).to.equal(true);
35
+ expect(toFetchResponse(chromeResponse({}), Buffer.alloc(0), false).redirected).to.equal(false);
36
+ });
37
+ it("keeps a body of raw origin bytes intact, so charset detection still works", async () => {
38
+ // Shift-JIS "テスト": the bytes must survive untranscoded for iconv to decode them later.
39
+ const shiftJis = Buffer.from([0x83, 0x65, 0x83, 0x58, 0x83, 0x67]);
40
+ const response = toFetchResponse(chromeResponse({}), shiftJis, false);
41
+ expect(Buffer.from(await response.arrayBuffer())).to.deep.equal(shiftJis);
42
+ });
43
+ it("splits the set-cookie header Chrome folded into one value", () => {
44
+ const response = toFetchResponse(chromeResponse({
45
+ headers: { "set-cookie": "a=1; Path=/\nb=2; Path=/" },
46
+ }), Buffer.alloc(0), false);
47
+ expect(response.headers.getSetCookie()).to.deep.equal([
48
+ "a=1; Path=/",
49
+ "b=2; Path=/",
50
+ ]);
51
+ });
52
+ it("drops the headers that describe a body Chrome already unwrapped", () => {
53
+ const response = toFetchResponse(chromeResponse({
54
+ headers: {
55
+ "content-encoding": "br",
56
+ "content-length": "42",
57
+ "content-type": "text/html; charset=utf-8",
58
+ },
59
+ }), Buffer.from("decompressed"), false);
60
+ expect(response.headers.get("content-encoding")).to.equal(null);
61
+ expect(response.headers.get("content-length")).to.equal(null);
62
+ expect(response.headers.get("content-type")).to.equal("text/html; charset=utf-8");
63
+ });
64
+ it("keeps an empty header value rather than dropping the header", () => {
65
+ const response = toFetchResponse(chromeResponse({ headers: { "x-empty": "" } }), Buffer.alloc(0), false);
66
+ expect(response.headers.has("x-empty")).to.equal(true);
67
+ });
68
+ it("omits the body for a status the Response constructor refuses one for", () => {
69
+ const response = toFetchResponse(chromeResponse({ status: 304, statusText: "Not Modified" }), Buffer.from("ignored"), false);
70
+ expect(response.status).to.equal(304);
71
+ expect(response.body).to.equal(null);
72
+ });
73
+ });
74
+ describe("toPlainHeaders", () => {
75
+ it("lowercases names, so a caller's User-Agent cannot survive alongside ours", () => {
76
+ expect(toPlainHeaders({ "User-Agent": "caller" })).to.deep.equal({
77
+ "user-agent": "caller",
78
+ });
79
+ });
80
+ it("accepts the three HeadersInit shapes", () => {
81
+ expect(toPlainHeaders(new Headers({ "x-a": "1" }))).to.deep.equal({
82
+ "x-a": "1",
83
+ });
84
+ expect(toPlainHeaders([["X-A", "1"]])).to.deep.equal({ "x-a": "1" });
85
+ expect(toPlainHeaders({ "X-A": "1" })).to.deep.equal({ "x-a": "1" });
86
+ });
87
+ it("answers an empty map for no headers", () => {
88
+ expect(toPlainHeaders()).to.deep.equal({});
89
+ });
90
+ });
@@ -0,0 +1,46 @@
1
+ import { Protocol, Viewport } from "puppeteer";
2
+ /** The devices the browser fetcher can present, one per Speed Kit device variation. */
3
+ export type FetchDevice = "desktop" | "mobile" | "tablet";
4
+ /** Everything a page needs to look like one device, as `openPage` applies it. */
5
+ export interface DeviceEmulation {
6
+ readonly device: FetchDevice;
7
+ /** Undefined when the Chrome version string carried no version, as for the headed user agent. */
8
+ readonly userAgent?: string;
9
+ /**
10
+ * Client hints to send with that user agent. Always set alongside one, because overriding the
11
+ * user agent switches Chrome's own hints off.
12
+ */
13
+ readonly metadata?: Protocol.Emulation.UserAgentMetadata;
14
+ /** Null leaves the window's own viewport in place, which is what a desktop fetch wants. */
15
+ readonly viewport: Viewport | null;
16
+ }
17
+ /** True for one of the user agents the Asset API sends for a device variation. */
18
+ export declare function isSpeedKitUserAgent(userAgent?: string): boolean;
19
+ /**
20
+ * The device a request is for, read off the user agent the caller would have sent.
21
+ *
22
+ * The user agent is the signal rather than the variation name, because it is what the origin's own
23
+ * device detection sees: a customer variation named `mobile` that overrides the user agent with a
24
+ * desktop one is answered with the desktop document in production, and has to be here too. The
25
+ * three defaults are matched exactly, and anything else falls back to reading its tokens, so a
26
+ * customer's own mobile user agent still selects a phone.
27
+ *
28
+ * @param userAgent - The caller's `user-agent` header, if it set one.
29
+ * @returns The device, or undefined when there is no user agent to read.
30
+ */
31
+ export declare function deviceForUserAgent(userAgent?: string): FetchDevice | undefined;
32
+ /**
33
+ * Builds the emulation for one device from the running Chrome's version.
34
+ *
35
+ * The user agent is derived from the binary rather than taken from a table, for the same reason
36
+ * `buildHeadedUserAgent` derives the desktop one: only the major version moves, and a stale one is
37
+ * exactly what a protection layer scores. Both mobile devices therefore present *Chrome on
38
+ * Android* — an iOS user agent would name Safari while the engine underneath answers every
39
+ * fingerprint as Blink, which is the mismatch these fetches exist to avoid. The cost is that an
40
+ * origin serving per-OS markup answers a mobile fetch with its Android document, where the Asset
41
+ * API would have asked as an iPhone.
42
+ *
43
+ * @param browserVersionString - `browser.version()`, e.g. `Chrome/146.0.7680.165`.
44
+ * @param device - The device to present.
45
+ */
46
+ export declare function deviceEmulation(browserVersionString: string, device: FetchDevice): DeviceEmulation;
@@ -0,0 +1,187 @@
1
+ import { buildHeadedUserAgent, chromeMajorVersion, } from "../onboarding/browser/executable/headed-user-agent.js";
2
+ import { DEFAULT_USER_AGENT, MOBILE_USER_AGENT, TABLET_USER_AGENT, } from "../origin-request/asset-variations.js";
3
+ /**
4
+ * Android's frozen platform token. Chrome's UA reduction pins every Android build to
5
+ * `Android 10; K` regardless of the device, so this is the literal string real Chrome sends —
6
+ * see https://www.chromium.org/updates/ua-reduction/.
7
+ */
8
+ const ANDROID_PLATFORM = "Linux; Android 10; K";
9
+ /** The `sec-ch-ua-platform-version` that goes with the frozen token above. */
10
+ const ANDROID_PLATFORM_VERSION = "10.0.0";
11
+ /** The `sec-ch-ua-model` that goes with it. */
12
+ const ANDROID_MODEL = "K";
13
+ /**
14
+ * The desktop platform hints per Node platform, matching the token `buildHeadedUserAgent` puts in
15
+ * the user agent for the same host.
16
+ *
17
+ * `platformVersion` is a high-entropy hint, so it only leaves when an origin asks for it through
18
+ * `Accept-CH`. Chrome on Linux answers that one with an empty string; the other two carry the
19
+ * version of a current OS build, because an empty one there would be the odd answer.
20
+ */
21
+ const DESKTOP_PLATFORMS = {
22
+ darwin: { platform: "macOS", version: "15.0.0" },
23
+ win32: { platform: "Windows", version: "19.0.0" },
24
+ };
25
+ const DEFAULT_DESKTOP_PLATFORM = { platform: "Linux", version: "" };
26
+ /**
27
+ * The GREASE brand Chrome mixes into `Sec-CH-UA` so parsers cannot assume a fixed list. The exact
28
+ * value rotates between Chrome versions; any of them is accepted, and leaving it out is the thing
29
+ * that stands out.
30
+ */
31
+ const GREASE_BRAND = { brand: "Not=A?Brand", version: "24" };
32
+ /**
33
+ * Viewports for the emulated devices. Deliberately not puppeteer's `KnownDevices`: those carry a
34
+ * frozen user agent each, and the frozen strings are stale enough to be a signal on their own —
35
+ * `Galaxy Tab S4` still says `Chrome/70`, `iPad Pro 11` pairs `CPU OS 12_2` with `Version/15.4`.
36
+ * Only the metrics are taken from a real device (a Pixel-class phone and a 11" tablet).
37
+ */
38
+ const VIEWPORTS = {
39
+ desktop: null,
40
+ mobile: {
41
+ width: 412,
42
+ height: 915,
43
+ deviceScaleFactor: 2.625,
44
+ isMobile: true,
45
+ hasTouch: true,
46
+ isLandscape: false,
47
+ },
48
+ tablet: {
49
+ width: 834,
50
+ height: 1112,
51
+ deviceScaleFactor: 2,
52
+ isMobile: true,
53
+ hasTouch: true,
54
+ isLandscape: false,
55
+ },
56
+ };
57
+ /** The device each of the server's default variation user agents stands for. */
58
+ const SPEED_KIT_USER_AGENTS = {
59
+ [DEFAULT_USER_AGENT]: "desktop",
60
+ [MOBILE_USER_AGENT]: "mobile",
61
+ [TABLET_USER_AGENT]: "tablet",
62
+ };
63
+ /** An Android build with no `Mobile` token is a tablet, as are the iPad and generic tablet UAs. */
64
+ const TABLET_USER_AGENT_TOKENS = /ipad|tablet|android(?!.*mobile)/i;
65
+ const MOBILE_USER_AGENT_TOKENS = /iphone|ipod|mobile|windows phone/i;
66
+ /** True for one of the user agents the Asset API sends for a device variation. */
67
+ export function isSpeedKitUserAgent(userAgent) {
68
+ return !!userAgent && userAgent in SPEED_KIT_USER_AGENTS;
69
+ }
70
+ /**
71
+ * The device a request is for, read off the user agent the caller would have sent.
72
+ *
73
+ * The user agent is the signal rather than the variation name, because it is what the origin's own
74
+ * device detection sees: a customer variation named `mobile` that overrides the user agent with a
75
+ * desktop one is answered with the desktop document in production, and has to be here too. The
76
+ * three defaults are matched exactly, and anything else falls back to reading its tokens, so a
77
+ * customer's own mobile user agent still selects a phone.
78
+ *
79
+ * @param userAgent - The caller's `user-agent` header, if it set one.
80
+ * @returns The device, or undefined when there is no user agent to read.
81
+ */
82
+ export function deviceForUserAgent(userAgent) {
83
+ if (!userAgent)
84
+ return undefined;
85
+ const known = SPEED_KIT_USER_AGENTS[userAgent];
86
+ if (known)
87
+ return known;
88
+ if (TABLET_USER_AGENT_TOKENS.test(userAgent))
89
+ return "tablet";
90
+ if (MOBILE_USER_AGENT_TOKENS.test(userAgent))
91
+ return "mobile";
92
+ return "desktop";
93
+ }
94
+ /**
95
+ * Builds the emulation for one device from the running Chrome's version.
96
+ *
97
+ * The user agent is derived from the binary rather than taken from a table, for the same reason
98
+ * `buildHeadedUserAgent` derives the desktop one: only the major version moves, and a stale one is
99
+ * exactly what a protection layer scores. Both mobile devices therefore present *Chrome on
100
+ * Android* — an iOS user agent would name Safari while the engine underneath answers every
101
+ * fingerprint as Blink, which is the mismatch these fetches exist to avoid. The cost is that an
102
+ * origin serving per-OS markup answers a mobile fetch with its Android document, where the Asset
103
+ * API would have asked as an iPhone.
104
+ *
105
+ * @param browserVersionString - `browser.version()`, e.g. `Chrome/146.0.7680.165`.
106
+ * @param device - The device to present.
107
+ */
108
+ export function deviceEmulation(browserVersionString, device) {
109
+ const viewport = VIEWPORTS[device];
110
+ const major = chromeMajorVersion(browserVersionString);
111
+ if (!major)
112
+ return { device, viewport };
113
+ if (device === "desktop") {
114
+ return {
115
+ device,
116
+ userAgent: buildHeadedUserAgent(browserVersionString),
117
+ metadata: desktopMetadata(major),
118
+ viewport,
119
+ };
120
+ }
121
+ // A tablet is the same Chrome without the `Mobile` token, which is what Chrome itself drops on a
122
+ // tablet build — and what the Asset API's own tablet user agent does.
123
+ const mobileToken = device === "mobile" ? "Mobile " : "";
124
+ return {
125
+ device,
126
+ userAgent: `Mozilla/5.0 (${ANDROID_PLATFORM}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 ${mobileToken}Safari/537.36`,
127
+ metadata: androidMetadata(major, device === "mobile"),
128
+ viewport,
129
+ };
130
+ }
131
+ /**
132
+ * The client hints the headed desktop Chrome would send.
133
+ *
134
+ * Needed even though the user agent already matches the host: overriding the user agent at all
135
+ * switches Chrome's own hints off, so without this the request carries no `Sec-CH-UA` whatsoever —
136
+ * which no real Chrome does, and which is the kind of absence a protection layer scores.
137
+ *
138
+ * @param platform - Node platform, i.e. what `buildHeadedUserAgent` built the user agent for.
139
+ */
140
+ function desktopMetadata(major, platform = process.platform) {
141
+ const desktop = DESKTOP_PLATFORMS[platform] ?? DEFAULT_DESKTOP_PLATFORM;
142
+ return {
143
+ ...brandsOf(major),
144
+ platform: desktop.platform,
145
+ platformVersion: desktop.version,
146
+ architecture: "x86",
147
+ bitness: "64",
148
+ model: "",
149
+ mobile: false,
150
+ wow64: false,
151
+ };
152
+ }
153
+ /**
154
+ * The client hints an Android Chrome sends.
155
+ *
156
+ * Without them the user agent says Android while `sec-ch-ua-platform` still says the host's own
157
+ * platform and `sec-ch-ua-mobile` says `?0` — a contradiction any protection layer reads, and one
158
+ * `page.emulate()` would leave in place because it only sets the user agent string.
159
+ */
160
+ function androidMetadata(major, mobile) {
161
+ return {
162
+ ...brandsOf(major),
163
+ platform: "Android",
164
+ platformVersion: ANDROID_PLATFORM_VERSION,
165
+ // Chrome reports neither on Android.
166
+ architecture: "",
167
+ bitness: "",
168
+ model: ANDROID_MODEL,
169
+ mobile,
170
+ wow64: false,
171
+ };
172
+ }
173
+ /** `Sec-CH-UA` and its full-version counterpart for one Chrome major. */
174
+ function brandsOf(major) {
175
+ const brands = [
176
+ { brand: "Chromium", version: major },
177
+ { brand: "Google Chrome", version: major },
178
+ GREASE_BRAND,
179
+ ];
180
+ return {
181
+ brands,
182
+ fullVersionList: brands.map(({ brand, version }) => ({
183
+ brand,
184
+ version: `${version}.0.0.0`,
185
+ })),
186
+ };
187
+ }
@@ -0,0 +1,73 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import { DEFAULT_USER_AGENT, MOBILE_USER_AGENT, TABLET_USER_AGENT, } from "../origin-request/asset-variations.js";
4
+ import { deviceEmulation, deviceForUserAgent, isSpeedKitUserAgent, } from "./device-emulation.js";
5
+ const CHROME_VERSION = "Chrome/146.0.7680.165";
6
+ describe("deviceForUserAgent", () => {
7
+ it("maps the three user agents of the Asset API's device variations", () => {
8
+ expect(deviceForUserAgent(DEFAULT_USER_AGENT)).to.equal("desktop");
9
+ expect(deviceForUserAgent(MOBILE_USER_AGENT)).to.equal("mobile");
10
+ expect(deviceForUserAgent(TABLET_USER_AGENT)).to.equal("tablet");
11
+ });
12
+ it("reads a customer's own mobile user agent as a phone", () => {
13
+ expect(deviceForUserAgent("Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36")).to.equal("mobile");
14
+ expect(deviceForUserAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1")).to.equal("mobile");
15
+ });
16
+ it("reads an Android without the Mobile token, and an iPad, as a tablet", () => {
17
+ expect(deviceForUserAgent("Mozilla/5.0 (Linux; Android 13; SM-X710) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")).to.equal("tablet");
18
+ expect(deviceForUserAgent("Mozilla/5.0 (iPad; CPU OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/604.1")).to.equal("tablet");
19
+ });
20
+ it("falls back to desktop for anything else", () => {
21
+ expect(deviceForUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36")).to.equal("desktop");
22
+ });
23
+ it("has no device for a request that sends no user agent", () => {
24
+ expect(deviceForUserAgent(undefined)).to.equal(undefined);
25
+ expect(deviceForUserAgent("")).to.equal(undefined);
26
+ });
27
+ });
28
+ describe("isSpeedKitUserAgent", () => {
29
+ it("knows the server's own user agents, which are dropped without a warning", () => {
30
+ expect(isSpeedKitUserAgent(MOBILE_USER_AGENT)).to.equal(true);
31
+ expect(isSpeedKitUserAgent("Mozilla/5.0 (custom)")).to.equal(false);
32
+ expect(isSpeedKitUserAgent(undefined)).to.equal(false);
33
+ });
34
+ });
35
+ describe("deviceEmulation", () => {
36
+ it("presents the desktop as the headed Chrome, with the window's own viewport", () => {
37
+ const emulation = deviceEmulation(CHROME_VERSION, "desktop");
38
+ expect(emulation.userAgent).to.include("Chrome/146.0.0.0 Safari/537.36");
39
+ expect(emulation.userAgent).to.not.include("SpeedKit");
40
+ expect(emulation.viewport).to.equal(null);
41
+ });
42
+ it("still sends desktop client hints, which the user agent override would otherwise drop", () => {
43
+ const metadata = deviceEmulation(CHROME_VERSION, "desktop").metadata;
44
+ expect(metadata?.mobile).to.equal(false);
45
+ expect(metadata?.platform).to.be.oneOf(["Linux", "macOS", "Windows"]);
46
+ expect(metadata?.brands?.map((brand) => brand.brand)).to.include("Google Chrome");
47
+ });
48
+ it("presents a phone as Chrome on Android, built from the running version", () => {
49
+ const emulation = deviceEmulation(CHROME_VERSION, "mobile");
50
+ expect(emulation.userAgent).to.equal("Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36");
51
+ expect(emulation.viewport?.isMobile).to.equal(true);
52
+ expect(emulation.viewport?.hasTouch).to.equal(true);
53
+ });
54
+ it("drops the Mobile token for a tablet, as Chrome itself does", () => {
55
+ const emulation = deviceEmulation(CHROME_VERSION, "tablet");
56
+ expect(emulation.userAgent).to.equal("Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36");
57
+ expect(emulation.viewport?.width).to.be.greaterThan(deviceEmulation(CHROME_VERSION, "mobile").viewport?.width ?? 0);
58
+ });
59
+ it("sends client hints that agree with the Android user agent", () => {
60
+ const mobile = deviceEmulation(CHROME_VERSION, "mobile").metadata;
61
+ const tablet = deviceEmulation(CHROME_VERSION, "tablet").metadata;
62
+ expect(mobile?.platform).to.equal("Android");
63
+ expect(mobile?.mobile).to.equal(true);
64
+ // A tablet's user agent has no Mobile token, so its hint must not claim one either.
65
+ expect(tablet?.mobile).to.equal(false);
66
+ expect(mobile?.brands?.map((brand) => brand.version)).to.include("146");
67
+ });
68
+ it("leaves the browser's own identity in place when the version is unreadable", () => {
69
+ for (const device of ["desktop", "mobile", "tablet"]) {
70
+ expect(deviceEmulation("unknown", device).userAgent).to.equal(undefined);
71
+ }
72
+ });
73
+ });
@@ -0,0 +1,4 @@
1
+ export * from "./browser-origin-fetch.js";
2
+ export * from "./browser-origin-fetcher.js";
3
+ export * from "./browser-response.js";
4
+ export * from "./device-emulation.js";
@@ -0,0 +1,4 @@
1
+ export * from "./browser-origin-fetch.js";
2
+ export * from "./browser-origin-fetcher.js";
3
+ export * from "./browser-response.js";
4
+ export * from "./device-emulation.js";
@@ -5,3 +5,10 @@
5
5
  * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
6
6
  */
7
7
  export declare function buildHeadedUserAgent(browserVersionString: string, platform?: string): string | undefined;
8
+ /**
9
+ * The Chrome major version of a `chrome --version` string, which is the only part of the reduced
10
+ * user agent that still moves.
11
+ *
12
+ * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
13
+ */
14
+ export declare function chromeMajorVersion(browserVersionString: string): string | undefined;
@@ -20,10 +20,19 @@ const DEFAULT_UNIFIED_PLATFORM = "X11; Linux x86_64";
20
20
  * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
21
21
  */
22
22
  export function buildHeadedUserAgent(browserVersionString, platform = process.platform) {
23
- const majorVersion = browserVersionString.match(/(\d+)\.\d+\.\d+\.\d+/)?.[1];
23
+ const majorVersion = chromeMajorVersion(browserVersionString);
24
24
  if (!majorVersion) {
25
25
  return undefined;
26
26
  }
27
27
  const unifiedPlatform = UNIFIED_PLATFORMS[platform] ?? DEFAULT_UNIFIED_PLATFORM;
28
28
  return `Mozilla/5.0 (${unifiedPlatform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${majorVersion}.0.0.0 Safari/537.36`;
29
29
  }
30
+ /**
31
+ * The Chrome major version of a `chrome --version` string, which is the only part of the reduced
32
+ * user agent that still moves.
33
+ *
34
+ * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
35
+ */
36
+ export function chromeMajorVersion(browserVersionString) {
37
+ return browserVersionString.match(/(\d+)\.\d+\.\d+\.\d+/)?.[1];
38
+ }
@@ -2,6 +2,38 @@ import { BaqendResponse } from "../browser/baqend-response.js";
2
2
  import { safe } from "../../../helpers/safe.js";
3
3
  import json2Csv from "json2csv";
4
4
  import { AthenaBucket } from "../../athena/index.js";
5
+ /** A query option the caller left out, or the number it set. */
6
+ function readNumber(params, name) {
7
+ const raw = params.get(name);
8
+ if (raw === null || raw.trim() === "") {
9
+ return undefined;
10
+ }
11
+ const value = Number(raw);
12
+ return Number.isFinite(value) ? value : undefined;
13
+ }
14
+ /** A query option the caller left out, or the flag it set. */
15
+ function readFlag(params, name) {
16
+ const raw = params.get(name);
17
+ if (raw === null) {
18
+ return undefined;
19
+ }
20
+ return raw !== "false" && raw !== "0";
21
+ }
22
+ /**
23
+ * The query options carried on the url. An option that is not there stays
24
+ * undefined so the value the command line was started with keeps applying —
25
+ * the editor sends what it shows, and it shows nothing until it has asked.
26
+ */
27
+ function readQueryOverrides(params) {
28
+ return {
29
+ bySet: readFlag(params, "bySet"),
30
+ days: readNumber(params, "days"),
31
+ excludeNoindex: readFlag(params, "excludeNoindex"),
32
+ excludeSuspicious: readFlag(params, "excludeSuspicious"),
33
+ hardNavigationsOnly: readFlag(params, "hardNavigationsOnly"),
34
+ minImpressions: readNumber(params, "minImpressions"),
35
+ };
36
+ }
5
37
  export class Dashboard {
6
38
  customerConfig;
7
39
  parameterQueryBuilder;
@@ -28,7 +60,14 @@ export class Dashboard {
28
60
  return this.createBaqendResponse(requestOrigin, "unsupported method");
29
61
  }
30
62
  if (url.pathname.startsWith("/query/create")) {
31
- return this.createBaqendResponse(requestOrigin, JSON.stringify({ query: this.parameterQueryBuilder.query() }), "application/json; charset=UTF-8");
63
+ const overrides = readQueryOverrides(url.searchParams);
64
+ return this.createBaqendResponse(requestOrigin,
65
+ // The options travel back with the query: the editor offering them
66
+ // has no other way to learn what the command line already set.
67
+ JSON.stringify({
68
+ query: this.parameterQueryBuilder.query(overrides),
69
+ options: this.parameterQueryBuilder.resolvedOptions(overrides),
70
+ }), "application/json; charset=UTF-8");
32
71
  }
33
72
  if (url.pathname.startsWith("/query/execute")) {
34
73
  const postData = request.postData;
@@ -5,6 +5,7 @@ import { AbortResponse } from "./browser/abort-response.js";
5
5
  import { task } from "./todo-collector.js";
6
6
  import { VanillaPuppeteer } from "puppeteer-extra";
7
7
  import { Command } from "@oclif/core/command";
8
+ export declare const HEADLESS_PARITY_ARGS: string[];
8
9
  declare const HEADLESS_DEFAULT_ARGS_TO_DROP: string[];
9
10
  export { HEADLESS_DEFAULT_ARGS_TO_DROP };
10
11
  export declare const BROWSER_EVENTS: {
@@ -72,12 +73,13 @@ export declare class OnboardingContext {
72
73
  readonly debuggingPort: boolean;
73
74
  readonly headless: boolean;
74
75
  readonly debugPort: number;
76
+ readonly browserFetch: boolean;
75
77
  static flags: Record<string, Command.Flag>;
76
78
  readonly DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
77
79
  readonly DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
78
80
  readonly DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
79
81
  readonly DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
80
- constructor(customerPath: string, configName: string, domain: string, local: boolean, ignoreContentSecurityPolicy: boolean, cliPath: string, verboseLevel?: boolean, useTestConfig?: boolean, debuggingPort?: boolean, headless?: boolean, debugPort?: number);
82
+ constructor(customerPath: string, configName: string, domain: string, local: boolean, ignoreContentSecurityPolicy: boolean, cliPath: string, verboseLevel?: boolean, useTestConfig?: boolean, debuggingPort?: boolean, headless?: boolean, debugPort?: number, browserFetch?: boolean);
81
83
  }
82
84
  export interface SpeedKitServerConfigAuthentication {
83
85
  password: string;
@@ -16,7 +16,7 @@ const DEFAULT_BROWSER_ARGS = [
16
16
  // Headless Chrome differs from headed in ways customer bot detection and responsive layout both
17
17
  // key on, so a page can render differently — or not at all — depending on the mode. Each entry
18
18
  // below erases one measured difference; see HEADLESS_DEFAULT_ARGS_TO_DROP for the counterpart.
19
- const HEADLESS_PARITY_ARGS = [
19
+ export const HEADLESS_PARITY_ARGS = [
20
20
  // Headless exposes no pointing device, so `hover: none` / `any-pointer: coarse` send sites down
21
21
  // their touch code path.
22
22
  "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
@@ -170,6 +170,7 @@ export class OnboardingContext {
170
170
  debuggingPort;
171
171
  headless;
172
172
  debugPort;
173
+ browserFetch;
173
174
  static flags = {
174
175
  ...CLI_CONFIG_NAME,
175
176
  ...CLI_CONFIG_IS_TEST,
@@ -205,12 +206,16 @@ export class OnboardingContext {
205
206
  default: false,
206
207
  description: "Run Chrome in headless mode (for CI environments). Implicitly enables remote debugging port.",
207
208
  }),
209
+ browserFetch: Flags.boolean({
210
+ default: false,
211
+ description: "Fetch every origin document with a second headless Chrome instead of a direct request. Only needed when bot protection refuses the CLI outright — a refusal is otherwise detected and rerouted on its own.",
212
+ }),
208
213
  };
209
214
  DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
210
215
  DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
211
216
  DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
212
217
  DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
213
- constructor(customerPath, configName, domain, local, ignoreContentSecurityPolicy = false, cliPath, verboseLevel, useTestConfig = false, debuggingPort = false, headless = false, debugPort = 9222) {
218
+ constructor(customerPath, configName, domain, local, ignoreContentSecurityPolicy = false, cliPath, verboseLevel, useTestConfig = false, debuggingPort = false, headless = false, debugPort = 9222, browserFetch = false) {
214
219
  this.customerPath = customerPath;
215
220
  this.configName = configName;
216
221
  this.domain = domain;
@@ -222,6 +227,7 @@ export class OnboardingContext {
222
227
  this.debuggingPort = debuggingPort;
223
228
  this.headless = headless;
224
229
  this.debugPort = debugPort;
230
+ this.browserFetch = browserFetch;
225
231
  }
226
232
  }
227
233
  export const USER_AGENT = {
@@ -29,6 +29,7 @@ import { ExtensionValidator } from "./browser/extension/extension-validator.js";
29
29
  import { DevtoolsExtensionApi } from "./browser/extension/devtools-extension-api.js";
30
30
  import { ExtensionBridge } from "./browser/extension/extension-bridge.js";
31
31
  import { TodoCollector } from "./todo-collector.js";
32
+ import { installBrowserOriginFetch } from "../browser-fetch/index.js";
32
33
  import { Crawler } from "./virtual-orestes-app/crawler.js";
33
34
  import { VirtualOrestesApp } from "./virtual-orestes-app/index.js";
34
35
  import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
@@ -55,6 +56,16 @@ export class OnboardingServiceFactory {
55
56
  const customerConfig = files.getCustomerConfig().config;
56
57
  // Before any origin request, so Chrome and the CLI agree on the route out.
57
58
  applyOriginProxy(customerConfig.chromeFlags, cli);
59
+ // Also before any origin request: an origin behind bot protection refuses the CLI outright,
60
+ // and `auto` only reaches for the browser once one actually has.
61
+ const browserFetch = installBrowserOriginFetch({
62
+ userConfig: this.cliConfig,
63
+ logger: cli,
64
+ mode: this.context.browserFetch ? "always" : "auto",
65
+ });
66
+ if (this.context.browserFetch) {
67
+ cli.writeWarning("Fetching every origin document through a second headless Chrome.");
68
+ }
58
69
  const domainToStart = await this.getUrlForTest(customerConfig, cli);
59
70
  const cache = new Cache(cli);
60
71
  const documentHandlerContext = {
@@ -87,7 +98,7 @@ export class OnboardingServiceFactory {
87
98
  const speedKitServiceWorkerJs = new SpeedKitServiceWorkerJs(customerConfig, files.getByName(INTEGRATION_FILES.CUSTOM.SW));
88
99
  const fetchHandler = await this.getFetchEventHandler(files, customerConfig, documentHandler, cache, cli, browserContext, athenaClient, messageHandler, fileWatcher, speedKitServiceWorkerJs);
89
100
  const extensionBridge = new ExtensionBridge(cli);
90
- return new OnboardingService(browserContext, customerConfig, fetchHandler, fileWatcher, cli, messageHandler, extensionBridge, speedKitServiceWorkerJs);
101
+ return new OnboardingService(browserContext, customerConfig, fetchHandler, fileWatcher, cli, messageHandler, extensionBridge, speedKitServiceWorkerJs, browserFetch);
91
102
  }
92
103
  async prepareMessagehandler(athenaClient, files, customerConfig) {
93
104
  const messageHandler = new DevtoolsExtensionApi();
@@ -6,6 +6,7 @@ import { FileWatcher } from "./file-events/file-watcher.js";
6
6
  import { CliService } from "../cli/index.js";
7
7
  import { DevtoolsExtensionApi } from "./browser/extension/devtools-extension-api.js";
8
8
  import { ExtensionBridge } from "./browser/extension/extension-bridge.js";
9
+ import { BrowserOriginFetchHandle } from "../browser-fetch/index.js";
9
10
  /**
10
11
  * Represents the OnboardingService which is responsible for setting up the browser to test a config using Puppeteer.
11
12
  */
@@ -18,10 +19,11 @@ export declare class OnboardingService {
18
19
  private developmentToolsMessages;
19
20
  private extensionBridge;
20
21
  private speedKitServiceWorkerJs;
22
+ private browserFetch?;
21
23
  private reinstallOnNextNavigate;
22
24
  private lastForeignSwReinstall;
23
25
  private readonly activatedSwVersions;
24
- constructor(browserContext: BrowserContext, customerConfig: CustomerConfig, fetchEventHandler: FetchEventHandler, fileWatcher: FileWatcher, cli: CliService, developmentToolsMessages: DevtoolsExtensionApi, extensionBridge: ExtensionBridge, speedKitServiceWorkerJs: SpeedKitServiceWorkerJs);
26
+ constructor(browserContext: BrowserContext, customerConfig: CustomerConfig, fetchEventHandler: FetchEventHandler, fileWatcher: FileWatcher, cli: CliService, developmentToolsMessages: DevtoolsExtensionApi, extensionBridge: ExtensionBridge, speedKitServiceWorkerJs: SpeedKitServiceWorkerJs, browserFetch?: BrowserOriginFetchHandle);
25
27
  /**
26
28
  * Prepare the puppeteer browser, wire event listeners, navigate to the
27
29
  * install page, then block until either the caller aborts the signal