@speedkit/cli 4.28.0 → 4.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +5 -2
  3. package/dist/commands/onboarding.js +2 -2
  4. package/dist/helpers/evaluate-speed-kit-config.d.ts +5 -0
  5. package/dist/helpers/evaluate-speed-kit-config.js +14 -7
  6. package/dist/helpers/origin-proxy.d.ts +30 -0
  7. package/dist/helpers/origin-proxy.js +29 -0
  8. package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
  9. package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
  10. package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
  11. package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
  12. package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
  13. package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
  14. package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
  15. package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
  16. package/dist/services/browser-fetch/browser-response.d.ts +21 -0
  17. package/dist/services/browser-fetch/browser-response.js +86 -0
  18. package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
  19. package/dist/services/browser-fetch/browser-response.spec.js +90 -0
  20. package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
  21. package/dist/services/browser-fetch/device-emulation.js +187 -0
  22. package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
  23. package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
  24. package/dist/services/browser-fetch/index.d.ts +4 -0
  25. package/dist/services/browser-fetch/index.js +4 -0
  26. package/dist/services/customer-config/customer-config-service-model.d.ts +6 -0
  27. package/dist/services/customer-config/customer-config-service-model.js +1 -0
  28. package/dist/services/customer-config/customer-config-service.d.ts +5 -0
  29. package/dist/services/customer-config/customer-config-service.js +15 -0
  30. package/dist/services/deploy/checks/activation/customer-approval-check.d.ts +21 -0
  31. package/dist/services/deploy/checks/activation/customer-approval-check.js +59 -0
  32. package/dist/services/deploy/checks/activation/customer-approval-check.spec.d.ts +1 -0
  33. package/dist/services/deploy/checks/activation/customer-approval-check.spec.js +64 -0
  34. package/dist/services/deploy/deploy-service-factory.js +2 -1
  35. package/dist/services/deploy/handler/install-resource-handler.d.ts +7 -1
  36. package/dist/services/deploy/handler/install-resource-handler.js +40 -19
  37. package/dist/services/deploy/handler/testing-installation-links.d.ts +10 -0
  38. package/dist/services/deploy/handler/testing-installation-links.js +27 -0
  39. package/dist/services/deploy/handler/testing-installation-links.spec.d.ts +1 -0
  40. package/dist/services/deploy/handler/testing-installation-links.spec.js +43 -0
  41. package/dist/services/integration-api/struct/customer-config.d.ts +7 -1
  42. package/dist/services/integration-api/struct/customer-config.js +10 -3
  43. package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
  44. package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
  45. package/dist/services/onboarding/onboarding-model.d.ts +3 -1
  46. package/dist/services/onboarding/onboarding-model.js +8 -2
  47. package/dist/services/onboarding/onboarding-service-factory.js +12 -1
  48. package/dist/services/onboarding/onboarding-service.d.ts +3 -1
  49. package/dist/services/onboarding/onboarding-service.js +7 -1
  50. package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
  51. package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
  52. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
  53. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
  54. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
  55. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
  56. package/dist/services/origin-request/asset-variations.d.ts +2 -0
  57. package/dist/services/origin-request/asset-variations.js +2 -2
  58. package/oclif.manifest.json +118 -112
  59. package/package.json +1 -1
@@ -0,0 +1,528 @@
1
+ import { mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
2
+ import { mkdtemp } from "node:fs/promises";
3
+ import path from "node:path";
4
+ import puppeteer from "puppeteer";
5
+ import { chromeProxyFlag, currentOriginProxy, } from "../../helpers/origin-proxy.js";
6
+ import { safe } from "../../helpers/safe.js";
7
+ import { HEADLESS_PARITY_ARGS } from "../onboarding/onboarding-model.js";
8
+ import { toFetchResponse, toPlainHeaders } from "./browser-response.js";
9
+ import { deviceEmulation, deviceForUserAgent, isSpeedKitUserAgent, } from "./device-emulation.js";
10
+ /**
11
+ * Folder under the CLI's temp folder that holds one throwaway profile per browser.
12
+ *
13
+ * Deliberately not one shared profile. A protection layer that refuses a request refuses the
14
+ * *session*, and it records that against the `cf_clearance` / `__cf_bm` cookies the profile
15
+ * carries — measured: once a session was refused, a browser reusing its profile was refused on
16
+ * every later request, across restarts, while a browser with a new profile was answered
17
+ * immediately. A profile that outlives the run therefore turns one bad session into a permanent
18
+ * one, which is worth far more than the ~1s of clearance it would have saved.
19
+ */
20
+ const PROFILE_FOLDER = "browser_fetch_profiles";
21
+ /** How long a leftover profile from a crashed run is kept before it is pruned. */
22
+ const PROFILE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
23
+ /**
24
+ * Shortest gap between two attempts to recover from a refused session.
25
+ *
26
+ * Recovering means throwing the browser away and starting a new one, so it must not happen per
27
+ * request: a genuinely blocked origin would otherwise launch a Chrome per fetch.
28
+ */
29
+ const RECYCLE_COOLDOWN_MS = 30_000;
30
+ /**
31
+ * Flags for a browser that only fetches. It shares nothing with the onboarding launch: no
32
+ * extensions, no devtools, no insecure-origin exception, and above all no CDP `Fetch` interception,
33
+ * which is browser-wide in onboarding and would answer these requests with onboarding's rewrites
34
+ * instead of the origin's bytes.
35
+ */
36
+ const FETCH_BROWSER_ARGS = [
37
+ "--no-first-run",
38
+ "--no-default-browser-check",
39
+ "--disable-background-networking",
40
+ "--disable-backgrounding-occluded-windows",
41
+ "--disable-renderer-backgrounding",
42
+ ];
43
+ /**
44
+ * Puppeteer's headless default that has no headed equivalent, dropped for the same reason
45
+ * onboarding drops it: a 0px scrollbar narrows the layout viewport.
46
+ */
47
+ const HEADLESS_DEFAULTS_TO_DROP = ["--hide-scrollbars"];
48
+ const DEFAULT_NAVIGATION_TIMEOUT = 45_000;
49
+ /** How long a bot-protection interstitial is given to solve itself before the retry gives up. */
50
+ const CHALLENGE_SETTLE_TIMEOUT = 20_000;
51
+ const CHALLENGE_POLL_INTERVAL = 1_000;
52
+ /**
53
+ * How far into a response body a protection page's `<title>` is looked for. A block page is a few
54
+ * KB in total; a real document's head is well inside this too.
55
+ */
56
+ const VERDICT_SCAN_BYTES = 4096;
57
+ /**
58
+ * Titles of an interstitial that clears itself once its script has run. Waiting for one is
59
+ * worthwhile.
60
+ */
61
+ const CHALLENGE_TITLE = /just a moment|verify you are human|checking your browser/i;
62
+ /**
63
+ * Titles of a page that refuses the request outright. Waiting for one is not worthwhile — it never
64
+ * turns into the document, so the refusal is answered straight away.
65
+ */
66
+ const BLOCK_PAGE_TITLE = /attention required|access denied|sorry, you have been blocked/i;
67
+ /**
68
+ * Whether a request is the document the fetch was for, rather than something the page went on to
69
+ * want.
70
+ *
71
+ * A redirect hop is a navigation request too, so a redirected fetch still reaches its destination.
72
+ * The main-frame check is what keeps an iframe's document out: an ad frame's document is a
73
+ * subresource of this fetch, whatever it is to the frame.
74
+ *
75
+ * @param request - Request Chrome paused.
76
+ * @param mainFrame - The page's main frame.
77
+ */
78
+ export function isDocumentRequest(request, mainFrame) {
79
+ return request.isNavigationRequest() && request.frame() === mainFrame;
80
+ }
81
+ /**
82
+ * Fetches customer origins with a real browser instead of a socket.
83
+ *
84
+ * Bot protection scores the client, not the request: a managed challenge answers Chrome and
85
+ * refuses every server-side fetcher, whatever library, HTTP version or header set it uses — a
86
+ * `cf_clearance` cookie lifted out of a browser does not travel either, because it is bound to the
87
+ * client that earned it. A browser is therefore the only transport that reaches such an origin, so
88
+ * this fetcher drives one and adapts its answers back into `Response`.
89
+ *
90
+ * What it costs, and why it is not the default:
91
+ * - a second Chrome, launched lazily on the first request and held until `close()`;
92
+ * - one page per origin host and device, kept open so the clearance cookie stays warm —
93
+ * cross-origin requests from a parked page are refused by CORS, so each host needs its own;
94
+ * - requests to one host are serialised, because a page navigates to one document at a time.
95
+ *
96
+ * Headless is safe here only because of `HEADLESS_PARITY_ARGS` and the headed user agent: measured
97
+ * against a Cloudflare-protected origin, plain headless Chrome is hard-blocked on the first
98
+ * response and never recovers, while the same launch with the parity treatment is answered 200
99
+ * without ever seeing an interstitial. The user agent is the discriminator, so both are load-bearing.
100
+ *
101
+ * That is also why no caller's user agent reaches the origin. The Asset API's device variations are
102
+ * reproduced by emulating the device instead — viewport, touch and client hints included — with a
103
+ * user agent built from this very Chrome. A `(compatible; SpeedKit/1.0)` suffix, or any string that
104
+ * disagrees with what the engine answers to a fingerprint, gives the browser away and costs exactly
105
+ * the access this fetcher exists to get. See `device-emulation.ts`.
106
+ */
107
+ export class BrowserOriginFetcher {
108
+ userConfig;
109
+ logger;
110
+ navigationTimeout;
111
+ browser;
112
+ /** In flight during the launch, so concurrent first requests share one browser. */
113
+ launch;
114
+ /** Keyed by device and host, because emulation belongs to a page and outlives a request. */
115
+ pages = new Map();
116
+ /** Tail of the request chain per host, which is how requests to one host are serialised. */
117
+ queues = new Map();
118
+ /** Version of the running Chrome, which every emulated user agent is built from. */
119
+ browserVersion;
120
+ warned = new Set();
121
+ /**
122
+ * Pages currently allowed to load anything, which is only ever true while a challenge is being
123
+ * waited out. A protection layer other than Cloudflare serves its interstitial from paths this
124
+ * fetcher cannot know, so waiting for one has to lift the document-only rule.
125
+ */
126
+ permissive = new Set();
127
+ closed = false;
128
+ /** Profile directory of the running browser, removed with it. */
129
+ profile;
130
+ /** When the session was last thrown away, so recovery cannot run per request. */
131
+ lastRecycle = 0;
132
+ constructor(userConfig, logger, navigationTimeout = DEFAULT_NAVIGATION_TIMEOUT) {
133
+ this.userConfig = userConfig;
134
+ this.logger = logger;
135
+ this.navigationTimeout = navigationTimeout;
136
+ }
137
+ /**
138
+ * Fetches one origin URL through the browser.
139
+ *
140
+ * `options.tolerateLegacyTls` has no counterpart here: Chrome decides for itself what handshake
141
+ * it accepts, and it is already told to ignore certificate errors.
142
+ *
143
+ * @param url - Absolute origin URL.
144
+ * @param init - Request headers and redirect mode. Other `RequestInit` fields do not apply: a
145
+ * navigation is always a GET.
146
+ */
147
+ fetch(url, init,
148
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
149
+ options) {
150
+ const host = new URL(url).host;
151
+ return this.enqueue(host, () => this.fetchOnPage(host, url, init));
152
+ }
153
+ /** Closes the browser and removes its profile. A later `fetch` launches a new one. */
154
+ async close() {
155
+ this.closed = true;
156
+ await this.discardBrowser();
157
+ }
158
+ /**
159
+ * Throws the running browser and its profile away, leaving the fetcher usable.
160
+ *
161
+ * Everything derived from that browser goes with it: its pages, the permissive marks on them,
162
+ * and the user agent read from its version.
163
+ */
164
+ async discardBrowser() {
165
+ let browser = this.browser;
166
+ if (!browser && this.launch) {
167
+ const launched = await safe(this.launch);
168
+ if (launched.success === true)
169
+ browser = launched.data;
170
+ }
171
+ const profile = this.profile;
172
+ this.browser = undefined;
173
+ this.launch = undefined;
174
+ this.profile = undefined;
175
+ this.pages.clear();
176
+ this.permissive.clear();
177
+ this.browserVersion = undefined;
178
+ if (browser)
179
+ await safe(browser.close());
180
+ // Only after Chrome has let go of it, or the lock file survives and the next launch inherits
181
+ // the cookies this is meant to drop.
182
+ if (profile)
183
+ rmSync(profile, { force: true, recursive: true });
184
+ }
185
+ /**
186
+ * Starts a new session after the origin refused this one, and reports whether it did.
187
+ *
188
+ * The refusal is recorded against the session's cookies, so a new profile is the only way back.
189
+ * Rate-limited by `RECYCLE_COOLDOWN_MS`: against an origin that refuses us however we come, the
190
+ * answer has to settle on "refused" rather than launch a browser per request.
191
+ */
192
+ async recycleSession(host) {
193
+ if (Date.now() - this.lastRecycle < RECYCLE_COOLDOWN_MS)
194
+ return false;
195
+ this.lastRecycle = Date.now();
196
+ this.logger?.comment(`[browserFetch]: ${host} refused this browser session, starting a new one with a clean profile`);
197
+ await this.discardBrowser();
198
+ return true;
199
+ }
200
+ /**
201
+ * Runs `task` after every request already queued for `host`.
202
+ *
203
+ * The chain is kept whatever a task does, so one failed request does not strand the ones behind
204
+ * it, and the tail is dropped once it is the last link — otherwise the map would grow a rejected
205
+ * promise per host for the lifetime of the process.
206
+ */
207
+ enqueue(host, task) {
208
+ const previous = this.queues.get(host) ?? Promise.resolve();
209
+ const result = previous.then(task, task);
210
+ const tail = result.then(() => undefined, () => undefined);
211
+ this.queues.set(host, tail);
212
+ void tail.then(() => {
213
+ if (this.queues.get(host) === tail)
214
+ this.queues.delete(host);
215
+ });
216
+ return result;
217
+ }
218
+ async fetchOnPage(host, url, init) {
219
+ const headers = toPlainHeaders(init?.headers);
220
+ const callerUserAgent = headers["user-agent"];
221
+ delete headers["user-agent"];
222
+ // The caller's user agent selects the device but is never sent: see the class comment.
223
+ const device = deviceForUserAgent(callerUserAgent) ?? "desktop";
224
+ if (callerUserAgent && !isSpeedKitUserAgent(callerUserAgent)) {
225
+ this.warnOnce(`user-agent:${host}`, `${host} is fetched as an emulated ${device}, not with the user agent its variation sets ` +
226
+ `("${callerUserAgent}") — the browser has to present its own to get past bot protection, ` +
227
+ "so a variation the origin resolves by user agent may answer differently than in production.");
228
+ }
229
+ const page = await this.pageFor(host, device);
230
+ await page.setExtraHTTPHeaders(headers);
231
+ let response = await this.navigate(page, url);
232
+ const verdict = await this.verdictOn(response);
233
+ // A refusal is recorded against this session, so the only thing left to try is a new one.
234
+ // `fetchFresh` re-enters through `pageFor`, which opens a page on the replacement browser.
235
+ if (verdict === "refused" && (await this.recycleSession(host))) {
236
+ const fresh = await safe(this.fetchFresh(host, device, url, headers));
237
+ if (fresh.success === true)
238
+ return fresh.data;
239
+ }
240
+ if (verdict === "challenged") {
241
+ response = await this.afterChallenge(page, url, response);
242
+ }
243
+ else if (verdict === "refused") {
244
+ this.warnOnce(`refused:${host}`, `${host} refused the browser outright, so its protection cannot be passed from here at all. The customer has to skip bot management for our egress IP.`);
245
+ }
246
+ return this.adapt(response, url, init?.redirect);
247
+ }
248
+ /**
249
+ * Repeats the fetch on a browser started after the refusal, as the same device.
250
+ *
251
+ * Throws when the replacement is refused too, so the caller falls back to reporting what the
252
+ * first attempt said.
253
+ */
254
+ async fetchFresh(host, device, url, headers) {
255
+ const page = await this.pageFor(host, device);
256
+ await page.setExtraHTTPHeaders(headers);
257
+ const response = await this.navigate(page, url);
258
+ const verdict = await this.verdictOn(response);
259
+ if (verdict === "refused") {
260
+ throw new Error(`${host} refused the new session as well`);
261
+ }
262
+ return this.adapt(verdict === "challenged"
263
+ ? await this.afterChallenge(page, url, response)
264
+ : response, url);
265
+ }
266
+ /**
267
+ * Navigates the page, waiting for the document rather than for the whole page.
268
+ *
269
+ * `domcontentloaded` rather than `commit`: a challenge only clears itself once its script has
270
+ * run, and a response that needs no challenge is already complete by then anyway.
271
+ *
272
+ * The page's identity is not touched here — it was emulated once when the page was opened, and
273
+ * changing a user agent between two navigations of one page is itself a signal.
274
+ */
275
+ async navigate(page, url) {
276
+ const response = await page.goto(url, {
277
+ waitUntil: "domcontentloaded",
278
+ timeout: this.navigationTimeout,
279
+ });
280
+ if (!response) {
281
+ throw new Error(`the browser received no response for ${url}`);
282
+ }
283
+ return response;
284
+ }
285
+ /**
286
+ * Waits out a bot-protection interstitial and reports what the origin answered afterwards.
287
+ *
288
+ * An interstitial replaces itself with the real document once its scripts have run, so the page
289
+ * is polled rather than reloaded — a reload would start the challenge over. The document-only
290
+ * rule is lifted for the wait, because those scripts are the whole mechanism. When the wait runs
291
+ * out, the blocked response is returned as it is: the caller has a status to report and a cause
292
+ * to name, which is more useful than an exception.
293
+ */
294
+ async afterChallenge(page, url, blocked) {
295
+ this.permissive.add(page);
296
+ try {
297
+ return await this.waitOutChallenge(page, url, blocked);
298
+ }
299
+ finally {
300
+ this.permissive.delete(page);
301
+ }
302
+ }
303
+ async waitOutChallenge(page, url, blocked) {
304
+ const deadline = Date.now() + CHALLENGE_SETTLE_TIMEOUT;
305
+ while (Date.now() < deadline) {
306
+ await new Promise((resolve) => setTimeout(resolve, CHALLENGE_POLL_INTERVAL));
307
+ if (CHALLENGE_TITLE.test(await this.titleOf(page)))
308
+ continue;
309
+ // The challenge cleared itself, so the clearance cookie is now on the page. Asking again is
310
+ // what turns it into a response with the origin's status and headers.
311
+ const settled = await safe(this.navigate(page, url));
312
+ if (settled.success === true)
313
+ return settled.data;
314
+ break;
315
+ }
316
+ this.warnOnce(`challenge:${new URL(url).host}`, `${new URL(url).host} kept challenging the browser for ${CHALLENGE_SETTLE_TIMEOUT / 1000}s without clearing. Its protection cannot be passed from here — the customer has to skip bot management for our egress IP.`);
317
+ return blocked;
318
+ }
319
+ /**
320
+ * Reads the response body and hands back a `Response`.
321
+ *
322
+ * With `redirect: "manual"` the first hop is answered instead of the destination: Chrome always
323
+ * follows a redirect, so the hop is recovered from the request's redirect chain. A body is never
324
+ * available for it, which matches what a manual redirect carries anyway.
325
+ */
326
+ async adapt(response, requestedUrl, redirect) {
327
+ const chain = response.request().redirectChain();
328
+ if (redirect === "manual" && chain.length > 0) {
329
+ const hop = chain[0].response();
330
+ if (hop)
331
+ return toFetchResponse(hop, Buffer.alloc(0), false);
332
+ }
333
+ const body = await safe(response.buffer());
334
+ if (body.success !== true) {
335
+ throw new Error(`the browser could not read the body of ${requestedUrl}: ${body.error}`);
336
+ }
337
+ return toFetchResponse(response, body.data, chain.length > 0);
338
+ }
339
+ /**
340
+ * Whether the origin answered, or its protection layer did — and if so, whether waiting helps.
341
+ *
342
+ * The status alone decides nothing: an origin may answer 403 for its own reasons, and a
343
+ * protection layer may answer 200 with an interstitial. So Cloudflare's own marker and the
344
+ * document's title are what separate the three outcomes.
345
+ *
346
+ * Read out of the response body rather than with `page.title()`, which would evaluate script in
347
+ * a page a bot detector is watching. Puppeteer memoises the body, so the read `adapt` does later
348
+ * costs nothing. The body is only searched as far as a `<title>` can reach.
349
+ */
350
+ async verdictOn(response) {
351
+ const mitigated = response.headers()["cf-mitigated"];
352
+ const head = await this.headOf(response);
353
+ if (mitigated === "challenge" || CHALLENGE_TITLE.test(head)) {
354
+ return "challenged";
355
+ }
356
+ if (BLOCK_PAGE_TITLE.test(head) || (!response.ok() && mitigated)) {
357
+ return "refused";
358
+ }
359
+ return "answered";
360
+ }
361
+ /**
362
+ * The start of the response body, which is as far as a `<title>` can be.
363
+ *
364
+ * A body that cannot be read is not a verdict either way, so it reads as empty and the status
365
+ * and headers decide alone.
366
+ */
367
+ async headOf(response) {
368
+ const body = await safe(response.buffer());
369
+ return body.success === true
370
+ ? body.data.subarray(0, VERDICT_SCAN_BYTES).toString("utf8")
371
+ : "";
372
+ }
373
+ /** The tab's title, or nothing when the page is in a state that cannot report one. */
374
+ async titleOf(page) {
375
+ const title = await safe(page.title());
376
+ return title.success === true ? title.data : "";
377
+ }
378
+ /**
379
+ * The page this host is fetched on as this device.
380
+ *
381
+ * A device gets its own page rather than a re-emulated one: emulation is page state, and a page
382
+ * that changes device between two navigations tells the origin more about us than either device
383
+ * would have.
384
+ */
385
+ async pageFor(host, device) {
386
+ const key = `${device}|${host}`;
387
+ const existing = this.pages.get(key);
388
+ if (existing) {
389
+ const page = await existing;
390
+ if (!page.isClosed())
391
+ return page;
392
+ this.pages.delete(key);
393
+ }
394
+ const opening = this.openPage(host, device);
395
+ this.pages.set(key, opening);
396
+ return opening;
397
+ }
398
+ /**
399
+ * Opens a page that loads documents and nothing else.
400
+ *
401
+ * Without this a fetch costs the origin a whole page load — measured against one customer, 87
402
+ * subresource requests for the home page and 182 for a category page. Three such fetches in a
403
+ * row were enough for that origin to blackhole the session, and every request after that was
404
+ * refused. Aborting subresources keeps one fetch to one origin request, which is both what the
405
+ * callers want and what keeps the fetcher from behaving like an attack. It also makes a fetch
406
+ * 3-5x faster, because nothing waits for images or scripts.
407
+ *
408
+ * The rule also keeps the browser out of Cloudflare's JS fingerprinting, which is itself a
409
+ * subresource: measured against the same customer, exempting `/cdn-cgi/` let the detection run,
410
+ * and it hard-blocked the session from the third request onwards — where aborting it left eight
411
+ * consecutive fetches answered. `afterChallenge` lifts the rule only when an interstitial has
412
+ * actually appeared, which is the one case those requests are needed for.
413
+ */
414
+ async openPage(host, device) {
415
+ const browser = await this.browserInstance();
416
+ const page = await browser.newPage();
417
+ page.setDefaultNavigationTimeout(this.navigationTimeout);
418
+ await this.emulate(page, device, host);
419
+ await page.setRequestInterception(true);
420
+ page.on("request", (request) => {
421
+ const allowed = isDocumentRequest(request, page.mainFrame()) ||
422
+ this.permissive.has(page);
423
+ // Either call throws for a request that is already gone — the page navigated away, or
424
+ // Chrome answered it from its own cache — and there is nothing to decide for those.
425
+ const decided = allowed
426
+ ? request.continue()
427
+ : request.abort("blockedbyclient");
428
+ void decided.catch(() => undefined);
429
+ });
430
+ return page;
431
+ }
432
+ /**
433
+ * Makes the page present one device, for as long as it lives.
434
+ *
435
+ * The viewport is left alone for a desktop fetch, where the launch's `defaultViewport: null`
436
+ * already hands the page the window's own metrics. `setUserAgent` carries the client hints with
437
+ * it, which `page.emulate()` would not.
438
+ *
439
+ * One difference to a real phone survives this: `HEADLESS_PARITY_ARGS` pins Blink's hover and
440
+ * pointer types to the desktop values browser-wide, so a mobile page still answers
441
+ * `hover: hover` / `pointer: fine`. That decides CSS, not the bytes of the document being
442
+ * fetched, and the parity args are what gets the browser answered at all.
443
+ */
444
+ async emulate(page, device, host) {
445
+ const emulation = deviceEmulation(this.browserVersion ?? "", device);
446
+ if (emulation.viewport)
447
+ await page.setViewport(emulation.viewport);
448
+ if (emulation.userAgent) {
449
+ await page.setUserAgent(emulation.userAgent, emulation.metadata);
450
+ }
451
+ this.logger?.comment(`[browserFetch]: ${host} is fetched as ${device}${emulation.userAgent ? ` (${emulation.userAgent})` : ""}`);
452
+ }
453
+ async browserInstance() {
454
+ if (this.browser?.connected)
455
+ return this.browser;
456
+ if (this.launch)
457
+ return this.launch;
458
+ this.launch = this.launchBrowser();
459
+ try {
460
+ this.browser = await this.launch;
461
+ }
462
+ finally {
463
+ this.launch = undefined;
464
+ }
465
+ return this.browser;
466
+ }
467
+ async launchBrowser() {
468
+ if (this.closed)
469
+ this.closed = false;
470
+ const profile = await this.createProfile();
471
+ this.profile = profile;
472
+ const proxy = currentOriginProxy();
473
+ const browser = await puppeteer.launch({
474
+ headless: true,
475
+ executablePath: path.resolve(this.userConfig.chromePath),
476
+ defaultViewport: null,
477
+ acceptInsecureCerts: true,
478
+ timeout: this.navigationTimeout,
479
+ args: [
480
+ ...FETCH_BROWSER_ARGS,
481
+ `--user-data-dir=${profile}`,
482
+ ...HEADLESS_PARITY_ARGS,
483
+ // Chrome gets the proxy the CLI is pinned to, or the fetcher would quietly leave from a
484
+ // different IP than every other origin request of the same session.
485
+ ...(proxy ? [chromeProxyFlag(proxy)] : []),
486
+ ],
487
+ ignoreDefaultArgs: ["--enable-automation", ...HEADLESS_DEFAULTS_TO_DROP],
488
+ });
489
+ // The version is only knowable once Chrome is up, so the identity is applied per page rather
490
+ // than as a launch flag. Nothing here runs a Service Worker, which is the one thing a per-page
491
+ // override would not reach.
492
+ this.browserVersion = await browser.version();
493
+ this.logger?.comment(`[browserFetch]: headless Chrome ${this.browserVersion} ready${proxy ? ` via ${chromeProxyFlag(proxy)}` : ""}`);
494
+ return browser;
495
+ }
496
+ /**
497
+ * Creates this browser's own profile directory, pruning anything a crashed run left behind.
498
+ *
499
+ * Pruning is by age rather than by wiping the parent: two `sk onboarding` sessions can run at
500
+ * once, and one must not delete the profile the other is holding open.
501
+ */
502
+ async createProfile() {
503
+ const parent = path.resolve(this.userConfig.tempFolder, PROFILE_FOLDER);
504
+ mkdirSync(parent, { recursive: true });
505
+ this.pruneProfiles(parent);
506
+ return mkdtemp(path.join(parent, "profile-"));
507
+ }
508
+ pruneProfiles(parent) {
509
+ const stale = safe(() => readdirSync(parent));
510
+ if (stale.success !== true)
511
+ return;
512
+ for (const entry of stale.data) {
513
+ const candidate = path.join(parent, entry);
514
+ const stats = safe(() => statSync(candidate));
515
+ if (stats.success !== true)
516
+ continue;
517
+ if (Date.now() - stats.data.mtimeMs < PROFILE_MAX_AGE_MS)
518
+ continue;
519
+ rmSync(candidate, { force: true, recursive: true });
520
+ }
521
+ }
522
+ warnOnce(key, message) {
523
+ if (this.warned.has(key))
524
+ return;
525
+ this.warned.add(key);
526
+ this.logger?.writeWarning(message);
527
+ }
528
+ }
@@ -0,0 +1,27 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import { isDocumentRequest } from "./browser-origin-fetcher.js";
4
+ const mainFrame = { name: "main" };
5
+ const childFrame = { name: "child" };
6
+ function request(isNavigation, frame) {
7
+ return {
8
+ isNavigationRequest: () => isNavigation,
9
+ frame: () => frame,
10
+ };
11
+ }
12
+ describe("isDocumentRequest", () => {
13
+ it("allows the document the fetch was for", () => {
14
+ expect(isDocumentRequest(request(true, mainFrame), mainFrame)).to.equal(true);
15
+ });
16
+ it("blocks a subresource of that document", () => {
17
+ // The measured reason this matters: one home page pulled 87 subresources and one category page
18
+ // 182, which is what got the session rate-blocked.
19
+ expect(isDocumentRequest(request(false, mainFrame), mainFrame)).to.equal(false);
20
+ });
21
+ it("blocks an iframe's document, which is a subresource of this fetch", () => {
22
+ expect(isDocumentRequest(request(true, childFrame), mainFrame)).to.equal(false);
23
+ });
24
+ it("blocks a subresource requested by an iframe", () => {
25
+ expect(isDocumentRequest(request(false, childFrame), mainFrame)).to.equal(false);
26
+ });
27
+ });
@@ -0,0 +1,21 @@
1
+ import { HTTPResponse } from "puppeteer";
2
+ /**
3
+ * Turns Chrome's response into the `Response` an `originFetch` caller expects.
4
+ *
5
+ * Two details a plain `new Response()` gets wrong, both of which call sites depend on:
6
+ * - `url` is empty on a constructed response, and `VirtualOrestesApp` reads it to decide whether
7
+ * a CDN redirected the request. Left empty, every request would look like a redirect.
8
+ * - `redirected` is likewise always false.
9
+ * Both are installed as own properties, which is what the getters answer for.
10
+ *
11
+ * @param response - The response Chrome received.
12
+ * @param body - Bytes of that response, already decompressed.
13
+ */
14
+ export declare function toFetchResponse(response: HTTPResponse, body: Buffer, redirected: boolean): Response;
15
+ /**
16
+ * Flattens a `RequestInit`'s headers into the plain map Chrome's `setExtraHTTPHeaders` takes.
17
+ *
18
+ * Names come back lowercased so a caller's `User-Agent` and our own `user-agent` cannot both end up
19
+ * on the request.
20
+ */
21
+ export declare function toPlainHeaders(headers?: HeadersInit): Record<string, string>;
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Statuses the `Response` constructor refuses a body for.
3
+ */
4
+ const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);
5
+ /**
6
+ * Headers that describe the transport of the body Chrome already unwrapped.
7
+ *
8
+ * `HTTPResponse.buffer()` answers the decompressed bytes, so carrying the origin's
9
+ * `content-encoding` and `content-length` over would describe a body nobody is holding. Charset is
10
+ * untouched: those bytes really are still in the origin's charset, and `getResponseText` transcodes
11
+ * them from `content-type`.
12
+ */
13
+ const TRANSPORT_HEADERS = new Set(["content-encoding", "content-length"]);
14
+ /**
15
+ * Turns Chrome's response into the `Response` an `originFetch` caller expects.
16
+ *
17
+ * Two details a plain `new Response()` gets wrong, both of which call sites depend on:
18
+ * - `url` is empty on a constructed response, and `VirtualOrestesApp` reads it to decide whether
19
+ * a CDN redirected the request. Left empty, every request would look like a redirect.
20
+ * - `redirected` is likewise always false.
21
+ * Both are installed as own properties, which is what the getters answer for.
22
+ *
23
+ * @param response - The response Chrome received.
24
+ * @param body - Bytes of that response, already decompressed.
25
+ */
26
+ export function toFetchResponse(response, body, redirected) {
27
+ const status = response.status();
28
+ const result = new Response(
29
+ // Copied into a plain `Uint8Array`: a Node `Buffer` is typed over `ArrayBufferLike`, which
30
+ // the DOM lib's `BufferSource` does not accept.
31
+ NULL_BODY_STATUSES.has(status) ? null : new Uint8Array(body), {
32
+ status,
33
+ statusText: response.statusText(),
34
+ headers: toHeaders(response.headers()),
35
+ });
36
+ Object.defineProperties(result, {
37
+ url: { value: response.url(), enumerable: true },
38
+ redirected: { value: redirected, enumerable: true },
39
+ });
40
+ return result;
41
+ }
42
+ /**
43
+ * Builds the response headers, splitting the ones Chrome folded into a single value.
44
+ *
45
+ * CDP reports repeated headers as one newline-joined string, and `Headers` rejects a value with a
46
+ * line break. `set-cookie` is the one that matters — `origin-request` reads it back with
47
+ * `getSetCookie()` — but any repeated header arrives the same way, so all of them are split.
48
+ */
49
+ function toHeaders(raw) {
50
+ const headers = new Headers();
51
+ for (const [name, value] of Object.entries(raw)) {
52
+ if (TRANSPORT_HEADERS.has(name.toLowerCase()))
53
+ continue;
54
+ for (const single of value.split("\n")) {
55
+ // A header Chrome reports empty is still a header, but an empty line from the split is not.
56
+ if (single === "" && value !== "")
57
+ continue;
58
+ headers.append(name, single);
59
+ }
60
+ }
61
+ return headers;
62
+ }
63
+ /**
64
+ * Flattens a `RequestInit`'s headers into the plain map Chrome's `setExtraHTTPHeaders` takes.
65
+ *
66
+ * Names come back lowercased so a caller's `User-Agent` and our own `user-agent` cannot both end up
67
+ * on the request.
68
+ */
69
+ export function toPlainHeaders(headers) {
70
+ if (!headers)
71
+ return {};
72
+ const result = {};
73
+ if (headers instanceof Headers) {
74
+ headers.forEach((value, name) => {
75
+ result[name.toLowerCase()] = value;
76
+ });
77
+ return result;
78
+ }
79
+ const entries = Array.isArray(headers)
80
+ ? headers
81
+ : Object.entries(headers);
82
+ for (const [name, value] of entries) {
83
+ result[String(name).toLowerCase()] = String(value);
84
+ }
85
+ return result;
86
+ }