@speedkit/cli 4.28.0 → 4.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +5 -2
- package/dist/commands/onboarding.js +2 -2
- package/dist/helpers/evaluate-speed-kit-config.d.ts +5 -0
- package/dist/helpers/evaluate-speed-kit-config.js +14 -7
- package/dist/helpers/origin-proxy.d.ts +30 -0
- package/dist/helpers/origin-proxy.js +29 -0
- package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
- package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
- package/dist/services/browser-fetch/browser-response.d.ts +21 -0
- package/dist/services/browser-fetch/browser-response.js +86 -0
- package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-response.spec.js +90 -0
- package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
- package/dist/services/browser-fetch/device-emulation.js +187 -0
- package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
- package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
- package/dist/services/browser-fetch/index.d.ts +4 -0
- package/dist/services/browser-fetch/index.js +4 -0
- package/dist/services/customer-config/customer-config-service-model.d.ts +6 -0
- package/dist/services/customer-config/customer-config-service-model.js +1 -0
- package/dist/services/customer-config/customer-config-service.d.ts +5 -0
- package/dist/services/customer-config/customer-config-service.js +15 -0
- package/dist/services/deploy/checks/activation/customer-approval-check.d.ts +21 -0
- package/dist/services/deploy/checks/activation/customer-approval-check.js +59 -0
- package/dist/services/deploy/checks/activation/customer-approval-check.spec.d.ts +1 -0
- package/dist/services/deploy/checks/activation/customer-approval-check.spec.js +64 -0
- package/dist/services/deploy/deploy-service-factory.js +2 -1
- package/dist/services/deploy/handler/install-resource-handler.d.ts +7 -1
- package/dist/services/deploy/handler/install-resource-handler.js +40 -19
- package/dist/services/deploy/handler/testing-installation-links.d.ts +10 -0
- package/dist/services/deploy/handler/testing-installation-links.js +27 -0
- package/dist/services/deploy/handler/testing-installation-links.spec.d.ts +1 -0
- package/dist/services/deploy/handler/testing-installation-links.spec.js +43 -0
- package/dist/services/integration-api/struct/customer-config.d.ts +7 -1
- package/dist/services/integration-api/struct/customer-config.js +10 -3
- package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
- package/dist/services/onboarding/onboarding-model.d.ts +3 -1
- package/dist/services/onboarding/onboarding-model.js +8 -2
- package/dist/services/onboarding/onboarding-service-factory.js +12 -1
- package/dist/services/onboarding/onboarding-service.d.ts +3 -1
- package/dist/services/onboarding/onboarding-service.js +7 -1
- package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
- package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
- package/dist/services/origin-request/asset-variations.d.ts +2 -0
- package/dist/services/origin-request/asset-variations.js +2 -2
- package/oclif.manifest.json +118 -112
- package/package.json +1 -1
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { Frame, HTTPRequest } from "puppeteer";
|
|
2
|
+
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
3
|
+
import { OriginFetchOptions } from "../../helpers/origin-proxy.js";
|
|
4
|
+
/** Only the writers this fetcher uses, so it does not depend on the whole CLI service. */
|
|
5
|
+
export interface BrowserFetchLogger {
|
|
6
|
+
writeWarning(message: string): void;
|
|
7
|
+
comment(message: string): void;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Whether a request is the document the fetch was for, rather than something the page went on to
|
|
11
|
+
* want.
|
|
12
|
+
*
|
|
13
|
+
* A redirect hop is a navigation request too, so a redirected fetch still reaches its destination.
|
|
14
|
+
* The main-frame check is what keeps an iframe's document out: an ad frame's document is a
|
|
15
|
+
* subresource of this fetch, whatever it is to the frame.
|
|
16
|
+
*
|
|
17
|
+
* @param request - Request Chrome paused.
|
|
18
|
+
* @param mainFrame - The page's main frame.
|
|
19
|
+
*/
|
|
20
|
+
export declare function isDocumentRequest(request: Pick<HTTPRequest, "isNavigationRequest" | "frame">, mainFrame: Frame): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Fetches customer origins with a real browser instead of a socket.
|
|
23
|
+
*
|
|
24
|
+
* Bot protection scores the client, not the request: a managed challenge answers Chrome and
|
|
25
|
+
* refuses every server-side fetcher, whatever library, HTTP version or header set it uses — a
|
|
26
|
+
* `cf_clearance` cookie lifted out of a browser does not travel either, because it is bound to the
|
|
27
|
+
* client that earned it. A browser is therefore the only transport that reaches such an origin, so
|
|
28
|
+
* this fetcher drives one and adapts its answers back into `Response`.
|
|
29
|
+
*
|
|
30
|
+
* What it costs, and why it is not the default:
|
|
31
|
+
* - a second Chrome, launched lazily on the first request and held until `close()`;
|
|
32
|
+
* - one page per origin host and device, kept open so the clearance cookie stays warm —
|
|
33
|
+
* cross-origin requests from a parked page are refused by CORS, so each host needs its own;
|
|
34
|
+
* - requests to one host are serialised, because a page navigates to one document at a time.
|
|
35
|
+
*
|
|
36
|
+
* Headless is safe here only because of `HEADLESS_PARITY_ARGS` and the headed user agent: measured
|
|
37
|
+
* against a Cloudflare-protected origin, plain headless Chrome is hard-blocked on the first
|
|
38
|
+
* response and never recovers, while the same launch with the parity treatment is answered 200
|
|
39
|
+
* without ever seeing an interstitial. The user agent is the discriminator, so both are load-bearing.
|
|
40
|
+
*
|
|
41
|
+
* That is also why no caller's user agent reaches the origin. The Asset API's device variations are
|
|
42
|
+
* reproduced by emulating the device instead — viewport, touch and client hints included — with a
|
|
43
|
+
* user agent built from this very Chrome. A `(compatible; SpeedKit/1.0)` suffix, or any string that
|
|
44
|
+
* disagrees with what the engine answers to a fingerprint, gives the browser away and costs exactly
|
|
45
|
+
* the access this fetcher exists to get. See `device-emulation.ts`.
|
|
46
|
+
*/
|
|
47
|
+
export declare class BrowserOriginFetcher {
|
|
48
|
+
private readonly userConfig;
|
|
49
|
+
private readonly logger?;
|
|
50
|
+
private readonly navigationTimeout;
|
|
51
|
+
private browser?;
|
|
52
|
+
/** In flight during the launch, so concurrent first requests share one browser. */
|
|
53
|
+
private launch?;
|
|
54
|
+
/** Keyed by device and host, because emulation belongs to a page and outlives a request. */
|
|
55
|
+
private readonly pages;
|
|
56
|
+
/** Tail of the request chain per host, which is how requests to one host are serialised. */
|
|
57
|
+
private readonly queues;
|
|
58
|
+
/** Version of the running Chrome, which every emulated user agent is built from. */
|
|
59
|
+
private browserVersion?;
|
|
60
|
+
private readonly warned;
|
|
61
|
+
/**
|
|
62
|
+
* Pages currently allowed to load anything, which is only ever true while a challenge is being
|
|
63
|
+
* waited out. A protection layer other than Cloudflare serves its interstitial from paths this
|
|
64
|
+
* fetcher cannot know, so waiting for one has to lift the document-only rule.
|
|
65
|
+
*/
|
|
66
|
+
private readonly permissive;
|
|
67
|
+
private closed;
|
|
68
|
+
/** Profile directory of the running browser, removed with it. */
|
|
69
|
+
private profile?;
|
|
70
|
+
/** When the session was last thrown away, so recovery cannot run per request. */
|
|
71
|
+
private lastRecycle;
|
|
72
|
+
constructor(userConfig: UserCliConfig, logger?: BrowserFetchLogger, navigationTimeout?: number);
|
|
73
|
+
/**
|
|
74
|
+
* Fetches one origin URL through the browser.
|
|
75
|
+
*
|
|
76
|
+
* `options.tolerateLegacyTls` has no counterpart here: Chrome decides for itself what handshake
|
|
77
|
+
* it accepts, and it is already told to ignore certificate errors.
|
|
78
|
+
*
|
|
79
|
+
* @param url - Absolute origin URL.
|
|
80
|
+
* @param init - Request headers and redirect mode. Other `RequestInit` fields do not apply: a
|
|
81
|
+
* navigation is always a GET.
|
|
82
|
+
*/
|
|
83
|
+
fetch(url: string, init?: RequestInit, options?: OriginFetchOptions): Promise<Response>;
|
|
84
|
+
/** Closes the browser and removes its profile. A later `fetch` launches a new one. */
|
|
85
|
+
close(): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* Throws the running browser and its profile away, leaving the fetcher usable.
|
|
88
|
+
*
|
|
89
|
+
* Everything derived from that browser goes with it: its pages, the permissive marks on them,
|
|
90
|
+
* and the user agent read from its version.
|
|
91
|
+
*/
|
|
92
|
+
private discardBrowser;
|
|
93
|
+
/**
|
|
94
|
+
* Starts a new session after the origin refused this one, and reports whether it did.
|
|
95
|
+
*
|
|
96
|
+
* The refusal is recorded against the session's cookies, so a new profile is the only way back.
|
|
97
|
+
* Rate-limited by `RECYCLE_COOLDOWN_MS`: against an origin that refuses us however we come, the
|
|
98
|
+
* answer has to settle on "refused" rather than launch a browser per request.
|
|
99
|
+
*/
|
|
100
|
+
private recycleSession;
|
|
101
|
+
/**
|
|
102
|
+
* Runs `task` after every request already queued for `host`.
|
|
103
|
+
*
|
|
104
|
+
* The chain is kept whatever a task does, so one failed request does not strand the ones behind
|
|
105
|
+
* it, and the tail is dropped once it is the last link — otherwise the map would grow a rejected
|
|
106
|
+
* promise per host for the lifetime of the process.
|
|
107
|
+
*/
|
|
108
|
+
private enqueue;
|
|
109
|
+
private fetchOnPage;
|
|
110
|
+
/**
|
|
111
|
+
* Repeats the fetch on a browser started after the refusal, as the same device.
|
|
112
|
+
*
|
|
113
|
+
* Throws when the replacement is refused too, so the caller falls back to reporting what the
|
|
114
|
+
* first attempt said.
|
|
115
|
+
*/
|
|
116
|
+
private fetchFresh;
|
|
117
|
+
/**
|
|
118
|
+
* Navigates the page, waiting for the document rather than for the whole page.
|
|
119
|
+
*
|
|
120
|
+
* `domcontentloaded` rather than `commit`: a challenge only clears itself once its script has
|
|
121
|
+
* run, and a response that needs no challenge is already complete by then anyway.
|
|
122
|
+
*
|
|
123
|
+
* The page's identity is not touched here — it was emulated once when the page was opened, and
|
|
124
|
+
* changing a user agent between two navigations of one page is itself a signal.
|
|
125
|
+
*/
|
|
126
|
+
private navigate;
|
|
127
|
+
/**
|
|
128
|
+
* Waits out a bot-protection interstitial and reports what the origin answered afterwards.
|
|
129
|
+
*
|
|
130
|
+
* An interstitial replaces itself with the real document once its scripts have run, so the page
|
|
131
|
+
* is polled rather than reloaded — a reload would start the challenge over. The document-only
|
|
132
|
+
* rule is lifted for the wait, because those scripts are the whole mechanism. When the wait runs
|
|
133
|
+
* out, the blocked response is returned as it is: the caller has a status to report and a cause
|
|
134
|
+
* to name, which is more useful than an exception.
|
|
135
|
+
*/
|
|
136
|
+
private afterChallenge;
|
|
137
|
+
private waitOutChallenge;
|
|
138
|
+
/**
|
|
139
|
+
* Reads the response body and hands back a `Response`.
|
|
140
|
+
*
|
|
141
|
+
* With `redirect: "manual"` the first hop is answered instead of the destination: Chrome always
|
|
142
|
+
* follows a redirect, so the hop is recovered from the request's redirect chain. A body is never
|
|
143
|
+
* available for it, which matches what a manual redirect carries anyway.
|
|
144
|
+
*/
|
|
145
|
+
private adapt;
|
|
146
|
+
/**
|
|
147
|
+
* Whether the origin answered, or its protection layer did — and if so, whether waiting helps.
|
|
148
|
+
*
|
|
149
|
+
* The status alone decides nothing: an origin may answer 403 for its own reasons, and a
|
|
150
|
+
* protection layer may answer 200 with an interstitial. So Cloudflare's own marker and the
|
|
151
|
+
* document's title are what separate the three outcomes.
|
|
152
|
+
*
|
|
153
|
+
* Read out of the response body rather than with `page.title()`, which would evaluate script in
|
|
154
|
+
* a page a bot detector is watching. Puppeteer memoises the body, so the read `adapt` does later
|
|
155
|
+
* costs nothing. The body is only searched as far as a `<title>` can reach.
|
|
156
|
+
*/
|
|
157
|
+
private verdictOn;
|
|
158
|
+
/**
|
|
159
|
+
* The start of the response body, which is as far as a `<title>` can be.
|
|
160
|
+
*
|
|
161
|
+
* A body that cannot be read is not a verdict either way, so it reads as empty and the status
|
|
162
|
+
* and headers decide alone.
|
|
163
|
+
*/
|
|
164
|
+
private headOf;
|
|
165
|
+
/** The tab's title, or nothing when the page is in a state that cannot report one. */
|
|
166
|
+
private titleOf;
|
|
167
|
+
/**
|
|
168
|
+
* The page this host is fetched on as this device.
|
|
169
|
+
*
|
|
170
|
+
* A device gets its own page rather than a re-emulated one: emulation is page state, and a page
|
|
171
|
+
* that changes device between two navigations tells the origin more about us than either device
|
|
172
|
+
* would have.
|
|
173
|
+
*/
|
|
174
|
+
private pageFor;
|
|
175
|
+
/**
|
|
176
|
+
* Opens a page that loads documents and nothing else.
|
|
177
|
+
*
|
|
178
|
+
* Without this a fetch costs the origin a whole page load — measured against one customer, 87
|
|
179
|
+
* subresource requests for the home page and 182 for a category page. Three such fetches in a
|
|
180
|
+
* row were enough for that origin to blackhole the session, and every request after that was
|
|
181
|
+
* refused. Aborting subresources keeps one fetch to one origin request, which is both what the
|
|
182
|
+
* callers want and what keeps the fetcher from behaving like an attack. It also makes a fetch
|
|
183
|
+
* 3-5x faster, because nothing waits for images or scripts.
|
|
184
|
+
*
|
|
185
|
+
* The rule also keeps the browser out of Cloudflare's JS fingerprinting, which is itself a
|
|
186
|
+
* subresource: measured against the same customer, exempting `/cdn-cgi/` let the detection run,
|
|
187
|
+
* and it hard-blocked the session from the third request onwards — where aborting it left eight
|
|
188
|
+
* consecutive fetches answered. `afterChallenge` lifts the rule only when an interstitial has
|
|
189
|
+
* actually appeared, which is the one case those requests are needed for.
|
|
190
|
+
*/
|
|
191
|
+
private openPage;
|
|
192
|
+
/**
|
|
193
|
+
* Makes the page present one device, for as long as it lives.
|
|
194
|
+
*
|
|
195
|
+
* The viewport is left alone for a desktop fetch, where the launch's `defaultViewport: null`
|
|
196
|
+
* already hands the page the window's own metrics. `setUserAgent` carries the client hints with
|
|
197
|
+
* it, which `page.emulate()` would not.
|
|
198
|
+
*
|
|
199
|
+
* One difference to a real phone survives this: `HEADLESS_PARITY_ARGS` pins Blink's hover and
|
|
200
|
+
* pointer types to the desktop values browser-wide, so a mobile page still answers
|
|
201
|
+
* `hover: hover` / `pointer: fine`. That decides CSS, not the bytes of the document being
|
|
202
|
+
* fetched, and the parity args are what gets the browser answered at all.
|
|
203
|
+
*/
|
|
204
|
+
private emulate;
|
|
205
|
+
private browserInstance;
|
|
206
|
+
private launchBrowser;
|
|
207
|
+
/**
|
|
208
|
+
* Creates this browser's own profile directory, pruning anything a crashed run left behind.
|
|
209
|
+
*
|
|
210
|
+
* Pruning is by age rather than by wiping the parent: two `sk onboarding` sessions can run at
|
|
211
|
+
* once, and one must not delete the profile the other is holding open.
|
|
212
|
+
*/
|
|
213
|
+
private createProfile;
|
|
214
|
+
private pruneProfiles;
|
|
215
|
+
private warnOnce;
|
|
216
|
+
}
|