@speedkit/cli 4.27.2 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +5 -2
- package/dist/commands/onboarding.js +2 -2
- package/dist/helpers/origin-proxy.d.ts +30 -0
- package/dist/helpers/origin-proxy.js +29 -0
- package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
- package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
- package/dist/services/browser-fetch/browser-response.d.ts +21 -0
- package/dist/services/browser-fetch/browser-response.js +86 -0
- package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
- package/dist/services/browser-fetch/browser-response.spec.js +90 -0
- package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
- package/dist/services/browser-fetch/device-emulation.js +187 -0
- package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
- package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
- package/dist/services/browser-fetch/index.d.ts +4 -0
- package/dist/services/browser-fetch/index.js +4 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
- package/dist/services/onboarding/dashboard/index.js +40 -1
- package/dist/services/onboarding/onboarding-model.d.ts +3 -1
- package/dist/services/onboarding/onboarding-model.js +8 -2
- package/dist/services/onboarding/onboarding-service-factory.js +12 -1
- package/dist/services/onboarding/onboarding-service.d.ts +3 -1
- package/dist/services/onboarding/onboarding-service.js +7 -1
- package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
- package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
- package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
- package/dist/services/origin-request/asset-variations.d.ts +2 -0
- package/dist/services/origin-request/asset-variations.js +2 -2
- package/dist/services/query-builder/query-builder-service.d.ts +18 -2
- package/dist/services/query-builder/query-builder-service.js +26 -10
- package/oclif.manifest.json +7 -1
- package/package.json +1 -1
|
@@ -18,10 +18,11 @@ export class OnboardingService {
|
|
|
18
18
|
developmentToolsMessages;
|
|
19
19
|
extensionBridge;
|
|
20
20
|
speedKitServiceWorkerJs;
|
|
21
|
+
browserFetch;
|
|
21
22
|
reinstallOnNextNavigate = false;
|
|
22
23
|
lastForeignSwReinstall = 0;
|
|
23
24
|
activatedSwVersions = new Set();
|
|
24
|
-
constructor(browserContext, customerConfig, fetchEventHandler, fileWatcher, cli, developmentToolsMessages, extensionBridge, speedKitServiceWorkerJs) {
|
|
25
|
+
constructor(browserContext, customerConfig, fetchEventHandler, fileWatcher, cli, developmentToolsMessages, extensionBridge, speedKitServiceWorkerJs, browserFetch) {
|
|
25
26
|
this.browserContext = browserContext;
|
|
26
27
|
this.customerConfig = customerConfig;
|
|
27
28
|
this.fetchEventHandler = fetchEventHandler;
|
|
@@ -30,6 +31,7 @@ export class OnboardingService {
|
|
|
30
31
|
this.developmentToolsMessages = developmentToolsMessages;
|
|
31
32
|
this.extensionBridge = extensionBridge;
|
|
32
33
|
this.speedKitServiceWorkerJs = speedKitServiceWorkerJs;
|
|
34
|
+
this.browserFetch = browserFetch;
|
|
33
35
|
}
|
|
34
36
|
/**
|
|
35
37
|
* Prepare the puppeteer browser, wire event listeners, navigate to the
|
|
@@ -51,6 +53,10 @@ export class OnboardingService {
|
|
|
51
53
|
await this.navigate(page, `${this.customerConfig.scope}speed-kit-install.html?url=${this.customerConfig.scope}`);
|
|
52
54
|
await waitForAbort(shutdownSignal);
|
|
53
55
|
await safe(browser.close());
|
|
56
|
+
// The fetch browser only exists if an origin refused a direct request, but closing an unused
|
|
57
|
+
// handle is free and leaving a Chrome behind is not.
|
|
58
|
+
if (this.browserFetch)
|
|
59
|
+
await safe(this.browserFetch.close());
|
|
54
60
|
}
|
|
55
61
|
async prepareBrowser(localController) {
|
|
56
62
|
const browser = await puppeteerExtra.default.launch(this.browserContext.getPuppeteerLaunchOptions());
|
|
@@ -60,6 +60,19 @@ export declare class VirtualOrestesApp {
|
|
|
60
60
|
* the page as an infrastructure failure and disconnect Speed Kit.
|
|
61
61
|
*/
|
|
62
62
|
private returnAbortResponse;
|
|
63
|
+
/**
|
|
64
|
+
* Answers an origin error the way the Asset API answers it, instead of caching it.
|
|
65
|
+
*
|
|
66
|
+
* A bot-protection page is served as an error status but still carries
|
|
67
|
+
* `content-type: text/html` and a body with `<html`, so it used to clear both content
|
|
68
|
+
* checks, run through the Document Handler and land in the dh-cache as the document. Every
|
|
69
|
+
* result after that was worthless, and nothing said why.
|
|
70
|
+
*
|
|
71
|
+
* The response is deliberately not cached. The service worker falls back to origin, which a
|
|
72
|
+
* real browser can reach even when the CLI cannot, so the page still renders while devtools
|
|
73
|
+
* names the cause.
|
|
74
|
+
*/
|
|
75
|
+
private returnOriginErrorResponse;
|
|
63
76
|
private returnErrorResponse;
|
|
64
77
|
private convertResponseHeadersToOrestesFormat;
|
|
65
78
|
private returnRedirect;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { BaqendResponse } from "../browser/baqend-response.js";
|
|
2
2
|
import { ABORT_SERVER_TIMING, ABORT_STATUS_CODE, isDocumentHandlerAbort, } from "../../document-handler-runtime/document-handler-abort.js";
|
|
3
|
+
import { isBlockedOriginStatus, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "./origin-error-cause.js";
|
|
3
4
|
import { safe } from "../../../helpers/safe.js";
|
|
4
5
|
import { META_CONTENT_TYPE_REGEX, } from "../onboarding-model.js";
|
|
5
6
|
import iconv from "iconv-lite";
|
|
@@ -118,6 +119,11 @@ export class VirtualOrestesApp {
|
|
|
118
119
|
const response = fetchResult?.response;
|
|
119
120
|
// URL we actually fetched (origin + variation query params); used for the redirect check below
|
|
120
121
|
const fetchedUrl = fetchResult?.url;
|
|
122
|
+
// Reported before the content-type gate below: a blocked origin answers with whatever its
|
|
123
|
+
// protection layer serves, and the cause has to reach devtools either way.
|
|
124
|
+
if (response && response.status >= 400) {
|
|
125
|
+
return this.returnOriginErrorResponse(response, originUrl, requestUrl, variation);
|
|
126
|
+
}
|
|
121
127
|
// todo check if we can handle more then htmlResponses
|
|
122
128
|
// only handle html responses
|
|
123
129
|
if (!response?.headers?.get("content-type")?.includes("html")) {
|
|
@@ -248,6 +254,36 @@ export class VirtualOrestesApp {
|
|
|
248
254
|
customHeaders.push({ name: "x-error", value: message }, { name: "server-timing", value: ABORT_SERVER_TIMING });
|
|
249
255
|
return new BaqendResponse(message, null, customHeaders, ABORT_STATUS_CODE);
|
|
250
256
|
}
|
|
257
|
+
/**
|
|
258
|
+
* Answers an origin error the way the Asset API answers it, instead of caching it.
|
|
259
|
+
*
|
|
260
|
+
* A bot-protection page is served as an error status but still carries
|
|
261
|
+
* `content-type: text/html` and a body with `<html`, so it used to clear both content
|
|
262
|
+
* checks, run through the Document Handler and land in the dh-cache as the document. Every
|
|
263
|
+
* result after that was worthless, and nothing said why.
|
|
264
|
+
*
|
|
265
|
+
* The response is deliberately not cached. The service worker falls back to origin, which a
|
|
266
|
+
* real browser can reach even when the CLI cannot, so the page still renders while devtools
|
|
267
|
+
* names the cause.
|
|
268
|
+
*/
|
|
269
|
+
returnOriginErrorResponse(response, originUrl, requestUrl, variation) {
|
|
270
|
+
const code = originErrorCode(response.status);
|
|
271
|
+
const cause = originErrorCauseName(code);
|
|
272
|
+
const message = `origin answered ${response.status} for ${originUrl} — not cached (${cause}, errorcode ${code})`;
|
|
273
|
+
this.cli.writeError(`[VirtualOrestes]: ${message}`);
|
|
274
|
+
if (isBlockedOriginStatus(response.status)) {
|
|
275
|
+
this.cli.writeWarning(`the origin blocked this request (server: ${response.headers.get("server") ?? "unknown"}). ` +
|
|
276
|
+
"Bot protection lets a real browser through but not a server-side fetch, so no cache " +
|
|
277
|
+
"entry can be built for this URL. Ask the customer to skip bot management for our egress IP.");
|
|
278
|
+
}
|
|
279
|
+
this.recordLocalCacheError(originUrl, requestUrl, variation, {
|
|
280
|
+
success: false,
|
|
281
|
+
error: message,
|
|
282
|
+
});
|
|
283
|
+
const customHeaders = this.prepareCustomHeaders(originUrl);
|
|
284
|
+
customHeaders.push({ name: "x-error", value: message }, { name: "server-timing", value: originErrorServerTiming(code) });
|
|
285
|
+
return new BaqendResponse(message, null, customHeaders, ORIGIN_ERROR_STATUS_CODE);
|
|
286
|
+
}
|
|
251
287
|
returnErrorResponse(customResponse, customHeaders) {
|
|
252
288
|
if (isDocumentHandlerAbort(customResponse.errorObj)) {
|
|
253
289
|
return this.returnAbortResponse(customResponse, customHeaders);
|
|
@@ -286,7 +322,9 @@ export class VirtualOrestesApp {
|
|
|
286
322
|
const serverTimingHeader = customHeaders.find((header) => {
|
|
287
323
|
return header.name === "server-timing";
|
|
288
324
|
});
|
|
289
|
-
|
|
325
|
+
// `errorcode` is the metric the service worker reads — an `error` metric was ignored, so
|
|
326
|
+
// the CDN redirect surfaced as a bare ClientError (`AssetAPIHandler.ts:200-205`).
|
|
327
|
+
serverTimingHeader.value += ",errorcode;desc=202";
|
|
290
328
|
const redirectResponse = new BaqendResponse("", null, customHeaders, 400);
|
|
291
329
|
this.cache.addEntry(requestUrl, redirectResponse);
|
|
292
330
|
return redirectResponse;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response causes for an origin answer the local Asset API refuses to cache.
|
|
3
|
+
*
|
|
4
|
+
* `VirtualOrestesApp` stands in for Baqend's asset server, so an origin error has to surface
|
|
5
|
+
* the way production surfaces it: a client-error status plus an `errorcode` metric on
|
|
6
|
+
* `server-timing`. The service worker builds the cause as
|
|
7
|
+
* `` `${errorCause?.desc || ClientError}${errorCode ? `-${errorCode.desc}` : ''}` ``
|
|
8
|
+
* (`AssetAPIHandler.ts:200-205`), so `errorcode;desc=109` reports `ClientError-109` and the
|
|
9
|
+
* code resolves to `OriginBlockedStatus` further downstream.
|
|
10
|
+
*
|
|
11
|
+
* Only the `errorcode` metric is sent, which is what Baqend sends. An `errorCause` metric
|
|
12
|
+
* would invent a cause name the backend never emits — the same reasoning as
|
|
13
|
+
* `ABORT_SERVER_TIMING` in `document-handler-abort.ts`.
|
|
14
|
+
*
|
|
15
|
+
* Codes and names come from "Response Source & Cause Combinations", which is the source of
|
|
16
|
+
* truth for the mapping:
|
|
17
|
+
* https://www.notion.so/baqend/Response-Source-Cause-Combinations-3aab063103384229a11952b8c5ab1a20
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Status the local Asset API answers an origin error with.
|
|
21
|
+
*
|
|
22
|
+
* It must stay below 500: the service worker treats 5xx from the Asset API as an
|
|
23
|
+
* infrastructure failure and disconnects Speed Kit, while a client error is classified from
|
|
24
|
+
* `server-timing` and falls back to origin (`AssetAPIHandler.ts:192-208`).
|
|
25
|
+
*/
|
|
26
|
+
export declare const ORIGIN_ERROR_STATUS_CODE = 400;
|
|
27
|
+
/**
|
|
28
|
+
* Statuses Orestes counts as "the origin blocked us" rather than as an ordinary error.
|
|
29
|
+
* Straight from the `OriginBlockedStatus` definition: 403, 406, 429, 430.
|
|
30
|
+
*/
|
|
31
|
+
export declare const BLOCKED_ORIGIN_STATUSES: Set<number>;
|
|
32
|
+
/** Speed Kit error codes for the origin answers this module classifies. */
|
|
33
|
+
export declare const ORIGIN_ERROR_CODE: {
|
|
34
|
+
/** 109 — the origin blocked the request (bot protection, WAF, rate limit). */
|
|
35
|
+
readonly BLOCKED: "109";
|
|
36
|
+
/** 104 — the origin answered with something Orestes considers a 410. */
|
|
37
|
+
readonly GONE: "104";
|
|
38
|
+
/** 105 — the origin answered with something Orestes considers a 404. */
|
|
39
|
+
readonly NOT_FOUND: "105";
|
|
40
|
+
/** 100 — catch-all for an origin error that fits none of the specific codes. */
|
|
41
|
+
readonly GENERIC: "100";
|
|
42
|
+
};
|
|
43
|
+
/** Cause name each code resolves to, for log output. The service worker only sends the code. */
|
|
44
|
+
export declare const ORIGIN_ERROR_CAUSE_NAME: Record<string, string>;
|
|
45
|
+
/** Whether the origin refused the request rather than failing to answer it. */
|
|
46
|
+
export declare function isBlockedOriginStatus(status: number): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Maps an origin status onto the Speed Kit error code that describes it.
|
|
49
|
+
*
|
|
50
|
+
* @param status - Status the origin answered with.
|
|
51
|
+
*/
|
|
52
|
+
export declare function originErrorCode(status: number): string;
|
|
53
|
+
/** Cause name for an error code, or the code itself when the mapping has no name for it. */
|
|
54
|
+
export declare function originErrorCauseName(code: string): string;
|
|
55
|
+
/**
|
|
56
|
+
* `server-timing` value that makes the service worker classify the response as this cause.
|
|
57
|
+
*
|
|
58
|
+
* @param code - Speed Kit error code, from `originErrorCode`.
|
|
59
|
+
*/
|
|
60
|
+
export declare function originErrorServerTiming(code: string): string;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response causes for an origin answer the local Asset API refuses to cache.
|
|
3
|
+
*
|
|
4
|
+
* `VirtualOrestesApp` stands in for Baqend's asset server, so an origin error has to surface
|
|
5
|
+
* the way production surfaces it: a client-error status plus an `errorcode` metric on
|
|
6
|
+
* `server-timing`. The service worker builds the cause as
|
|
7
|
+
* `` `${errorCause?.desc || ClientError}${errorCode ? `-${errorCode.desc}` : ''}` ``
|
|
8
|
+
* (`AssetAPIHandler.ts:200-205`), so `errorcode;desc=109` reports `ClientError-109` and the
|
|
9
|
+
* code resolves to `OriginBlockedStatus` further downstream.
|
|
10
|
+
*
|
|
11
|
+
* Only the `errorcode` metric is sent, which is what Baqend sends. An `errorCause` metric
|
|
12
|
+
* would invent a cause name the backend never emits — the same reasoning as
|
|
13
|
+
* `ABORT_SERVER_TIMING` in `document-handler-abort.ts`.
|
|
14
|
+
*
|
|
15
|
+
* Codes and names come from "Response Source & Cause Combinations", which is the source of
|
|
16
|
+
* truth for the mapping:
|
|
17
|
+
* https://www.notion.so/baqend/Response-Source-Cause-Combinations-3aab063103384229a11952b8c5ab1a20
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Status the local Asset API answers an origin error with.
|
|
21
|
+
*
|
|
22
|
+
* It must stay below 500: the service worker treats 5xx from the Asset API as an
|
|
23
|
+
* infrastructure failure and disconnects Speed Kit, while a client error is classified from
|
|
24
|
+
* `server-timing` and falls back to origin (`AssetAPIHandler.ts:192-208`).
|
|
25
|
+
*/
|
|
26
|
+
export const ORIGIN_ERROR_STATUS_CODE = 400;
|
|
27
|
+
/**
|
|
28
|
+
* Statuses Orestes counts as "the origin blocked us" rather than as an ordinary error.
|
|
29
|
+
* Straight from the `OriginBlockedStatus` definition: 403, 406, 429, 430.
|
|
30
|
+
*/
|
|
31
|
+
export const BLOCKED_ORIGIN_STATUSES = new Set([403, 406, 429, 430]);
|
|
32
|
+
/** Speed Kit error codes for the origin answers this module classifies. */
|
|
33
|
+
export const ORIGIN_ERROR_CODE = {
|
|
34
|
+
/** 109 — the origin blocked the request (bot protection, WAF, rate limit). */
|
|
35
|
+
BLOCKED: "109",
|
|
36
|
+
/** 104 — the origin answered with something Orestes considers a 410. */
|
|
37
|
+
GONE: "104",
|
|
38
|
+
/** 105 — the origin answered with something Orestes considers a 404. */
|
|
39
|
+
NOT_FOUND: "105",
|
|
40
|
+
/** 100 — catch-all for an origin error that fits none of the specific codes. */
|
|
41
|
+
GENERIC: "100",
|
|
42
|
+
};
|
|
43
|
+
/** Cause name each code resolves to, for log output. The service worker only sends the code. */
|
|
44
|
+
export const ORIGIN_ERROR_CAUSE_NAME = {
|
|
45
|
+
[ORIGIN_ERROR_CODE.BLOCKED]: "OriginBlockedStatus",
|
|
46
|
+
[ORIGIN_ERROR_CODE.GONE]: "OriginStatusGone",
|
|
47
|
+
[ORIGIN_ERROR_CODE.NOT_FOUND]: "OriginStatusNotFound",
|
|
48
|
+
[ORIGIN_ERROR_CODE.GENERIC]: "OriginGenericError",
|
|
49
|
+
};
|
|
50
|
+
/** Whether the origin refused the request rather than failing to answer it. */
|
|
51
|
+
export function isBlockedOriginStatus(status) {
|
|
52
|
+
return BLOCKED_ORIGIN_STATUSES.has(status);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Maps an origin status onto the Speed Kit error code that describes it.
|
|
56
|
+
*
|
|
57
|
+
* @param status - Status the origin answered with.
|
|
58
|
+
*/
|
|
59
|
+
export function originErrorCode(status) {
|
|
60
|
+
if (isBlockedOriginStatus(status))
|
|
61
|
+
return ORIGIN_ERROR_CODE.BLOCKED;
|
|
62
|
+
if (status === 404)
|
|
63
|
+
return ORIGIN_ERROR_CODE.NOT_FOUND;
|
|
64
|
+
if (status === 410)
|
|
65
|
+
return ORIGIN_ERROR_CODE.GONE;
|
|
66
|
+
return ORIGIN_ERROR_CODE.GENERIC;
|
|
67
|
+
}
|
|
68
|
+
/** Cause name for an error code, or the code itself when the mapping has no name for it. */
|
|
69
|
+
export function originErrorCauseName(code) {
|
|
70
|
+
return ORIGIN_ERROR_CAUSE_NAME[code] ?? code;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* `server-timing` value that makes the service worker classify the response as this cause.
|
|
74
|
+
*
|
|
75
|
+
* @param code - Speed Kit error code, from `originErrorCode`.
|
|
76
|
+
*/
|
|
77
|
+
export function originErrorServerTiming(code) {
|
|
78
|
+
return `errorcode;desc=${code}`;
|
|
79
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { describe, it } from "mocha";
|
|
3
|
+
import { BLOCKED_ORIGIN_STATUSES, isBlockedOriginStatus, ORIGIN_ERROR_CODE, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "./origin-error-cause.js";
|
|
4
|
+
/**
|
|
5
|
+
* Mirrors the service worker's `parseServerTimingHeader`, so the header this module emits is
|
|
6
|
+
* asserted against the parser that has to read it.
|
|
7
|
+
*/
|
|
8
|
+
function parseServerTiming(header) {
|
|
9
|
+
const metrics = {};
|
|
10
|
+
header.split(",").forEach((metric) => {
|
|
11
|
+
const key = metric.includes(";") ? metric.trim().split(";")[0] : metric;
|
|
12
|
+
const descMatch = metric.match(/desc=([^;]*)/);
|
|
13
|
+
metrics[key] = { desc: descMatch ? descMatch[1] : "" };
|
|
14
|
+
});
|
|
15
|
+
return metrics;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Mirrors how the service worker builds the response cause in
|
|
19
|
+
* `AssetAPIHandler.ts:200-205`, to prove which cause the emitted header produces.
|
|
20
|
+
*/
|
|
21
|
+
function composeResponseCause(header) {
|
|
22
|
+
const metrics = parseServerTiming(header);
|
|
23
|
+
const errorCode = metrics.errorcode;
|
|
24
|
+
const errorCodeString = errorCode ? `-${errorCode.desc}` : "";
|
|
25
|
+
return `${metrics.errorCause?.desc || "ClientError"}${errorCodeString}`;
|
|
26
|
+
}
|
|
27
|
+
describe("originErrorCode", () => {
|
|
28
|
+
it("maps every blocked status onto OriginBlockedStatus", () => {
|
|
29
|
+
for (const status of BLOCKED_ORIGIN_STATUSES) {
|
|
30
|
+
expect(originErrorCode(status), `status ${status}`).to.equal(ORIGIN_ERROR_CODE.BLOCKED);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
it("keeps 404 and 410 on their own causes", () => {
|
|
34
|
+
expect(originErrorCode(404)).to.equal(ORIGIN_ERROR_CODE.NOT_FOUND);
|
|
35
|
+
expect(originErrorCode(410)).to.equal(ORIGIN_ERROR_CODE.GONE);
|
|
36
|
+
});
|
|
37
|
+
it("falls back to the generic origin error", () => {
|
|
38
|
+
expect(originErrorCode(400)).to.equal(ORIGIN_ERROR_CODE.GENERIC);
|
|
39
|
+
expect(originErrorCode(451)).to.equal(ORIGIN_ERROR_CODE.GENERIC);
|
|
40
|
+
expect(originErrorCode(503)).to.equal(ORIGIN_ERROR_CODE.GENERIC);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
describe("isBlockedOriginStatus", () => {
|
|
44
|
+
it("recognises a refusal but not an ordinary error", () => {
|
|
45
|
+
expect(isBlockedOriginStatus(403)).to.equal(true);
|
|
46
|
+
expect(isBlockedOriginStatus(429)).to.equal(true);
|
|
47
|
+
expect(isBlockedOriginStatus(404)).to.equal(false);
|
|
48
|
+
expect(isBlockedOriginStatus(500)).to.equal(false);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe("originErrorServerTiming", () => {
|
|
52
|
+
it("makes the service worker report the blocked-origin cause", () => {
|
|
53
|
+
const header = originErrorServerTiming(originErrorCode(403));
|
|
54
|
+
expect(composeResponseCause(header)).to.equal("ClientError-109");
|
|
55
|
+
});
|
|
56
|
+
it("sends only the errorcode metric, never an invented cause name", () => {
|
|
57
|
+
const metrics = parseServerTiming(originErrorServerTiming(ORIGIN_ERROR_CODE.BLOCKED));
|
|
58
|
+
expect(Object.keys(metrics)).to.deep.equal(["errorcode"]);
|
|
59
|
+
expect(metrics.errorCause).to.equal(undefined);
|
|
60
|
+
});
|
|
61
|
+
it("survives being joined onto the response's other server-timing metrics", () => {
|
|
62
|
+
const header = `pop;desc=LOCAL;dur=1,cache;desc=STREAM, ${originErrorServerTiming(ORIGIN_ERROR_CODE.BLOCKED)}`;
|
|
63
|
+
expect(composeResponseCause(header)).to.equal("ClientError-109");
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
describe("originErrorCauseName", () => {
|
|
67
|
+
it("names the causes the codes resolve to", () => {
|
|
68
|
+
expect(originErrorCauseName(ORIGIN_ERROR_CODE.BLOCKED)).to.equal("OriginBlockedStatus");
|
|
69
|
+
expect(originErrorCauseName(ORIGIN_ERROR_CODE.NOT_FOUND)).to.equal("OriginStatusNotFound");
|
|
70
|
+
});
|
|
71
|
+
it("falls back to the bare code for a cause it does not know", () => {
|
|
72
|
+
expect(originErrorCauseName("999")).to.equal("999");
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
describe("ORIGIN_ERROR_STATUS_CODE", () => {
|
|
76
|
+
it("stays below the range that disconnects Speed Kit", () => {
|
|
77
|
+
expect(ORIGIN_ERROR_STATUS_CODE).to.be.lessThan(500);
|
|
78
|
+
});
|
|
79
|
+
it("stays a client error, so the cause is read from server-timing", () => {
|
|
80
|
+
expect(ORIGIN_ERROR_STATUS_CODE).to.be.at.least(400);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
@@ -32,6 +32,8 @@ export interface ResolvedRequest {
|
|
|
32
32
|
export declare const DEFAULT_TYPE = "default";
|
|
33
33
|
/** User agents the server falls back to. Keep in sync with AssetVariations.kt. */
|
|
34
34
|
export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15 (compatible; SpeedKit/1.0)";
|
|
35
|
+
export declare const MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1 (compatible; SpeedKit/1.0)";
|
|
36
|
+
export declare const TABLET_USER_AGENT = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 (compatible; SpeedKit/1.0)";
|
|
35
37
|
/** Thrown where the server would throw UndefinedVariationException. */
|
|
36
38
|
export declare class UndefinedVariationError extends Error {
|
|
37
39
|
constructor(variationType: string);
|
|
@@ -15,8 +15,8 @@ export const DEFAULT_TYPE = "default";
|
|
|
15
15
|
const WILD_CARD = "*";
|
|
16
16
|
/** User agents the server falls back to. Keep in sync with AssetVariations.kt. */
|
|
17
17
|
export const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15 (compatible; SpeedKit/1.0)";
|
|
18
|
-
const MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1 (compatible; SpeedKit/1.0)";
|
|
19
|
-
const TABLET_USER_AGENT = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 (compatible; SpeedKit/1.0)";
|
|
18
|
+
export const MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1 (compatible; SpeedKit/1.0)";
|
|
19
|
+
export const TABLET_USER_AGENT = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 (compatible; SpeedKit/1.0)";
|
|
20
20
|
const DEVICE_DEFAULTS = {
|
|
21
21
|
[DEFAULT_TYPE]: { headers: { "user-agent": DEFAULT_USER_AGENT } },
|
|
22
22
|
mobile: { headers: { "user-agent": MOBILE_USER_AGENT } },
|
|
@@ -2,6 +2,14 @@ import { QueryBuilderContext } from "./query-builder-model.js";
|
|
|
2
2
|
import { ParsedRuleSets } from "./rules/index.js";
|
|
3
3
|
import { CliServiceInterface } from "../cli/index.js";
|
|
4
4
|
import { AthenaService } from "../athena/index.js";
|
|
5
|
+
/**
|
|
6
|
+
* The options a caller may set per query instead of per command — what the
|
|
7
|
+
* onboarding dashboard's query editor offers. Everything else a query is built
|
|
8
|
+
* from (the app, its rule sets, the catalog) is fixed for the process.
|
|
9
|
+
*/
|
|
10
|
+
export type QueryOverrides = Pick<QueryBuilderContext, "bySet" | "days" | "excludeNoindex" | "excludeSuspicious" | "hardNavigationsOnly" | "minImpressions">;
|
|
11
|
+
/** {@link QueryOverrides} with every default filled in. */
|
|
12
|
+
export type ResolvedQueryOptions = Required<QueryOverrides>;
|
|
5
13
|
export declare class QueryBuilderService {
|
|
6
14
|
private context;
|
|
7
15
|
private ruleSets;
|
|
@@ -17,8 +25,16 @@ export declare class QueryBuilderService {
|
|
|
17
25
|
* directly instead of exporting it (e.g. `auto-prewarm`).
|
|
18
26
|
*/
|
|
19
27
|
execute<T>(): Promise<T[]>;
|
|
20
|
-
/** The query as SQL. */
|
|
21
|
-
query(): string;
|
|
28
|
+
/** The query as SQL, optionally shaped by per-call overrides. */
|
|
29
|
+
query(overrides?: QueryOverrides): string;
|
|
30
|
+
/**
|
|
31
|
+
* The options a query would be built with right now, the defaults filled in.
|
|
32
|
+
*
|
|
33
|
+
* A caller that offers the options for editing — the onboarding dashboard —
|
|
34
|
+
* has to show what they currently are, and only the service knows which of
|
|
35
|
+
* them the command line left to a default.
|
|
36
|
+
*/
|
|
37
|
+
resolvedOptions(overrides?: QueryOverrides): ResolvedQueryOptions;
|
|
22
38
|
private build;
|
|
23
39
|
/**
|
|
24
40
|
* Points out where the query is not exactly what the config says — a rule
|
|
@@ -92,29 +92,45 @@ export class QueryBuilderService {
|
|
|
92
92
|
async execute() {
|
|
93
93
|
return this.runQuery(this.query());
|
|
94
94
|
}
|
|
95
|
-
/** The query as SQL. */
|
|
96
|
-
query() {
|
|
97
|
-
return this.build().sql;
|
|
95
|
+
/** The query as SQL, optionally shaped by per-call overrides. */
|
|
96
|
+
query(overrides = {}) {
|
|
97
|
+
return this.build(overrides).sql;
|
|
98
98
|
}
|
|
99
|
-
|
|
99
|
+
/**
|
|
100
|
+
* The options a query would be built with right now, the defaults filled in.
|
|
101
|
+
*
|
|
102
|
+
* A caller that offers the options for editing — the onboarding dashboard —
|
|
103
|
+
* has to show what they currently are, and only the service knows which of
|
|
104
|
+
* them the command line left to a default.
|
|
105
|
+
*/
|
|
106
|
+
resolvedOptions(overrides = {}) {
|
|
107
|
+
return {
|
|
108
|
+
bySet: overrides.bySet ?? this.context.bySet ?? false,
|
|
109
|
+
days: overrides.days ??
|
|
110
|
+
this.context.days ??
|
|
111
|
+
DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
112
|
+
excludeNoindex: overrides.excludeNoindex ?? this.context.excludeNoindex ?? false,
|
|
113
|
+
excludeSuspicious: overrides.excludeSuspicious ?? this.context.excludeSuspicious ?? false,
|
|
114
|
+
hardNavigationsOnly: overrides.hardNavigationsOnly ??
|
|
115
|
+
this.context.hardNavigationsOnly ??
|
|
116
|
+
false,
|
|
117
|
+
minImpressions: overrides.minImpressions ?? this.context.minImpressions ?? 1,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
build(overrides = {}) {
|
|
100
121
|
const input = {
|
|
101
122
|
app: this.context.app,
|
|
102
123
|
catalog: this.context.catalog,
|
|
103
|
-
bySet: this.context.bySet,
|
|
104
124
|
count: this.context.count,
|
|
105
|
-
days: this.context.days ?? DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
106
125
|
devices: this.context.devices,
|
|
107
126
|
excludeNotFound: this.context.excludeNotFound,
|
|
108
|
-
excludeNoindex: this.context.excludeNoindex,
|
|
109
|
-
excludeSuspicious: this.context.excludeSuspicious,
|
|
110
|
-
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
111
127
|
hosts: this.context.hosts,
|
|
112
128
|
limit: this.context.limit,
|
|
113
129
|
match: this.context.match,
|
|
114
130
|
minHits: this.context.minHits,
|
|
115
|
-
minImpressions: this.context.minImpressions,
|
|
116
131
|
ruleSets: this.ruleSets,
|
|
117
132
|
variations: this.context.variations,
|
|
133
|
+
...this.resolvedOptions(overrides),
|
|
118
134
|
};
|
|
119
135
|
return this.context.queryType === QueryType.prewarm
|
|
120
136
|
? buildPrewarmQuery(input)
|
package/oclif.manifest.json
CHANGED
|
@@ -654,6 +654,12 @@
|
|
|
654
654
|
"name": "headless",
|
|
655
655
|
"allowNo": false,
|
|
656
656
|
"type": "boolean"
|
|
657
|
+
},
|
|
658
|
+
"browserFetch": {
|
|
659
|
+
"description": "Fetch every origin document with a second headless Chrome instead of a direct request. Only needed when bot protection refuses the CLI outright — a refusal is otherwise detected and rerouted on its own.",
|
|
660
|
+
"name": "browserFetch",
|
|
661
|
+
"allowNo": false,
|
|
662
|
+
"type": "boolean"
|
|
657
663
|
}
|
|
658
664
|
},
|
|
659
665
|
"hasDynamicHelp": false,
|
|
@@ -1645,5 +1651,5 @@
|
|
|
1645
1651
|
]
|
|
1646
1652
|
}
|
|
1647
1653
|
},
|
|
1648
|
-
"version": "4.
|
|
1654
|
+
"version": "4.29.0"
|
|
1649
1655
|
}
|