@speedkit/cli 4.28.0 → 4.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +5 -2
  3. package/dist/commands/onboarding.js +2 -2
  4. package/dist/helpers/evaluate-speed-kit-config.d.ts +5 -0
  5. package/dist/helpers/evaluate-speed-kit-config.js +14 -7
  6. package/dist/helpers/origin-proxy.d.ts +30 -0
  7. package/dist/helpers/origin-proxy.js +29 -0
  8. package/dist/services/browser-fetch/browser-origin-fetch.d.ts +43 -0
  9. package/dist/services/browser-fetch/browser-origin-fetch.js +63 -0
  10. package/dist/services/browser-fetch/browser-origin-fetch.spec.d.ts +1 -0
  11. package/dist/services/browser-fetch/browser-origin-fetch.spec.js +174 -0
  12. package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +216 -0
  13. package/dist/services/browser-fetch/browser-origin-fetcher.js +528 -0
  14. package/dist/services/browser-fetch/browser-origin-fetcher.spec.d.ts +1 -0
  15. package/dist/services/browser-fetch/browser-origin-fetcher.spec.js +27 -0
  16. package/dist/services/browser-fetch/browser-response.d.ts +21 -0
  17. package/dist/services/browser-fetch/browser-response.js +86 -0
  18. package/dist/services/browser-fetch/browser-response.spec.d.ts +1 -0
  19. package/dist/services/browser-fetch/browser-response.spec.js +90 -0
  20. package/dist/services/browser-fetch/device-emulation.d.ts +46 -0
  21. package/dist/services/browser-fetch/device-emulation.js +187 -0
  22. package/dist/services/browser-fetch/device-emulation.spec.d.ts +1 -0
  23. package/dist/services/browser-fetch/device-emulation.spec.js +73 -0
  24. package/dist/services/browser-fetch/index.d.ts +4 -0
  25. package/dist/services/browser-fetch/index.js +4 -0
  26. package/dist/services/customer-config/customer-config-service-model.d.ts +6 -0
  27. package/dist/services/customer-config/customer-config-service-model.js +1 -0
  28. package/dist/services/customer-config/customer-config-service.d.ts +5 -0
  29. package/dist/services/customer-config/customer-config-service.js +15 -0
  30. package/dist/services/deploy/checks/activation/customer-approval-check.d.ts +21 -0
  31. package/dist/services/deploy/checks/activation/customer-approval-check.js +59 -0
  32. package/dist/services/deploy/checks/activation/customer-approval-check.spec.d.ts +1 -0
  33. package/dist/services/deploy/checks/activation/customer-approval-check.spec.js +64 -0
  34. package/dist/services/deploy/deploy-service-factory.js +2 -1
  35. package/dist/services/deploy/handler/install-resource-handler.d.ts +7 -1
  36. package/dist/services/deploy/handler/install-resource-handler.js +40 -19
  37. package/dist/services/deploy/handler/testing-installation-links.d.ts +10 -0
  38. package/dist/services/deploy/handler/testing-installation-links.js +27 -0
  39. package/dist/services/deploy/handler/testing-installation-links.spec.d.ts +1 -0
  40. package/dist/services/deploy/handler/testing-installation-links.spec.js +43 -0
  41. package/dist/services/integration-api/struct/customer-config.d.ts +7 -1
  42. package/dist/services/integration-api/struct/customer-config.js +10 -3
  43. package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
  44. package/dist/services/onboarding/browser/executable/headed-user-agent.js +10 -1
  45. package/dist/services/onboarding/onboarding-model.d.ts +3 -1
  46. package/dist/services/onboarding/onboarding-model.js +8 -2
  47. package/dist/services/onboarding/onboarding-service-factory.js +12 -1
  48. package/dist/services/onboarding/onboarding-service.d.ts +3 -1
  49. package/dist/services/onboarding/onboarding-service.js +7 -1
  50. package/dist/services/onboarding/virtual-orestes-app/index.d.ts +13 -0
  51. package/dist/services/onboarding/virtual-orestes-app/index.js +39 -1
  52. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.d.ts +60 -0
  53. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.js +79 -0
  54. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.d.ts +1 -0
  55. package/dist/services/onboarding/virtual-orestes-app/origin-error-cause.spec.js +82 -0
  56. package/dist/services/origin-request/asset-variations.d.ts +2 -0
  57. package/dist/services/origin-request/asset-variations.js +2 -2
  58. package/oclif.manifest.json +118 -112
  59. package/package.json +1 -1
@@ -5,6 +5,7 @@ import InstallResourceBuilder from "../builder/install-resource-builder.js";
5
5
  import DeletedFileCheck from "./install-resource-modifier/deleted-file-check.js";
6
6
  import { CustomerConfig, FILE_TYPE, InstallResourceInterface } from "../../integration-api/index.js";
7
7
  import { ConfigApiService } from "../../config-api/index.js";
8
+ import { CustomerApprovalCheck } from "../checks/activation/customer-approval-check.js";
8
9
  export default class InstallResourceHandler implements DeployHandlerInterface {
9
10
  private readonly customer;
10
11
  private readonly configApi;
@@ -13,10 +14,15 @@ export default class InstallResourceHandler implements DeployHandlerInterface {
13
14
  private cli;
14
15
  private builder;
15
16
  private deletedFilesChecker;
16
- constructor(customer: CustomerConfig, configApi: ConfigApiService, configName: string, diffService: DiffService, cli: CliService, builder: InstallResourceBuilder, deletedFilesChecker: DeletedFileCheck);
17
+ private customerApprovalCheck;
18
+ constructor(customer: CustomerConfig, configApi: ConfigApiService, configName: string, diffService: DiffService, cli: CliService, builder: InstallResourceBuilder, deletedFilesChecker: DeletedFileCheck, customerApprovalCheck: CustomerApprovalCheck);
17
19
  canHandle(type: FILE_TYPE): boolean;
18
20
  handle(files: InstallResourceInterface[]): Promise<void>;
19
21
  upload(files: InstallResourceInterface[]): Promise<void>;
22
+ private activate;
23
+ private activateAsTesting;
24
+ /** The origins with the debug parameter attached, ready to be clicked. */
25
+ private writeTestingInstallationLinks;
20
26
  private processFile;
21
27
  private confirm;
22
28
  private getInstallResource;
@@ -1,6 +1,6 @@
1
1
  import { FILE_TYPE, } from "../../integration-api/index.js";
2
- import { copyToClipboard } from "../../../helpers/clipboard.js";
3
2
  import { safe } from "../../../helpers/safe.js";
3
+ import { buildTestingInstallationLinks } from "./testing-installation-links.js";
4
4
  export default class InstallResourceHandler {
5
5
  customer;
6
6
  configApi;
@@ -9,7 +9,8 @@ export default class InstallResourceHandler {
9
9
  cli;
10
10
  builder;
11
11
  deletedFilesChecker;
12
- constructor(customer, configApi, configName, diffService, cli, builder, deletedFilesChecker) {
12
+ customerApprovalCheck;
13
+ constructor(customer, configApi, configName, diffService, cli, builder, deletedFilesChecker, customerApprovalCheck) {
13
14
  this.customer = customer;
14
15
  this.configApi = configApi;
15
16
  this.diffService = diffService;
@@ -17,6 +18,7 @@ export default class InstallResourceHandler {
17
18
  this.cli = cli;
18
19
  this.builder = builder;
19
20
  this.deletedFilesChecker = deletedFilesChecker;
21
+ this.customerApprovalCheck = customerApprovalCheck;
20
22
  }
21
23
  canHandle(type) {
22
24
  return type === FILE_TYPE.INSTALL_FILE;
@@ -57,25 +59,16 @@ export default class InstallResourceHandler {
57
59
  throw updateResult.errorObj;
58
60
  }
59
61
  this.cli.successAction("DEPLOY:UPDATE:INSTALL_RESOURCE");
60
- if (!(await this.cli.confirm("Activate new installation?"))) {
61
- if (await this.cli.confirm("Activate as testing installation instead?")) {
62
- this.cli.startAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST", "activate testing installResource");
63
- const activateTestingResult = await safe(this.configApi.activateInstallTesting(install));
64
- if (activateTestingResult.success === false) {
65
- this.cli.failAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST", activateTestingResult.error);
66
- throw activateTestingResult.errorObj;
67
- }
68
- this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST");
69
- const debugParameter = `?debugSpeedKit=${install.id.split("/").pop()}`;
70
- if (await copyToClipboard(debugParameter)) {
71
- this.cli.writeSuccess("Query parameter to activate the testing installation has been copied to the clipboard 📋");
72
- }
73
- else {
74
- this.cli.writeSuccess(`Use the following parameter to active the testing installation in your browser: ${debugParameter}`);
75
- }
76
- }
62
+ // Without the customer's approval the installation may only go live as a
63
+ // testing installation, so the regular activation is not even offered.
64
+ const approved = await this.customerApprovalCheck.isActivationApproved(install);
65
+ if (approved && (await this.cli.confirm("Activate new installation?"))) {
66
+ await this.activate(install);
77
67
  return;
78
68
  }
69
+ await this.activateAsTesting(install, approved);
70
+ }
71
+ async activate(install) {
79
72
  this.cli.startAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE", "activate installResource");
80
73
  const activateResult = await safe(this.configApi.activateInstall(install));
81
74
  if (activateResult.success === false) {
@@ -84,6 +77,34 @@ export default class InstallResourceHandler {
84
77
  }
85
78
  this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE");
86
79
  }
80
+ async activateAsTesting(install, wasActivationOffered) {
81
+ const question = wasActivationOffered
82
+ ? "Activate as testing installation instead?"
83
+ : "Activate as testing installation?";
84
+ if (!(await this.cli.confirm(question))) {
85
+ return;
86
+ }
87
+ this.cli.startAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST", "activate testing installResource");
88
+ const activateTestingResult = await safe(this.configApi.activateInstallTesting(install));
89
+ if (activateTestingResult.success === false) {
90
+ this.cli.failAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST", activateTestingResult.error);
91
+ throw activateTestingResult.errorObj;
92
+ }
93
+ this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST");
94
+ this.writeTestingInstallationLinks(install);
95
+ }
96
+ /** The origins with the debug parameter attached, ready to be clicked. */
97
+ writeTestingInstallationLinks(install) {
98
+ const { links, hidden } = buildTestingInstallationLinks(this.customer.origins, install.id.split("/").pop());
99
+ this.cli.write("");
100
+ this.cli.write("Open the testing installation:");
101
+ for (const link of links) {
102
+ this.cli.write(` ${link}`);
103
+ }
104
+ if (hidden > 0) {
105
+ this.cli.comment(` … and ${hidden} more origin${hidden === 1 ? "" : "s"}`);
106
+ }
107
+ }
87
108
  async processFile(file, installResource) {
88
109
  if (!installResource) {
89
110
  await this.confirm(file, true);
@@ -0,0 +1,10 @@
1
+ /** How many origins are listed before the rest is summarised as a count. */
2
+ export declare const MAX_LISTED_ORIGINS = 10;
3
+ /**
4
+ * The links that open a testing installation — one per origin, capped so a
5
+ * customer with dozens of origins does not bury the rest of the deploy output.
6
+ */
7
+ export declare function buildTestingInstallationLinks(origins: string[], testInstallationId: string): {
8
+ links: string[];
9
+ hidden: number;
10
+ };
@@ -0,0 +1,27 @@
1
+ import { safe } from "../../../helpers/safe.js";
2
+ /** How many origins are listed before the rest is summarised as a count. */
3
+ export const MAX_LISTED_ORIGINS = 10;
4
+ /** The query parameter that switches a browser to a testing installation. */
5
+ const DEBUG_PARAMETER = "debugSpeedKit";
6
+ /**
7
+ * The links that open a testing installation — one per origin, capped so a
8
+ * customer with dozens of origins does not bury the rest of the deploy output.
9
+ */
10
+ export function buildTestingInstallationLinks(origins, testInstallationId) {
11
+ const links = origins
12
+ .slice(0, MAX_LISTED_ORIGINS)
13
+ .map((origin) => buildLink(origin, testInstallationId));
14
+ return { links, hidden: Math.max(origins.length - MAX_LISTED_ORIGINS, 0) };
15
+ }
16
+ function buildLink(origin, testInstallationId) {
17
+ const parsed = safe(() => {
18
+ const url = new URL(origin);
19
+ url.searchParams.set(DEBUG_PARAMETER, testInstallationId);
20
+ return url.toString();
21
+ });
22
+ // An origin the URL parser rejects is still worth printing — the deploy has
23
+ // happened by now, and a slightly off link beats no link at all.
24
+ return parsed.success
25
+ ? parsed.data
26
+ : `${origin}?${DEBUG_PARAMETER}=${testInstallationId}`;
27
+ }
@@ -0,0 +1,43 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import { buildTestingInstallationLinks, MAX_LISTED_ORIGINS, } from "./testing-installation-links.js";
4
+ function origins(count) {
5
+ return Array.from({ length: count }, (_, i) => `https://shop-${i}.example.com`);
6
+ }
7
+ describe("buildTestingInstallationLinks", () => {
8
+ it("attaches the debug parameter to every origin", () => {
9
+ const { links, hidden } = buildTestingInstallationLinks(["https://www.example.com", "https://www.example.de"], "hJ3kP9qw");
10
+ expect(links).to.deep.equal([
11
+ "https://www.example.com/?debugSpeedKit=hJ3kP9qw",
12
+ "https://www.example.de/?debugSpeedKit=hJ3kP9qw",
13
+ ]);
14
+ expect(hidden).to.equal(0);
15
+ });
16
+ it("keeps a port and an existing query on the origin", () => {
17
+ const { links } = buildTestingInstallationLinks(["https://staging.example.com:8443?preview=1"], "id");
18
+ expect(links).to.deep.equal([
19
+ "https://staging.example.com:8443/?preview=1&debugSpeedKit=id",
20
+ ]);
21
+ });
22
+ it("still builds a link for an origin the URL parser rejects", () => {
23
+ const { links } = buildTestingInstallationLinks(["www.example.com"], "id");
24
+ expect(links).to.deep.equal(["www.example.com?debugSpeedKit=id"]);
25
+ });
26
+ it("lists everything while the cap is not reached", () => {
27
+ const { links, hidden } = buildTestingInstallationLinks(origins(MAX_LISTED_ORIGINS), "id");
28
+ expect(links).to.have.lengthOf(MAX_LISTED_ORIGINS);
29
+ expect(hidden).to.equal(0);
30
+ });
31
+ it("caps the list and reports how many origins it left out", () => {
32
+ const { links, hidden } = buildTestingInstallationLinks(origins(MAX_LISTED_ORIGINS + 3), "id");
33
+ expect(links).to.have.lengthOf(MAX_LISTED_ORIGINS);
34
+ expect(links[0]).to.equal("https://shop-0.example.com/?debugSpeedKit=id");
35
+ expect(hidden).to.equal(3);
36
+ });
37
+ it("handles a config without origins", () => {
38
+ expect(buildTestingInstallationLinks([], "id")).to.deep.equal({
39
+ links: [],
40
+ hidden: 0,
41
+ });
42
+ });
43
+ });
@@ -15,6 +15,12 @@ export declare class CustomerConfig {
15
15
  readonly forceInstall?: boolean;
16
16
  readonly chromeFlags?: string[];
17
17
  readonly dependencies?: Record<string, string>;
18
- constructor(app: string, origins: string[], domain: string, swPath: string, scope: string, appDomain?: string, installPath?: string, installParameters?: string, name?: string, forceInstall?: boolean, chromeFlags?: string[], dependencies?: Record<string, string>);
18
+ /**
19
+ * The customer has to sign off every rollout to real users. With this set,
20
+ * `sk deploy` asks for that approval before it offers to activate an
21
+ * installation whose split is greater than 0 for any origin.
22
+ */
23
+ readonly needsCustomerApproval?: boolean;
24
+ constructor(app: string, origins: string[], domain: string, swPath: string, scope: string, appDomain?: string, installPath?: string, installParameters?: string, name?: string, forceInstall?: boolean, chromeFlags?: string[], dependencies?: Record<string, string>, needsCustomerApproval?: boolean);
19
25
  static createFrom(configFile: Partial<CustomerConfig>, configName?: string): CustomerConfig;
20
26
  }
@@ -12,7 +12,13 @@ export class CustomerConfig {
12
12
  forceInstall;
13
13
  chromeFlags;
14
14
  dependencies;
15
- constructor(app, origins, domain, swPath, scope, appDomain, installPath, installParameters, name, forceInstall, chromeFlags, dependencies) {
15
+ /**
16
+ * The customer has to sign off every rollout to real users. With this set,
17
+ * `sk deploy` asks for that approval before it offers to activate an
18
+ * installation whose split is greater than 0 for any origin.
19
+ */
20
+ needsCustomerApproval;
21
+ constructor(app, origins, domain, swPath, scope, appDomain, installPath, installParameters, name, forceInstall, chromeFlags, dependencies, needsCustomerApproval) {
16
22
  this.app = app;
17
23
  this.origins = origins;
18
24
  this.domain = domain;
@@ -25,11 +31,12 @@ export class CustomerConfig {
25
31
  this.forceInstall = forceInstall;
26
32
  this.chromeFlags = chromeFlags;
27
33
  this.dependencies = dependencies;
34
+ this.needsCustomerApproval = needsCustomerApproval;
28
35
  }
29
36
  static createFrom(configFile, configName) {
30
- const { app, origins, domain, swPath, scope, appDomain, installPath, installParams, name, forceInstall, chromeFlags, dependencies, } = configFile;
37
+ const { app, origins, domain, swPath, scope, appDomain, installPath, installParams, name, forceInstall, chromeFlags, dependencies, needsCustomerApproval, } = configFile;
31
38
  return new CustomerConfig(new RequiredParameter("app", app).isString().getValue(), new RequiredParameter("origins", origins).isArray().getValue(), domain, new RequiredParameter("swPath", swPath).isString().getValue(), new RequiredParameter("scope", scope).isString().getValue(), appDomain || `${app}.app.baqend.com`, installPath ||
32
- `https://${app}.app.baqend.com/v1/speedkit/install.js?d=${name || configName}`, installParams || 'async="" crossorigin="anonymous"', name, forceInstall || false, chromeFlags, dependencies || {});
39
+ `https://${app}.app.baqend.com/v1/speedkit/install.js?d=${name || configName}`, installParams || 'async="" crossorigin="anonymous"', name, forceInstall || false, chromeFlags, dependencies || {}, needsCustomerApproval || false);
33
40
  }
34
41
  }
35
42
  class RequiredParameter {
@@ -5,3 +5,10 @@
5
5
  * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
6
6
  */
7
7
  export declare function buildHeadedUserAgent(browserVersionString: string, platform?: string): string | undefined;
8
+ /**
9
+ * The Chrome major version of a `chrome --version` string, which is the only part of the reduced
10
+ * user agent that still moves.
11
+ *
12
+ * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
13
+ */
14
+ export declare function chromeMajorVersion(browserVersionString: string): string | undefined;
@@ -20,10 +20,19 @@ const DEFAULT_UNIFIED_PLATFORM = "X11; Linux x86_64";
20
20
  * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
21
21
  */
22
22
  export function buildHeadedUserAgent(browserVersionString, platform = process.platform) {
23
- const majorVersion = browserVersionString.match(/(\d+)\.\d+\.\d+\.\d+/)?.[1];
23
+ const majorVersion = chromeMajorVersion(browserVersionString);
24
24
  if (!majorVersion) {
25
25
  return undefined;
26
26
  }
27
27
  const unifiedPlatform = UNIFIED_PLATFORMS[platform] ?? DEFAULT_UNIFIED_PLATFORM;
28
28
  return `Mozilla/5.0 (${unifiedPlatform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${majorVersion}.0.0.0 Safari/537.36`;
29
29
  }
30
+ /**
31
+ * The Chrome major version of a `chrome --version` string, which is the only part of the reduced
32
+ * user agent that still moves.
33
+ *
34
+ * @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
35
+ */
36
+ export function chromeMajorVersion(browserVersionString) {
37
+ return browserVersionString.match(/(\d+)\.\d+\.\d+\.\d+/)?.[1];
38
+ }
@@ -5,6 +5,7 @@ import { AbortResponse } from "./browser/abort-response.js";
5
5
  import { task } from "./todo-collector.js";
6
6
  import { VanillaPuppeteer } from "puppeteer-extra";
7
7
  import { Command } from "@oclif/core/command";
8
+ export declare const HEADLESS_PARITY_ARGS: string[];
8
9
  declare const HEADLESS_DEFAULT_ARGS_TO_DROP: string[];
9
10
  export { HEADLESS_DEFAULT_ARGS_TO_DROP };
10
11
  export declare const BROWSER_EVENTS: {
@@ -72,12 +73,13 @@ export declare class OnboardingContext {
72
73
  readonly debuggingPort: boolean;
73
74
  readonly headless: boolean;
74
75
  readonly debugPort: number;
76
+ readonly browserFetch: boolean;
75
77
  static flags: Record<string, Command.Flag>;
76
78
  readonly DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
77
79
  readonly DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
78
80
  readonly DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
79
81
  readonly DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
80
- constructor(customerPath: string, configName: string, domain: string, local: boolean, ignoreContentSecurityPolicy: boolean, cliPath: string, verboseLevel?: boolean, useTestConfig?: boolean, debuggingPort?: boolean, headless?: boolean, debugPort?: number);
82
+ constructor(customerPath: string, configName: string, domain: string, local: boolean, ignoreContentSecurityPolicy: boolean, cliPath: string, verboseLevel?: boolean, useTestConfig?: boolean, debuggingPort?: boolean, headless?: boolean, debugPort?: number, browserFetch?: boolean);
81
83
  }
82
84
  export interface SpeedKitServerConfigAuthentication {
83
85
  password: string;
@@ -16,7 +16,7 @@ const DEFAULT_BROWSER_ARGS = [
16
16
  // Headless Chrome differs from headed in ways customer bot detection and responsive layout both
17
17
  // key on, so a page can render differently — or not at all — depending on the mode. Each entry
18
18
  // below erases one measured difference; see HEADLESS_DEFAULT_ARGS_TO_DROP for the counterpart.
19
- const HEADLESS_PARITY_ARGS = [
19
+ export const HEADLESS_PARITY_ARGS = [
20
20
  // Headless exposes no pointing device, so `hover: none` / `any-pointer: coarse` send sites down
21
21
  // their touch code path.
22
22
  "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
@@ -170,6 +170,7 @@ export class OnboardingContext {
170
170
  debuggingPort;
171
171
  headless;
172
172
  debugPort;
173
+ browserFetch;
173
174
  static flags = {
174
175
  ...CLI_CONFIG_NAME,
175
176
  ...CLI_CONFIG_IS_TEST,
@@ -205,12 +206,16 @@ export class OnboardingContext {
205
206
  default: false,
206
207
  description: "Run Chrome in headless mode (for CI environments). Implicitly enables remote debugging port.",
207
208
  }),
209
+ browserFetch: Flags.boolean({
210
+ default: false,
211
+ description: "Fetch every origin document with a second headless Chrome instead of a direct request. Only needed when bot protection refuses the CLI outright — a refusal is otherwise detected and rerouted on its own.",
212
+ }),
208
213
  };
209
214
  DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
210
215
  DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
211
216
  DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
212
217
  DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
213
- constructor(customerPath, configName, domain, local, ignoreContentSecurityPolicy = false, cliPath, verboseLevel, useTestConfig = false, debuggingPort = false, headless = false, debugPort = 9222) {
218
+ constructor(customerPath, configName, domain, local, ignoreContentSecurityPolicy = false, cliPath, verboseLevel, useTestConfig = false, debuggingPort = false, headless = false, debugPort = 9222, browserFetch = false) {
214
219
  this.customerPath = customerPath;
215
220
  this.configName = configName;
216
221
  this.domain = domain;
@@ -222,6 +227,7 @@ export class OnboardingContext {
222
227
  this.debuggingPort = debuggingPort;
223
228
  this.headless = headless;
224
229
  this.debugPort = debugPort;
230
+ this.browserFetch = browserFetch;
225
231
  }
226
232
  }
227
233
  export const USER_AGENT = {
@@ -29,6 +29,7 @@ import { ExtensionValidator } from "./browser/extension/extension-validator.js";
29
29
  import { DevtoolsExtensionApi } from "./browser/extension/devtools-extension-api.js";
30
30
  import { ExtensionBridge } from "./browser/extension/extension-bridge.js";
31
31
  import { TodoCollector } from "./todo-collector.js";
32
+ import { installBrowserOriginFetch } from "../browser-fetch/index.js";
32
33
  import { Crawler } from "./virtual-orestes-app/crawler.js";
33
34
  import { VirtualOrestesApp } from "./virtual-orestes-app/index.js";
34
35
  import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
@@ -55,6 +56,16 @@ export class OnboardingServiceFactory {
55
56
  const customerConfig = files.getCustomerConfig().config;
56
57
  // Before any origin request, so Chrome and the CLI agree on the route out.
57
58
  applyOriginProxy(customerConfig.chromeFlags, cli);
59
+ // Also before any origin request: an origin behind bot protection refuses the CLI outright,
60
+ // and `auto` only reaches for the browser once one actually has.
61
+ const browserFetch = installBrowserOriginFetch({
62
+ userConfig: this.cliConfig,
63
+ logger: cli,
64
+ mode: this.context.browserFetch ? "always" : "auto",
65
+ });
66
+ if (this.context.browserFetch) {
67
+ cli.writeWarning("Fetching every origin document through a second headless Chrome.");
68
+ }
58
69
  const domainToStart = await this.getUrlForTest(customerConfig, cli);
59
70
  const cache = new Cache(cli);
60
71
  const documentHandlerContext = {
@@ -87,7 +98,7 @@ export class OnboardingServiceFactory {
87
98
  const speedKitServiceWorkerJs = new SpeedKitServiceWorkerJs(customerConfig, files.getByName(INTEGRATION_FILES.CUSTOM.SW));
88
99
  const fetchHandler = await this.getFetchEventHandler(files, customerConfig, documentHandler, cache, cli, browserContext, athenaClient, messageHandler, fileWatcher, speedKitServiceWorkerJs);
89
100
  const extensionBridge = new ExtensionBridge(cli);
90
- return new OnboardingService(browserContext, customerConfig, fetchHandler, fileWatcher, cli, messageHandler, extensionBridge, speedKitServiceWorkerJs);
101
+ return new OnboardingService(browserContext, customerConfig, fetchHandler, fileWatcher, cli, messageHandler, extensionBridge, speedKitServiceWorkerJs, browserFetch);
91
102
  }
92
103
  async prepareMessagehandler(athenaClient, files, customerConfig) {
93
104
  const messageHandler = new DevtoolsExtensionApi();
@@ -6,6 +6,7 @@ import { FileWatcher } from "./file-events/file-watcher.js";
6
6
  import { CliService } from "../cli/index.js";
7
7
  import { DevtoolsExtensionApi } from "./browser/extension/devtools-extension-api.js";
8
8
  import { ExtensionBridge } from "./browser/extension/extension-bridge.js";
9
+ import { BrowserOriginFetchHandle } from "../browser-fetch/index.js";
9
10
  /**
10
11
  * Represents the OnboardingService which is responsible for setting up the browser to test a config using Puppeteer.
11
12
  */
@@ -18,10 +19,11 @@ export declare class OnboardingService {
18
19
  private developmentToolsMessages;
19
20
  private extensionBridge;
20
21
  private speedKitServiceWorkerJs;
22
+ private browserFetch?;
21
23
  private reinstallOnNextNavigate;
22
24
  private lastForeignSwReinstall;
23
25
  private readonly activatedSwVersions;
24
- constructor(browserContext: BrowserContext, customerConfig: CustomerConfig, fetchEventHandler: FetchEventHandler, fileWatcher: FileWatcher, cli: CliService, developmentToolsMessages: DevtoolsExtensionApi, extensionBridge: ExtensionBridge, speedKitServiceWorkerJs: SpeedKitServiceWorkerJs);
26
+ constructor(browserContext: BrowserContext, customerConfig: CustomerConfig, fetchEventHandler: FetchEventHandler, fileWatcher: FileWatcher, cli: CliService, developmentToolsMessages: DevtoolsExtensionApi, extensionBridge: ExtensionBridge, speedKitServiceWorkerJs: SpeedKitServiceWorkerJs, browserFetch?: BrowserOriginFetchHandle);
25
27
  /**
26
28
  * Prepare the puppeteer browser, wire event listeners, navigate to the
27
29
  * install page, then block until either the caller aborts the signal
@@ -18,10 +18,11 @@ export class OnboardingService {
18
18
  developmentToolsMessages;
19
19
  extensionBridge;
20
20
  speedKitServiceWorkerJs;
21
+ browserFetch;
21
22
  reinstallOnNextNavigate = false;
22
23
  lastForeignSwReinstall = 0;
23
24
  activatedSwVersions = new Set();
24
- constructor(browserContext, customerConfig, fetchEventHandler, fileWatcher, cli, developmentToolsMessages, extensionBridge, speedKitServiceWorkerJs) {
25
+ constructor(browserContext, customerConfig, fetchEventHandler, fileWatcher, cli, developmentToolsMessages, extensionBridge, speedKitServiceWorkerJs, browserFetch) {
25
26
  this.browserContext = browserContext;
26
27
  this.customerConfig = customerConfig;
27
28
  this.fetchEventHandler = fetchEventHandler;
@@ -30,6 +31,7 @@ export class OnboardingService {
30
31
  this.developmentToolsMessages = developmentToolsMessages;
31
32
  this.extensionBridge = extensionBridge;
32
33
  this.speedKitServiceWorkerJs = speedKitServiceWorkerJs;
34
+ this.browserFetch = browserFetch;
33
35
  }
34
36
  /**
35
37
  * Prepare the puppeteer browser, wire event listeners, navigate to the
@@ -51,6 +53,10 @@ export class OnboardingService {
51
53
  await this.navigate(page, `${this.customerConfig.scope}speed-kit-install.html?url=${this.customerConfig.scope}`);
52
54
  await waitForAbort(shutdownSignal);
53
55
  await safe(browser.close());
56
+ // The fetch browser only exists if an origin refused a direct request, but closing an unused
57
+ // handle is free and leaving a Chrome behind is not.
58
+ if (this.browserFetch)
59
+ await safe(this.browserFetch.close());
54
60
  }
55
61
  async prepareBrowser(localController) {
56
62
  const browser = await puppeteerExtra.default.launch(this.browserContext.getPuppeteerLaunchOptions());
@@ -60,6 +60,19 @@ export declare class VirtualOrestesApp {
60
60
  * the page as an infrastructure failure and disconnect Speed Kit.
61
61
  */
62
62
  private returnAbortResponse;
63
+ /**
64
+ * Answers an origin error the way the Asset API answers it, instead of caching it.
65
+ *
66
+ * A bot-protection page is served as an error status but still carries
67
+ * `content-type: text/html` and a body with `<html`, so it used to clear both content
68
+ * checks, run through the Document Handler and land in the dh-cache as the document. Every
69
+ * result after that was worthless, and nothing said why.
70
+ *
71
+ * The response is deliberately not cached. The service worker falls back to origin, which a
72
+ * real browser can reach even when the CLI cannot, so the page still renders while devtools
73
+ * names the cause.
74
+ */
75
+ private returnOriginErrorResponse;
63
76
  private returnErrorResponse;
64
77
  private convertResponseHeadersToOrestesFormat;
65
78
  private returnRedirect;
@@ -1,5 +1,6 @@
1
1
  import { BaqendResponse } from "../browser/baqend-response.js";
2
2
  import { ABORT_SERVER_TIMING, ABORT_STATUS_CODE, isDocumentHandlerAbort, } from "../../document-handler-runtime/document-handler-abort.js";
3
+ import { isBlockedOriginStatus, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "./origin-error-cause.js";
3
4
  import { safe } from "../../../helpers/safe.js";
4
5
  import { META_CONTENT_TYPE_REGEX, } from "../onboarding-model.js";
5
6
  import iconv from "iconv-lite";
@@ -118,6 +119,11 @@ export class VirtualOrestesApp {
118
119
  const response = fetchResult?.response;
119
120
  // URL we actually fetched (origin + variation query params); used for the redirect check below
120
121
  const fetchedUrl = fetchResult?.url;
122
+ // Reported before the content-type gate below: a blocked origin answers with whatever its
123
+ // protection layer serves, and the cause has to reach devtools either way.
124
+ if (response && response.status >= 400) {
125
+ return this.returnOriginErrorResponse(response, originUrl, requestUrl, variation);
126
+ }
121
127
  // todo check if we can handle more then htmlResponses
122
128
  // only handle html responses
123
129
  if (!response?.headers?.get("content-type")?.includes("html")) {
@@ -248,6 +254,36 @@ export class VirtualOrestesApp {
248
254
  customHeaders.push({ name: "x-error", value: message }, { name: "server-timing", value: ABORT_SERVER_TIMING });
249
255
  return new BaqendResponse(message, null, customHeaders, ABORT_STATUS_CODE);
250
256
  }
257
+ /**
258
+ * Answers an origin error the way the Asset API answers it, instead of caching it.
259
+ *
260
+ * A bot-protection page is served as an error status but still carries
261
+ * `content-type: text/html` and a body with `<html`, so it used to clear both content
262
+ * checks, run through the Document Handler and land in the dh-cache as the document. Every
263
+ * result after that was worthless, and nothing said why.
264
+ *
265
+ * The response is deliberately not cached. The service worker falls back to origin, which a
266
+ * real browser can reach even when the CLI cannot, so the page still renders while devtools
267
+ * names the cause.
268
+ */
269
+ returnOriginErrorResponse(response, originUrl, requestUrl, variation) {
270
+ const code = originErrorCode(response.status);
271
+ const cause = originErrorCauseName(code);
272
+ const message = `origin answered ${response.status} for ${originUrl} — not cached (${cause}, errorcode ${code})`;
273
+ this.cli.writeError(`[VirtualOrestes]: ${message}`);
274
+ if (isBlockedOriginStatus(response.status)) {
275
+ this.cli.writeWarning(`the origin blocked this request (server: ${response.headers.get("server") ?? "unknown"}). ` +
276
+ "Bot protection lets a real browser through but not a server-side fetch, so no cache " +
277
+ "entry can be built for this URL. Ask the customer to skip bot management for our egress IP.");
278
+ }
279
+ this.recordLocalCacheError(originUrl, requestUrl, variation, {
280
+ success: false,
281
+ error: message,
282
+ });
283
+ const customHeaders = this.prepareCustomHeaders(originUrl);
284
+ customHeaders.push({ name: "x-error", value: message }, { name: "server-timing", value: originErrorServerTiming(code) });
285
+ return new BaqendResponse(message, null, customHeaders, ORIGIN_ERROR_STATUS_CODE);
286
+ }
251
287
  returnErrorResponse(customResponse, customHeaders) {
252
288
  if (isDocumentHandlerAbort(customResponse.errorObj)) {
253
289
  return this.returnAbortResponse(customResponse, customHeaders);
@@ -286,7 +322,9 @@ export class VirtualOrestesApp {
286
322
  const serverTimingHeader = customHeaders.find((header) => {
287
323
  return header.name === "server-timing";
288
324
  });
289
- serverTimingHeader.value += ",error;desc=202";
325
+ // `errorcode` is the metric the service worker reads — an `error` metric was ignored, so
326
+ // the CDN redirect surfaced as a bare ClientError (`AssetAPIHandler.ts:200-205`).
327
+ serverTimingHeader.value += ",errorcode;desc=202";
290
328
  const redirectResponse = new BaqendResponse("", null, customHeaders, 400);
291
329
  this.cache.addEntry(requestUrl, redirectResponse);
292
330
  return redirectResponse;
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Response causes for an origin answer the local Asset API refuses to cache.
3
+ *
4
+ * `VirtualOrestesApp` stands in for Baqend's asset server, so an origin error has to surface
5
+ * the way production surfaces it: a client-error status plus an `errorcode` metric on
6
+ * `server-timing`. The service worker builds the cause as
7
+ * `` `${errorCause?.desc || ClientError}${errorCode ? `-${errorCode.desc}` : ''}` ``
8
+ * (`AssetAPIHandler.ts:200-205`), so `errorcode;desc=109` reports `ClientError-109` and the
9
+ * code resolves to `OriginBlockedStatus` further downstream.
10
+ *
11
+ * Only the `errorcode` metric is sent, which is what Baqend sends. An `errorCause` metric
12
+ * would invent a cause name the backend never emits — the same reasoning as
13
+ * `ABORT_SERVER_TIMING` in `document-handler-abort.ts`.
14
+ *
15
+ * Codes and names come from "Response Source & Cause Combinations", which is the source of
16
+ * truth for the mapping:
17
+ * https://www.notion.so/baqend/Response-Source-Cause-Combinations-3aab063103384229a11952b8c5ab1a20
18
+ */
19
+ /**
20
+ * Status the local Asset API answers an origin error with.
21
+ *
22
+ * It must stay below 500: the service worker treats 5xx from the Asset API as an
23
+ * infrastructure failure and disconnects Speed Kit, while a client error is classified from
24
+ * `server-timing` and falls back to origin (`AssetAPIHandler.ts:192-208`).
25
+ */
26
+ export declare const ORIGIN_ERROR_STATUS_CODE = 400;
27
+ /**
28
+ * Statuses Orestes counts as "the origin blocked us" rather than as an ordinary error.
29
+ * Straight from the `OriginBlockedStatus` definition: 403, 406, 429, 430.
30
+ */
31
+ export declare const BLOCKED_ORIGIN_STATUSES: Set<number>;
32
+ /** Speed Kit error codes for the origin answers this module classifies. */
33
+ export declare const ORIGIN_ERROR_CODE: {
34
+ /** 109 — the origin blocked the request (bot protection, WAF, rate limit). */
35
+ readonly BLOCKED: "109";
36
+ /** 104 — the origin answered with something Orestes considers a 410. */
37
+ readonly GONE: "104";
38
+ /** 105 — the origin answered with something Orestes considers a 404. */
39
+ readonly NOT_FOUND: "105";
40
+ /** 100 — catch-all for an origin error that fits none of the specific codes. */
41
+ readonly GENERIC: "100";
42
+ };
43
+ /** Cause name each code resolves to, for log output. The service worker only sends the code. */
44
+ export declare const ORIGIN_ERROR_CAUSE_NAME: Record<string, string>;
45
+ /** Whether the origin refused the request rather than failing to answer it. */
46
+ export declare function isBlockedOriginStatus(status: number): boolean;
47
+ /**
48
+ * Maps an origin status onto the Speed Kit error code that describes it.
49
+ *
50
+ * @param status - Status the origin answered with.
51
+ */
52
+ export declare function originErrorCode(status: number): string;
53
+ /** Cause name for an error code, or the code itself when the mapping has no name for it. */
54
+ export declare function originErrorCauseName(code: string): string;
55
+ /**
56
+ * `server-timing` value that makes the service worker classify the response as this cause.
57
+ *
58
+ * @param code - Speed Kit error code, from `originErrorCode`.
59
+ */
60
+ export declare function originErrorServerTiming(code: string): string;