@speedkit/cli 4.20.1 → 4.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +1 -1
- package/dist/services/onboarding/browser/executable/headed-user-agent.d.ts +7 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.js +29 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.spec.d.ts +1 -0
- package/dist/services/onboarding/browser/executable/headed-user-agent.spec.js +30 -0
- package/dist/services/onboarding/dashboard/diff-against-current-page.js +2 -2
- package/dist/services/onboarding/onboarding-model.d.ts +9 -1
- package/dist/services/onboarding/onboarding-model.js +46 -2
- package/dist/services/onboarding/onboarding-model.spec.js +71 -1
- package/dist/services/onboarding/virtual-orestes-app/crawler.d.ts +13 -1
- package/dist/services/onboarding/virtual-orestes-app/crawler.js +43 -20
- package/dist/services/onboarding/virtual-orestes-app/index.js +6 -3
- package/oclif.manifest.json +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
## [4.20.3](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.20.2...v4.20.3) (2026-08-07)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* **onboarding:** make headless launch match headed ([aa9af3e](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/aa9af3ea69602c82ef30ffa547b33a32092258e5))
|
|
7
|
+
|
|
8
|
+
## [4.20.2](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.20.1...v4.20.2) (2026-08-07)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* **onboarding:** partial variations local mode ([bf809b1](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/bf809b10b3f6ef8918675971ee950735c3250213))
|
|
14
|
+
|
|
1
15
|
## [4.20.1](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.20.0...v4.20.1) (2026-08-07)
|
|
2
16
|
|
|
3
17
|
|
package/README.md
CHANGED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The user agent headed Chrome sends for this build. Headless would otherwise say
|
|
3
|
+
* `HeadlessChrome/<version>`, which some origins treat as a crawler.
|
|
4
|
+
*
|
|
5
|
+
* @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
|
|
6
|
+
*/
|
|
7
|
+
export declare function buildHeadedUserAgent(browserVersionString: string, platform?: string): string | undefined;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chrome's reduced User-Agent, as documented at https://www.chromium.org/updates/ua-reduction/:
|
|
3
|
+
*
|
|
4
|
+
* Mozilla/5.0 (<unifiedPlatform>) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/<major>.0.0.0 Safari/537.36
|
|
5
|
+
*
|
|
6
|
+
* Only the major version moves — everything after it is frozen at 0.0.0, and the platform tokens are
|
|
7
|
+
* literal values that "will not update even if a user is on an updated operating system or device".
|
|
8
|
+
* So taking the major version from the installed binary keeps this current on its own.
|
|
9
|
+
*/
|
|
10
|
+
const UNIFIED_PLATFORMS = {
|
|
11
|
+
darwin: "Macintosh; Intel Mac OS X 10_15_7",
|
|
12
|
+
win32: "Windows NT 10.0; Win64; x64",
|
|
13
|
+
};
|
|
14
|
+
// Chrome reports X11 for every other *nix build it ships.
|
|
15
|
+
const DEFAULT_UNIFIED_PLATFORM = "X11; Linux x86_64";
|
|
16
|
+
/**
|
|
17
|
+
* The user agent headed Chrome sends for this build. Headless would otherwise say
|
|
18
|
+
* `HeadlessChrome/<version>`, which some origins treat as a crawler.
|
|
19
|
+
*
|
|
20
|
+
* @param browserVersionString `chrome --version` output, e.g. `Google Chrome for Testing 146.0.7680.165`
|
|
21
|
+
*/
|
|
22
|
+
export function buildHeadedUserAgent(browserVersionString, platform = process.platform) {
|
|
23
|
+
const majorVersion = browserVersionString.match(/(\d+)\.\d+\.\d+\.\d+/)?.[1];
|
|
24
|
+
if (!majorVersion) {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
const unifiedPlatform = UNIFIED_PLATFORMS[platform] ?? DEFAULT_UNIFIED_PLATFORM;
|
|
28
|
+
return `Mozilla/5.0 (${unifiedPlatform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${majorVersion}.0.0.0 Safari/537.36`;
|
|
29
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { describe, it } from "mocha";
|
|
3
|
+
import { buildHeadedUserAgent } from "./headed-user-agent.js";
|
|
4
|
+
const CHROME_VERSION = "Google Chrome for Testing 146.0.7680.165";
|
|
5
|
+
describe("buildHeadedUserAgent", () => {
|
|
6
|
+
it("builds the reduced user agent for Linux", () => {
|
|
7
|
+
expect(buildHeadedUserAgent(CHROME_VERSION, "linux")).to.equal("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36");
|
|
8
|
+
});
|
|
9
|
+
it("builds the reduced user agent for macOS", () => {
|
|
10
|
+
expect(buildHeadedUserAgent(CHROME_VERSION, "darwin")).to.equal("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36");
|
|
11
|
+
});
|
|
12
|
+
it("builds the reduced user agent for Windows", () => {
|
|
13
|
+
expect(buildHeadedUserAgent(CHROME_VERSION, "win32")).to.equal("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36");
|
|
14
|
+
});
|
|
15
|
+
it("falls back to the X11 platform token on other unix builds", () => {
|
|
16
|
+
expect(buildHeadedUserAgent(CHROME_VERSION, "freebsd")).to.include("(X11; Linux x86_64)");
|
|
17
|
+
});
|
|
18
|
+
it("freezes everything after the major version", () => {
|
|
19
|
+
expect(buildHeadedUserAgent("Google Chrome 131.0.6778.86", "linux")).to.include("Chrome/131.0.0.0 ");
|
|
20
|
+
});
|
|
21
|
+
it("reads the version from the Windows ProductName + FileVersion shape", () => {
|
|
22
|
+
expect(buildHeadedUserAgent("Google Chrome 146.0.7680.165", "win32")).to.include("Chrome/146.0.0.0 ");
|
|
23
|
+
});
|
|
24
|
+
it("never advertises the headless token", () => {
|
|
25
|
+
expect(buildHeadedUserAgent(CHROME_VERSION, "linux")).to.not.include("HeadlessChrome");
|
|
26
|
+
});
|
|
27
|
+
it("returns undefined when the version string has no version in it", () => {
|
|
28
|
+
expect(buildHeadedUserAgent("unknown", "linux")).to.equal(undefined);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -13,8 +13,8 @@ export class DiffAgainstCurrentPage {
|
|
|
13
13
|
async postDiff(currentHtml, currentUrl, variant = "default", transform = false) {
|
|
14
14
|
this.cli.writeWarning(`received diff for url: ${currentUrl}`);
|
|
15
15
|
this.cli.startAction(`DIFF:FETCH:FILE:${currentUrl}`, `fetch remote`);
|
|
16
|
-
const
|
|
17
|
-
let originHtml = await
|
|
16
|
+
const fetchResult = await this.crawler.fetchRemote(currentUrl, variant);
|
|
17
|
+
let originHtml = await fetchResult?.response?.text();
|
|
18
18
|
if (!originHtml) {
|
|
19
19
|
this.cli.failAction(`DIFF:FETCH:FILE:${currentUrl}`);
|
|
20
20
|
throw new Error("Could not fetch remote page");
|
|
@@ -5,6 +5,8 @@ import { AbortResponse } from "./browser/abort-response.js";
|
|
|
5
5
|
import { task } from "./todo-collector.js";
|
|
6
6
|
import { VanillaPuppeteer } from "puppeteer-extra";
|
|
7
7
|
import { Command } from "@oclif/core/command";
|
|
8
|
+
declare const HEADLESS_DEFAULT_ARGS_TO_DROP: string[];
|
|
9
|
+
export { HEADLESS_DEFAULT_ARGS_TO_DROP };
|
|
8
10
|
export declare const BROWSER_EVENTS: {
|
|
9
11
|
DISCONNECTED: string;
|
|
10
12
|
FETCH_AUTH_REQUIRED: string;
|
|
@@ -43,6 +45,12 @@ export declare class BrowserContext {
|
|
|
43
45
|
readonly browserArgs: string[];
|
|
44
46
|
readonly chromePath: string;
|
|
45
47
|
constructor(domain: string, browserVersionString?: string, chromeFlags?: string[], userConfig?: UserCliConfig, userAgent?: string, debuggingPort?: boolean, headless?: boolean, debugPort?: number);
|
|
48
|
+
/**
|
|
49
|
+
* Applied as a launch flag rather than over CDP on purpose: a per-page
|
|
50
|
+
* `Network.setUserAgentOverride` never reaches the Service Worker, so Speed Kit's dynamic-block
|
|
51
|
+
* refetch would still leak the headless token and re-poison the merged body.
|
|
52
|
+
*/
|
|
53
|
+
private getHeadlessParityArgs;
|
|
46
54
|
private isChromeBrowser;
|
|
47
55
|
private getBrowserExtensionPath;
|
|
48
56
|
getPuppeteerLaunchOptions(): Parameters<VanillaPuppeteer["launch"]>[0];
|
|
@@ -83,7 +91,7 @@ export type SpeedKitServerConfigVariation = Record<string, {
|
|
|
83
91
|
accept?: string;
|
|
84
92
|
"user-agent"?: string;
|
|
85
93
|
};
|
|
86
|
-
queryParams?: [];
|
|
94
|
+
queryParams?: string[];
|
|
87
95
|
regex?: null | string;
|
|
88
96
|
urlPrefix?: string;
|
|
89
97
|
}>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Flags } from "@oclif/core";
|
|
2
2
|
import { CLI_CONFIG_IS_TEST, CLI_CONFIG_NAME, CLIParameters, CLIParametersChar, } from "../../models/cli-parameters.js";
|
|
3
3
|
import { BrowserConfig } from "./browser/config/browser-config.js";
|
|
4
|
+
import { buildHeadedUserAgent } from "./browser/executable/headed-user-agent.js";
|
|
4
5
|
import { resolve } from "node:path";
|
|
5
6
|
const DEFAULT_BROWSER_ARGS = [
|
|
6
7
|
"--enable-features=Translate,NetworkService",
|
|
@@ -12,6 +13,25 @@ const DEFAULT_BROWSER_ARGS = [
|
|
|
12
13
|
// triggered by the flag above. Value is arbitrary and only used as an identifier.
|
|
13
14
|
"--test-type=speed-kit-cli",
|
|
14
15
|
];
|
|
16
|
+
// Headless Chrome differs from headed in ways customer bot detection and responsive layout both
|
|
17
|
+
// key on, so a page can render differently — or not at all — depending on the mode. Each entry
|
|
18
|
+
// below erases one measured difference; see HEADLESS_DEFAULT_ARGS_TO_DROP for the counterpart.
|
|
19
|
+
const HEADLESS_PARITY_ARGS = [
|
|
20
|
+
// Headless exposes no pointing device, so `hover: none` / `any-pointer: coarse` send sites down
|
|
21
|
+
// their touch code path.
|
|
22
|
+
"--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
|
|
23
|
+
// Headless otherwise pins the screen to 800x600, i.e. a tablet breakpoint. --screen-info is what
|
|
24
|
+
// overrides it; Chrome's own --ozone-override-screen-size is not reachable via ignoreDefaultArgs.
|
|
25
|
+
"--window-size=1920,1200",
|
|
26
|
+
"--screen-info={1920x1200}",
|
|
27
|
+
];
|
|
28
|
+
// Puppeteer-injected headless defaults with no headed equivalent. ignoreDefaultArgs only filters what
|
|
29
|
+
// Puppeteer itself adds, so a spec asserts every entry is still in its defaultArgs.
|
|
30
|
+
const HEADLESS_DEFAULT_ARGS_TO_DROP = [
|
|
31
|
+
// Scrollbar width 0 instead of 15 narrows the layout viewport and skews layout-shift work.
|
|
32
|
+
"--hide-scrollbars",
|
|
33
|
+
];
|
|
34
|
+
export { HEADLESS_DEFAULT_ARGS_TO_DROP };
|
|
15
35
|
export const BROWSER_EVENTS = {
|
|
16
36
|
DISCONNECTED: "disconnected",
|
|
17
37
|
FETCH_AUTH_REQUIRED: "Fetch.authRequired",
|
|
@@ -61,7 +81,8 @@ export class BrowserContext {
|
|
|
61
81
|
this.domain = domain;
|
|
62
82
|
this.chromeFlags = chromeFlags || [];
|
|
63
83
|
this.chromePath = resolve(userConfig.chromePath);
|
|
64
|
-
|
|
84
|
+
// Copy: pushing onto the shared module-level array leaked args into every later instance.
|
|
85
|
+
const browserArguments = [...DEFAULT_BROWSER_ARGS];
|
|
65
86
|
if (userConfig?.chromeUserProfilePath) {
|
|
66
87
|
browserArguments.push(`--user-data-dir=${userConfig?.chromeUserProfilePath}`);
|
|
67
88
|
}
|
|
@@ -70,12 +91,32 @@ export class BrowserContext {
|
|
|
70
91
|
const temporaryProfileDirectory = browserConfig.loadProfileTempDir();
|
|
71
92
|
browserArguments.push(`--user-data-dir=${temporaryProfileDirectory}`);
|
|
72
93
|
}
|
|
94
|
+
if (this.headless) {
|
|
95
|
+
this.userAgent =
|
|
96
|
+
this.userAgent ?? buildHeadedUserAgent(browserVersionString);
|
|
97
|
+
}
|
|
73
98
|
this.browserArgs = [
|
|
74
99
|
...browserArguments,
|
|
75
100
|
`--unsafely-treat-insecure-origin-as-secure=${domain}`,
|
|
101
|
+
...this.getHeadlessParityArgs(),
|
|
102
|
+
// Last, so a customer's chromeFlags override our defaults — Chrome takes the last occurrence.
|
|
76
103
|
...this.chromeFlags,
|
|
77
104
|
];
|
|
78
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* Applied as a launch flag rather than over CDP on purpose: a per-page
|
|
108
|
+
* `Network.setUserAgentOverride` never reaches the Service Worker, so Speed Kit's dynamic-block
|
|
109
|
+
* refetch would still leak the headless token and re-poison the merged body.
|
|
110
|
+
*/
|
|
111
|
+
getHeadlessParityArgs() {
|
|
112
|
+
if (!this.headless) {
|
|
113
|
+
return [];
|
|
114
|
+
}
|
|
115
|
+
return [
|
|
116
|
+
...HEADLESS_PARITY_ARGS,
|
|
117
|
+
...(this.userAgent ? [`--user-agent=${this.userAgent}`] : []),
|
|
118
|
+
];
|
|
119
|
+
}
|
|
79
120
|
isChromeBrowser() {
|
|
80
121
|
return !!/chrome/i.test(this.browserVersionString);
|
|
81
122
|
}
|
|
@@ -95,7 +136,10 @@ export class BrowserContext {
|
|
|
95
136
|
args: this.browserArgs,
|
|
96
137
|
// Strip Puppeteer's default `--enable-automation` so Chrome doesn't show
|
|
97
138
|
// the "Chrome is being controlled by automated test software" infobar.
|
|
98
|
-
ignoreDefaultArgs: [
|
|
139
|
+
ignoreDefaultArgs: [
|
|
140
|
+
"--enable-automation",
|
|
141
|
+
...(this.headless ? HEADLESS_DEFAULT_ARGS_TO_DROP : []),
|
|
142
|
+
],
|
|
99
143
|
};
|
|
100
144
|
if (this.userConfig?.chromeExtensionPaths) {
|
|
101
145
|
if (this.isChromeBrowser()) {
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { expect } from "chai";
|
|
2
2
|
import { describe, it } from "mocha";
|
|
3
|
-
import { BrowserContext } from "./onboarding-model.js";
|
|
3
|
+
import { BrowserContext, HEADLESS_DEFAULT_ARGS_TO_DROP, } from "./onboarding-model.js";
|
|
4
|
+
import { buildHeadedUserAgent } from "./browser/executable/headed-user-agent.js";
|
|
4
5
|
// Providing chromeUserProfilePath keeps BrowserContext off the filesystem
|
|
5
6
|
// (it skips the temp-profile branch in the constructor).
|
|
6
7
|
const userConfig = {
|
|
7
8
|
chromePath: "/usr/bin/google-chrome",
|
|
8
9
|
chromeUserProfilePath: "/tmp/sk-test-profile",
|
|
9
10
|
};
|
|
11
|
+
function newContext(headless, { chromeFlags = [], userAgent, browserVersionString = "Google Chrome for Testing 146.0.7680.165", } = {}) {
|
|
12
|
+
return new BrowserContext("example.com", browserVersionString, chromeFlags, userConfig, userAgent, false, headless, undefined);
|
|
13
|
+
}
|
|
14
|
+
function launchArgs(headless, overrides = {}) {
|
|
15
|
+
return newContext(headless, overrides).getPuppeteerLaunchOptions()
|
|
16
|
+
.args;
|
|
17
|
+
}
|
|
10
18
|
function remoteDebuggingArg(debuggingPort, headless, debugPort) {
|
|
11
19
|
const context = new BrowserContext("example.com", "Chrome/120", [], userConfig, undefined, debuggingPort, headless, debugPort);
|
|
12
20
|
const args = context.getPuppeteerLaunchOptions().args;
|
|
@@ -26,3 +34,65 @@ describe("BrowserContext.getPuppeteerLaunchOptions remote debugging port", () =>
|
|
|
26
34
|
expect(remoteDebuggingArg(false, false, 9223)).to.equal(undefined);
|
|
27
35
|
});
|
|
28
36
|
});
|
|
37
|
+
describe("BrowserContext headless/headed parity", () => {
|
|
38
|
+
const userAgentOf = (args) => args.find((arg) => arg.startsWith("--user-agent="));
|
|
39
|
+
it("derives the headed user agent from the installed Chrome version", () => {
|
|
40
|
+
expect(userAgentOf(launchArgs(true))).to.equal(`--user-agent=${buildHeadedUserAgent("Google Chrome for Testing 146.0.7680.165")}`);
|
|
41
|
+
});
|
|
42
|
+
it("never contains the HeadlessChrome token", () => {
|
|
43
|
+
expect(launchArgs(true).join(" ")).to.not.include("HeadlessChrome");
|
|
44
|
+
});
|
|
45
|
+
it("leaves the user agent to Chrome in headed mode", () => {
|
|
46
|
+
expect(userAgentOf(launchArgs(false))).to.equal(undefined);
|
|
47
|
+
});
|
|
48
|
+
it("prefers an explicitly provided user agent", () => {
|
|
49
|
+
expect(userAgentOf(launchArgs(true, { userAgent: "custom-agent" }))).to.equal("--user-agent=custom-agent");
|
|
50
|
+
});
|
|
51
|
+
it("omits the flag when the Chrome version cannot be read", () => {
|
|
52
|
+
expect(userAgentOf(launchArgs(true, { browserVersionString: "unknown" }))).to.equal(undefined);
|
|
53
|
+
});
|
|
54
|
+
it("restores hover and fine-pointer support", () => {
|
|
55
|
+
expect(launchArgs(true)).to.include("--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4");
|
|
56
|
+
});
|
|
57
|
+
it("overrides the 800x600 headless screen", () => {
|
|
58
|
+
expect(launchArgs(true)).to.include.members([
|
|
59
|
+
"--window-size=1920,1200",
|
|
60
|
+
"--screen-info={1920x1200}",
|
|
61
|
+
]);
|
|
62
|
+
});
|
|
63
|
+
it("applies no parity args in headed mode", () => {
|
|
64
|
+
const headedArgs = launchArgs(false);
|
|
65
|
+
expect(headedArgs).to.not.include("--window-size=1920,1200");
|
|
66
|
+
expect(headedArgs.some((arg) => arg.startsWith("--blink-settings="))).to.equal(false);
|
|
67
|
+
});
|
|
68
|
+
it("lets customer chromeFlags win over the parity defaults", () => {
|
|
69
|
+
const args = launchArgs(true, {
|
|
70
|
+
chromeFlags: ["--window-size=390,844", "--user-agent=mobile-agent"],
|
|
71
|
+
});
|
|
72
|
+
// Chrome takes the last occurrence of a switch, so the customer values must come last.
|
|
73
|
+
expect(args.lastIndexOf("--window-size=390,844")).to.be.greaterThan(args.lastIndexOf("--window-size=1920,1200"));
|
|
74
|
+
expect(args.lastIndexOf("--user-agent=mobile-agent")).to.be.greaterThan(args.findIndex((arg) => arg.startsWith("--user-agent=Mozilla/5.0")));
|
|
75
|
+
});
|
|
76
|
+
it("drops the Puppeteer headless defaults that have no headed equivalent", () => {
|
|
77
|
+
expect(newContext(true).getPuppeteerLaunchOptions().ignoreDefaultArgs).to.include.members(["--enable-automation", "--hide-scrollbars"]);
|
|
78
|
+
});
|
|
79
|
+
// ignoreDefaultArgs filters Puppeteer's own args by exact string. Anything Chrome adds itself is
|
|
80
|
+
// unreachable that way, and a Puppeteer bump that renames or drops an arg would silently turn an
|
|
81
|
+
// entry into a no-op — which is how --ozone-override-screen-size got in here in the first place.
|
|
82
|
+
it("only ignores args Puppeteer actually adds", async () => {
|
|
83
|
+
const { defaultArgs } = await import("puppeteer");
|
|
84
|
+
const puppeteerHeadlessArgs = defaultArgs({ headless: true });
|
|
85
|
+
expect(puppeteerHeadlessArgs).to.include.members([
|
|
86
|
+
...HEADLESS_DEFAULT_ARGS_TO_DROP,
|
|
87
|
+
"--enable-automation",
|
|
88
|
+
]);
|
|
89
|
+
});
|
|
90
|
+
it("keeps only --enable-automation ignored in headed mode", () => {
|
|
91
|
+
expect(newContext(false).getPuppeteerLaunchOptions().ignoreDefaultArgs).to.deep.equal(["--enable-automation"]);
|
|
92
|
+
});
|
|
93
|
+
it("does not leak args between instances", () => {
|
|
94
|
+
const first = launchArgs(true).length;
|
|
95
|
+
launchArgs(true);
|
|
96
|
+
expect(launchArgs(true).length).to.equal(first);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
@@ -6,7 +6,10 @@ export declare class Crawler {
|
|
|
6
6
|
private agentConfig;
|
|
7
7
|
private speedKitServerConfig?;
|
|
8
8
|
constructor(cli: CliService, agentConfig: Agent, speedKitServerConfig?: SpeedKitServerConfig);
|
|
9
|
-
fetchRemote(originUrl: string, variation: string): Promise<
|
|
9
|
+
fetchRemote(originUrl: string, variation: string): Promise<{
|
|
10
|
+
response: Response;
|
|
11
|
+
url: string;
|
|
12
|
+
} | undefined>;
|
|
10
13
|
private setDefaultUserAgent;
|
|
11
14
|
/**
|
|
12
15
|
* Mirrors how the Baqend asset server applies the `variations` block from
|
|
@@ -28,8 +31,17 @@ export declare class Crawler {
|
|
|
28
31
|
* cookie like `fmarktcookie=${storeId}` becomes
|
|
29
32
|
* `fmarktcookie=2879130`.
|
|
30
33
|
* 3. Origin-level basic auth fallback (only if no variation matched).
|
|
34
|
+
*
|
|
35
|
+
* Returns the (query-param-augmented) origin URL alongside the request init,
|
|
36
|
+
* so query-param variations like `Sortierung=${sort}` reach the origin.
|
|
31
37
|
*/
|
|
32
38
|
private prepareOriginRequest;
|
|
39
|
+
/**
|
|
40
|
+
* Applies a matched variation: merges its headers into `init` and appends its
|
|
41
|
+
* `queryParams` onto `url`, substituting regex named groups (`${name}`) — so
|
|
42
|
+
* `Sortierung=${sort}` with `{ sort: "3" }` becomes `Sortierung=3`.
|
|
43
|
+
*/
|
|
44
|
+
private applyVariation;
|
|
33
45
|
/**
|
|
34
46
|
* Replaces `${name}` placeholders in each header value with the matching
|
|
35
47
|
* named capture group from a regex match. Used by `prepareOriginRequest`
|
|
@@ -16,18 +16,21 @@ export class Crawler {
|
|
|
16
16
|
const headers = {
|
|
17
17
|
"User-Agent": userAgent,
|
|
18
18
|
};
|
|
19
|
+
// origin URL with the variation's query params applied — returned so callers compare it
|
|
20
|
+
// (not the bare originUrl) against response.url for redirect detection
|
|
21
|
+
const { url, init } = this.prepareOriginRequest(originUrl, variation, {
|
|
22
|
+
headers,
|
|
23
|
+
});
|
|
19
24
|
const requestInit = {
|
|
20
|
-
...
|
|
21
|
-
headers,
|
|
22
|
-
}),
|
|
25
|
+
...init,
|
|
23
26
|
agent: this.agentConfig,
|
|
24
27
|
};
|
|
25
28
|
const uuid = randomUUID();
|
|
26
|
-
this.cli.startAction(`DOCUMENTHANDLER:FETCH:${uuid}`, `[documentHandler:fetch]: ${
|
|
27
|
-
const fetchResponse = await safe(fetch(
|
|
29
|
+
this.cli.startAction(`DOCUMENTHANDLER:FETCH:${uuid}`, `[documentHandler:fetch]: ${url}`);
|
|
30
|
+
const fetchResponse = await safe(fetch(url, requestInit));
|
|
28
31
|
if (fetchResponse.success === true) {
|
|
29
32
|
this.cli.successAction(`DOCUMENTHANDLER:FETCH:${uuid}`);
|
|
30
|
-
return fetchResponse.data;
|
|
33
|
+
return { response: fetchResponse.data, url };
|
|
31
34
|
}
|
|
32
35
|
this.cli.failAction(`DOCUMENTHANDLER:FETCH:${uuid}`, `error while fetching Asset: ${originUrl}`);
|
|
33
36
|
this.cli.writeError(fetchResponse.error);
|
|
@@ -62,32 +65,35 @@ export class Crawler {
|
|
|
62
65
|
* cookie like `fmarktcookie=${storeId}` becomes
|
|
63
66
|
* `fmarktcookie=2879130`.
|
|
64
67
|
* 3. Origin-level basic auth fallback (only if no variation matched).
|
|
68
|
+
*
|
|
69
|
+
* Returns the (query-param-augmented) origin URL alongside the request init,
|
|
70
|
+
* so query-param variations like `Sortierung=${sort}` reach the origin.
|
|
65
71
|
*/
|
|
66
72
|
prepareOriginRequest(originUrl, variationParameter, init) {
|
|
73
|
+
const url = new URL(originUrl);
|
|
67
74
|
if (!this.speedKitServerConfig) {
|
|
68
|
-
return init;
|
|
75
|
+
return { url: url.toString(), init };
|
|
69
76
|
}
|
|
70
77
|
const variations = this.speedKitServerConfig.speedKit.variations;
|
|
71
|
-
// Wildcard variation
|
|
78
|
+
// Wildcard variation applies to every request, regardless of which
|
|
72
79
|
// specific variation (if any) is requested.
|
|
73
80
|
if (variations?.["*"]) {
|
|
74
|
-
|
|
75
|
-
init.headers = { ...init.headers, ...config.headers };
|
|
81
|
+
this.applyVariation(url, init, variations["*"], undefined);
|
|
76
82
|
}
|
|
77
83
|
if (!variationParameter) {
|
|
78
|
-
return init;
|
|
84
|
+
return { url: url.toString(), init };
|
|
79
85
|
}
|
|
80
86
|
// First-match-wins over the variations in declaration order. Each entry
|
|
81
87
|
// can match by exact key or by its Java-style `regex` field; whichever
|
|
82
|
-
// entry matches first applies its headers and short-circuits
|
|
88
|
+
// entry matches first applies its headers + query params and short-circuits.
|
|
83
89
|
for (const variationKey in variations) {
|
|
84
90
|
if (variationKey === "*") {
|
|
85
91
|
continue;
|
|
86
92
|
}
|
|
87
93
|
const variationConfig = variations[variationKey];
|
|
88
94
|
if (variationParameter === variationKey) {
|
|
89
|
-
|
|
90
|
-
return init;
|
|
95
|
+
this.applyVariation(url, init, variationConfig, undefined);
|
|
96
|
+
return { url: url.toString(), init };
|
|
91
97
|
}
|
|
92
98
|
if (!variationConfig.regex) {
|
|
93
99
|
continue;
|
|
@@ -102,18 +108,35 @@ export class Crawler {
|
|
|
102
108
|
if (!match) {
|
|
103
109
|
continue;
|
|
104
110
|
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
...this.substituteNamedGroups(variationConfig.headers, match.groups),
|
|
108
|
-
};
|
|
109
|
-
return init;
|
|
111
|
+
this.applyVariation(url, init, variationConfig, match.groups);
|
|
112
|
+
return { url: url.toString(), init };
|
|
110
113
|
}
|
|
111
114
|
// No variation matched: fall back to origin basic auth if configured.
|
|
112
115
|
const authentication = this.prepareAuthRequest(originUrl);
|
|
113
116
|
if (authentication) {
|
|
114
117
|
init.headers["Authorization"] = authentication;
|
|
115
118
|
}
|
|
116
|
-
return init;
|
|
119
|
+
return { url: url.toString(), init };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Applies a matched variation: merges its headers into `init` and appends its
|
|
123
|
+
* `queryParams` onto `url`, substituting regex named groups (`${name}`) — so
|
|
124
|
+
* `Sortierung=${sort}` with `{ sort: "3" }` becomes `Sortierung=3`.
|
|
125
|
+
*/
|
|
126
|
+
applyVariation(url, init, variationConfig, groups) {
|
|
127
|
+
init.headers = {
|
|
128
|
+
...init.headers,
|
|
129
|
+
...this.substituteNamedGroups(variationConfig.headers, groups),
|
|
130
|
+
};
|
|
131
|
+
for (const queryParam of variationConfig.queryParams ?? []) {
|
|
132
|
+
const substituted = groups
|
|
133
|
+
? queryParam.replace(/\$\{(\w+)}/g, (match, name) => groups[name] ?? match)
|
|
134
|
+
: queryParam;
|
|
135
|
+
const eq = substituted.indexOf("=");
|
|
136
|
+
const key = eq === -1 ? substituted : substituted.slice(0, eq);
|
|
137
|
+
const value = eq === -1 ? "" : substituted.slice(eq + 1);
|
|
138
|
+
url.searchParams.set(key, value);
|
|
139
|
+
}
|
|
117
140
|
}
|
|
118
141
|
/**
|
|
119
142
|
* Replaces `${name}` placeholders in each header value with the matching
|
|
@@ -113,7 +113,10 @@ export class VirtualOrestesApp {
|
|
|
113
113
|
const variation = UrlObject.searchParams.has("bqvariation")
|
|
114
114
|
? UrlObject.searchParams.get("bqvariation").trim()
|
|
115
115
|
: "default";
|
|
116
|
-
const
|
|
116
|
+
const fetchResult = await this.crawler.fetchRemote(originUrl, variation);
|
|
117
|
+
const response = fetchResult?.response;
|
|
118
|
+
// URL we actually fetched (origin + variation query params); used for the redirect check below
|
|
119
|
+
const fetchedUrl = fetchResult?.url;
|
|
117
120
|
// todo check if we can handle more then htmlResponses
|
|
118
121
|
// only handle html responses
|
|
119
122
|
if (!response?.headers?.get("content-type")?.includes("html")) {
|
|
@@ -123,8 +126,8 @@ export class VirtualOrestesApp {
|
|
|
123
126
|
?.get("Cache-Control")
|
|
124
127
|
?.includes("no-store");
|
|
125
128
|
const customHeaders = this.prepareCustomHeaders(originUrl, hasNoStoreFlag);
|
|
126
|
-
// CDN Redirect
|
|
127
|
-
if (
|
|
129
|
+
// CDN Redirect — compare the fetched URL, not bare originUrl (variation params aren't a redirect)
|
|
130
|
+
if (fetchedUrl !== response.url) {
|
|
128
131
|
return this.handleCdnRedirect(customHeaders, response, requestUrl);
|
|
129
132
|
}
|
|
130
133
|
if (response.status > 300 && response.status < 400) {
|
package/oclif.manifest.json
CHANGED