@speedkit/cli 4.22.1 → 4.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +37 -1
- package/dist/commands/origin-request.d.ts +21 -0
- package/dist/commands/origin-request.js +49 -0
- package/dist/hooks/init/dns-result-order.d.ts +12 -0
- package/dist/hooks/init/dns-result-order.js +14 -0
- package/dist/hooks/init/dns-result-order.spec.d.ts +1 -0
- package/dist/hooks/init/dns-result-order.spec.js +15 -0
- package/dist/services/document-handler-runtime/context/document-handler-runtime-context.d.ts +1 -0
- package/dist/services/document-handler-runtime/context/document-handler-runtime-context.js +4 -0
- package/dist/services/document-handler-runtime/server/request.d.ts +1 -0
- package/dist/services/document-handler-runtime/server/request.js +7 -1
- package/dist/services/document-handler-runtime/templates/execute-document-handler.js +2 -0
- package/dist/services/document-handler-runtime/templates/orestes-mock.js +2 -0
- package/dist/services/document-handler-runtime/templates/test.js +3 -0
- package/dist/services/onboarding/virtual-orestes-app/crawler.d.ts +12 -43
- package/dist/services/onboarding/virtual-orestes-app/crawler.js +42 -133
- package/dist/services/origin-request/asset-variations.d.ts +47 -0
- package/dist/services/origin-request/asset-variations.js +170 -0
- package/dist/services/origin-request/asset-variations.spec.d.ts +1 -0
- package/dist/services/origin-request/asset-variations.spec.js +74 -0
- package/dist/services/origin-request/index.d.ts +4 -0
- package/dist/services/origin-request/index.js +4 -0
- package/dist/services/origin-request/origin-request-context.d.ts +13 -0
- package/dist/services/origin-request/origin-request-context.js +23 -0
- package/dist/services/origin-request/origin-request-service-factory.d.ts +13 -0
- package/dist/services/origin-request/origin-request-service-factory.js +32 -0
- package/dist/services/origin-request/origin-request-service.d.ts +33 -0
- package/dist/services/origin-request/origin-request-service.js +135 -0
- package/oclif.manifest.json +90 -1
- package/package.json +6 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
## [4.23.1](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.23.0...v4.23.1) (2026-08-21)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* **cli:** resolve IPv4 addresses before IPv6 ([057fa59](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/057fa59bb06f3d49af5b8631681e7850a062d585))
|
|
7
|
+
* **onboarding:** allow one hour Shopify staleness for local runs ([4e31124](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/4e31124aa3c217f698ef3bb48a51c83fb5395ca4))
|
|
8
|
+
|
|
9
|
+
# [4.23.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.22.1...v4.23.0) (2026-08-21)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
### Bug Fixes
|
|
13
|
+
|
|
14
|
+
* **onboarding:** fetch origin exactly as the server does ([8da54b0](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/8da54b0ac0ffa04671cb3c7e97988e8ac871f2c5))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
* **origin-request:** simulate the server's origin request for a variation ([dc414e0](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/dc414e03142cab154c8f179cdb67f58922a6d38b))
|
|
20
|
+
|
|
1
21
|
## [4.22.1](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.22.0...v4.22.1) (2026-08-21)
|
|
2
22
|
|
|
3
23
|
|
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
|
|
|
21
21
|
$ sk COMMAND
|
|
22
22
|
running command...
|
|
23
23
|
$ sk (--version)
|
|
24
|
-
@speedkit/cli/4.
|
|
24
|
+
@speedkit/cli/4.23.1 linux-x64 node-v22.23.2
|
|
25
25
|
$ sk --help [COMMAND]
|
|
26
26
|
USAGE
|
|
27
27
|
$ sk COMMAND
|
|
@@ -88,6 +88,7 @@ By either...
|
|
|
88
88
|
* [`sk help [COMMAND]`](#sk-help-command)
|
|
89
89
|
* [`sk login [APP]`](#sk-login-app)
|
|
90
90
|
* [`sk onboarding CUSTOMERPATH`](#sk-onboarding-customerpath)
|
|
91
|
+
* [`sk origin-request CUSTOMERPATH PAGEURL`](#sk-origin-request-customerpath-pageurl)
|
|
91
92
|
* [`sk param-diff PAGEURL`](#sk-param-diff-pageurl)
|
|
92
93
|
* [`sk prewarm APP PATH`](#sk-prewarm-app-path)
|
|
93
94
|
* [`sk pull CUSTOMERPATH`](#sk-pull-customerpath)
|
|
@@ -620,6 +621,41 @@ EXAMPLES
|
|
|
620
621
|
$ sk onboarding customers/decathlon.de -c production -d www.decathlon.de
|
|
621
622
|
```
|
|
622
623
|
|
|
624
|
+
## `sk origin-request CUSTOMERPATH PAGEURL`
|
|
625
|
+
|
|
626
|
+
Request a page from origin with the exact headers Speed Kit's server sends for a cache variation, using the app's deployed runtime config
|
|
627
|
+
|
|
628
|
+
```
|
|
629
|
+
USAGE
|
|
630
|
+
$ sk origin-request CUSTOMERPATH PAGEURL [-c <value>] [-v <value>] [--host <value>] [--body <value>] [--grep
|
|
631
|
+
<value>] [--json] [--variations <value>]
|
|
632
|
+
|
|
633
|
+
ARGUMENTS
|
|
634
|
+
CUSTOMERPATH The customer config path
|
|
635
|
+
PAGEURL Page to request from origin
|
|
636
|
+
|
|
637
|
+
FLAGS
|
|
638
|
+
-c, --configName=<value> [default: production] The costumer config name
|
|
639
|
+
-v, --variation=<value> Cache variation to simulate, e.g. desktop-fmarkt-variant-e_1907090. Omitted means the
|
|
640
|
+
default variation
|
|
641
|
+
--body=<value> Write the response body to this file
|
|
642
|
+
--grep=<value> Print body lines matching this regex, case-insensitive
|
|
643
|
+
--host=<value> Send this Host header, emulating an origin-map host override
|
|
644
|
+
--json Machine-readable output
|
|
645
|
+
--variations=<value> Simulate against a runtime config in a local file instead of the app's deployed one
|
|
646
|
+
|
|
647
|
+
DESCRIPTION
|
|
648
|
+
Request a page from origin with the exact headers Speed Kit's server sends for a cache variation, using the app's
|
|
649
|
+
deployed runtime config
|
|
650
|
+
|
|
651
|
+
EXAMPLES
|
|
652
|
+
$ sk origin-request <customerPath> <pageUrl>
|
|
653
|
+
|
|
654
|
+
$ sk origin-request customers/decathlon.de https://www.decathlon.de/ -v mobile
|
|
655
|
+
|
|
656
|
+
$ sk origin-request customers/decathlon.de https://www.decathlon.de/ --grep "storeId" --body /tmp/origin.html
|
|
657
|
+
```
|
|
658
|
+
|
|
623
659
|
## `sk param-diff PAGEURL`
|
|
624
660
|
|
|
625
661
|
Shows difference in documents with query parameters on/off
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Command } from "@oclif/core";
|
|
2
|
+
export default class OriginRequest extends Command {
|
|
3
|
+
static description: string;
|
|
4
|
+
static args: {
|
|
5
|
+
pageUrl: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
6
|
+
customerPath: import("@oclif/core/interfaces").Arg<string, {
|
|
7
|
+
exists?: boolean;
|
|
8
|
+
}>;
|
|
9
|
+
};
|
|
10
|
+
static flags: {
|
|
11
|
+
variation: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
+
host: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
+
body: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
+
grep: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
15
|
+
json: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
16
|
+
variations: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
+
configName: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
18
|
+
};
|
|
19
|
+
static examples: string[];
|
|
20
|
+
run(): Promise<void>;
|
|
21
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Args, Command, Flags } from "@oclif/core";
|
|
2
|
+
import { CLI_CONFIG_NAME, CLI_CUSTOMER_CONFIG, CLIParameters, CLIParametersChar, CLIParametersExample, } from "../models/cli-parameters.js";
|
|
3
|
+
import { CliConfig } from "../helpers/cli-config.js";
|
|
4
|
+
import { OriginRequestContext, OriginRequestServiceFactory, } from "../services/origin-request/index.js";
|
|
5
|
+
export default class OriginRequest extends Command {
|
|
6
|
+
static description = "Request a page from origin with the exact headers Speed Kit's server sends for a cache variation, using the app's deployed runtime config";
|
|
7
|
+
static args = {
|
|
8
|
+
...CLI_CUSTOMER_CONFIG,
|
|
9
|
+
[CLIParameters.PageUrl]: Args.string({
|
|
10
|
+
name: CLIParameters.PageUrl,
|
|
11
|
+
required: true,
|
|
12
|
+
description: "Page to request from origin",
|
|
13
|
+
example: CLIParametersExample.PageUrl,
|
|
14
|
+
}),
|
|
15
|
+
};
|
|
16
|
+
static flags = {
|
|
17
|
+
...CLI_CONFIG_NAME,
|
|
18
|
+
[CLIParameters.Variation]: Flags.string({
|
|
19
|
+
char: CLIParametersChar.Variation,
|
|
20
|
+
description: "Cache variation to simulate, e.g. desktop-fmarkt-variant-e_1907090. Omitted means the default variation",
|
|
21
|
+
}),
|
|
22
|
+
host: Flags.string({
|
|
23
|
+
description: "Send this Host header, emulating an origin-map host override",
|
|
24
|
+
}),
|
|
25
|
+
body: Flags.string({
|
|
26
|
+
description: "Write the response body to this file",
|
|
27
|
+
}),
|
|
28
|
+
grep: Flags.string({
|
|
29
|
+
description: "Print body lines matching this regex, case-insensitive",
|
|
30
|
+
}),
|
|
31
|
+
json: Flags.boolean({
|
|
32
|
+
description: "Machine-readable output",
|
|
33
|
+
}),
|
|
34
|
+
variations: Flags.string({
|
|
35
|
+
description: "Simulate against a runtime config in a local file instead of the app's deployed one",
|
|
36
|
+
}),
|
|
37
|
+
};
|
|
38
|
+
static examples = [
|
|
39
|
+
`$ sk origin-request <${CLIParameters.CustomerPath}> <${CLIParameters.PageUrl}>`,
|
|
40
|
+
`$ sk origin-request ${CLIParametersExample.CustomerPath} ${CLIParametersExample.PageUrl} -${CLIParametersChar.Variation} mobile`,
|
|
41
|
+
`$ sk origin-request ${CLIParametersExample.CustomerPath} ${CLIParametersExample.PageUrl} --grep "storeId" --body /tmp/origin.html`,
|
|
42
|
+
];
|
|
43
|
+
async run() {
|
|
44
|
+
const { args: { customerPath, pageUrl }, flags: { configName, variation, host, body, grep, json, variations }, } = await this.parse(OriginRequest);
|
|
45
|
+
const context = new OriginRequestContext(customerPath, configName, pageUrl, variation, host, body, grep, json, variations);
|
|
46
|
+
const service = await new OriginRequestServiceFactory(context, new CliConfig(this.config).load()).buildService();
|
|
47
|
+
await service.run();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Hook } from "@oclif/core";
|
|
2
|
+
/**
|
|
3
|
+
* Puts IPv4 addresses first for every lookup the CLI makes. This is the runtime equivalent of
|
|
4
|
+
* `NODE_OPTIONS=--dns-result-order=ipv4first`, so no entrypoint has to export that variable.
|
|
5
|
+
*
|
|
6
|
+
* Node resolves `localhost` to `::1` first. WSL in mirrored networking mode shares `localhost`
|
|
7
|
+
* between Windows and Linux, but Chrome's remote debugging port and the local dev server both
|
|
8
|
+
* listen on IPv4 only. A lookup that answers `::1` therefore never reaches the listener on the
|
|
9
|
+
* other side of the boundary, and the connection fails instead of crossing over.
|
|
10
|
+
*/
|
|
11
|
+
declare const hook: Hook<"init">;
|
|
12
|
+
export default hook;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { setDefaultResultOrder } from "node:dns";
|
|
2
|
+
/**
|
|
3
|
+
* Puts IPv4 addresses first for every lookup the CLI makes. This is the runtime equivalent of
|
|
4
|
+
* `NODE_OPTIONS=--dns-result-order=ipv4first`, so no entrypoint has to export that variable.
|
|
5
|
+
*
|
|
6
|
+
* Node resolves `localhost` to `::1` first. WSL in mirrored networking mode shares `localhost`
|
|
7
|
+
* between Windows and Linux, but Chrome's remote debugging port and the local dev server both
|
|
8
|
+
* listen on IPv4 only. A lookup that answers `::1` therefore never reaches the listener on the
|
|
9
|
+
* other side of the boundary, and the connection fails instead of crossing over.
|
|
10
|
+
*/
|
|
11
|
+
const hook = async () => {
|
|
12
|
+
setDefaultResultOrder("ipv4first");
|
|
13
|
+
};
|
|
14
|
+
export default hook;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { after, describe, it } from "mocha";
|
|
3
|
+
import { getDefaultResultOrder, setDefaultResultOrder } from "node:dns";
|
|
4
|
+
import hook from "./dns-result-order.js";
|
|
5
|
+
describe("dns-result-order hook", () => {
|
|
6
|
+
const previousOrder = getDefaultResultOrder();
|
|
7
|
+
after(() => {
|
|
8
|
+
setDefaultResultOrder(previousOrder);
|
|
9
|
+
});
|
|
10
|
+
it("puts IPv4 addresses first", async () => {
|
|
11
|
+
setDefaultResultOrder("verbatim");
|
|
12
|
+
await hook.call(null, {});
|
|
13
|
+
expect(getDefaultResultOrder()).to.equal("ipv4first");
|
|
14
|
+
});
|
|
15
|
+
});
|
package/dist/services/document-handler-runtime/context/document-handler-runtime-context.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export declare const LOCAL_DYNAMIC_FETCHER_CONFIG = "config_dynamicBlocks";
|
|
|
6
6
|
export declare const LOCAL_STYLES = "config_dynamicStyles";
|
|
7
7
|
export declare const LATEST_DOCUMENT_HANDLER_URL = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
|
|
8
8
|
export declare const LATEST_DYNAMIC_FETCHER_URL = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
|
|
9
|
+
export declare const LOCAL_TRIGGERED_BY = "STREAM:QA";
|
|
9
10
|
export declare const TEST_DOCUMENT_HANDLER_FILE_NAME = "testDocumentHandler";
|
|
10
11
|
export declare const TEST_DATABASE_MOCK_FILE_NAME = "database-mock";
|
|
11
12
|
export declare const TEST_ORESTES_MOCK_FILE_NAME = "orestes-mock";
|
|
@@ -6,6 +6,10 @@ export const LOCAL_DYNAMIC_FETCHER_CONFIG = "config_dynamicBlocks";
|
|
|
6
6
|
export const LOCAL_STYLES = "config_dynamicStyles";
|
|
7
7
|
export const LATEST_DOCUMENT_HANDLER_URL = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
|
|
8
8
|
export const LATEST_DYNAMIC_FETCHER_URL = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
|
|
9
|
+
// Local runs have no persistent cache, so an origin HTML is served straight to the developer and may
|
|
10
|
+
// legitimately be older than the default five minutes. STREAM:QA relaxes the document handler's
|
|
11
|
+
// Shopify staleness check to one hour, see ONE_HOUR_STALENESS_TRIGGERS in document-handler.
|
|
12
|
+
export const LOCAL_TRIGGERED_BY = "STREAM:QA";
|
|
9
13
|
export const TEST_DOCUMENT_HANDLER_FILE_NAME = "testDocumentHandler";
|
|
10
14
|
export const TEST_DATABASE_MOCK_FILE_NAME = "database-mock";
|
|
11
15
|
export const TEST_ORESTES_MOCK_FILE_NAME = "orestes-mock";
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
import { LOCAL_TRIGGERED_BY } from "../context/document-handler-runtime-context.js";
|
|
1
2
|
export class Request {
|
|
2
3
|
query;
|
|
3
4
|
body;
|
|
4
5
|
headers;
|
|
5
6
|
constructor(url, body, variation, originalHeaders, requestHeaders = {}) {
|
|
6
|
-
this.query = {
|
|
7
|
+
this.query = {
|
|
8
|
+
variation,
|
|
9
|
+
headers: JSON.stringify(originalHeaders),
|
|
10
|
+
url,
|
|
11
|
+
triggeredBy: LOCAL_TRIGGERED_BY,
|
|
12
|
+
};
|
|
7
13
|
this.body = body;
|
|
8
14
|
this.headers = {};
|
|
9
15
|
for (const key of Object.keys(requestHeaders)) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import AbstractTemplate from "./abstract-template.js";
|
|
2
|
+
import { LOCAL_TRIGGERED_BY } from "../context/document-handler-runtime-context.js";
|
|
2
3
|
export default class ExecuteDocumentHandler extends AbstractTemplate {
|
|
3
4
|
buildContext(domain, file) {
|
|
4
5
|
return { domain, file };
|
|
@@ -35,6 +36,7 @@ async function run(iterator) {
|
|
|
35
36
|
variation: item.variation || 'DEFAULT',
|
|
36
37
|
headers: item?.headers ? item.headers : JSON.stringify(DEFAULT_HEADERS),
|
|
37
38
|
url: item.url,
|
|
39
|
+
triggeredBy: '${LOCAL_TRIGGERED_BY}',
|
|
38
40
|
}
|
|
39
41
|
let req = {
|
|
40
42
|
query, body: file
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import AbstractTemplate from "./abstract-template.js";
|
|
2
|
+
import { LOCAL_TRIGGERED_BY } from "../context/document-handler-runtime-context.js";
|
|
2
3
|
export default class OrestesMock extends AbstractTemplate {
|
|
3
4
|
buildContext() {
|
|
4
5
|
return {};
|
|
@@ -79,6 +80,7 @@ const server = http.createServer(async function (request, response) {
|
|
|
79
80
|
variation: UrlObject.searchParams.has('bqvariation')?UrlObject.searchParams.get('bqvariation'):'DEFAULT',
|
|
80
81
|
headers: '',
|
|
81
82
|
url: url,
|
|
83
|
+
triggeredBy: '${LOCAL_TRIGGERED_BY}',
|
|
82
84
|
}
|
|
83
85
|
let req = {query, body}
|
|
84
86
|
const output = await documentHandler.transform(database,req)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import AbstractTemplate from "./abstract-template.js";
|
|
2
|
+
import { LOCAL_TRIGGERED_BY } from "../context/document-handler-runtime-context.js";
|
|
2
3
|
export default class Test extends AbstractTemplate {
|
|
3
4
|
buildContext(domain, name, timeout = 2000) {
|
|
4
5
|
return { domain, name, timeout };
|
|
@@ -25,6 +26,7 @@ describe(\`documentHandler run on \${file}\`, function () {
|
|
|
25
26
|
variation: 'DEFAULT',
|
|
26
27
|
headers: JSON.stringify(DEFAULT_HEADERS),
|
|
27
28
|
url: url,
|
|
29
|
+
triggeredBy: '${LOCAL_TRIGGERED_BY}',
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
let req = {query, body}
|
|
@@ -42,6 +44,7 @@ describe(\`create files to diff for \${file}\`,function (){
|
|
|
42
44
|
variation: 'DEFAULT',
|
|
43
45
|
headers: JSON.stringify(DEFAULT_HEADERS),
|
|
44
46
|
url: url,
|
|
47
|
+
triggeredBy: '${LOCAL_TRIGGERED_BY}',
|
|
45
48
|
}
|
|
46
49
|
|
|
47
50
|
let req = {query, body}
|
|
@@ -5,58 +5,27 @@ export declare class Crawler {
|
|
|
5
5
|
private cli;
|
|
6
6
|
private agentConfig;
|
|
7
7
|
private speedKitServerConfig?;
|
|
8
|
+
/** Warnings already shown, so a per-fetch condition is reported once per session. */
|
|
9
|
+
private readonly warned;
|
|
8
10
|
constructor(cli: CliService, agentConfig: Agent, speedKitServerConfig?: SpeedKitServerConfig);
|
|
11
|
+
private warnOnce;
|
|
9
12
|
fetchRemote(originUrl: string, variation: string): Promise<{
|
|
10
13
|
response: Response;
|
|
11
14
|
url: string;
|
|
12
15
|
} | undefined>;
|
|
13
|
-
private setDefaultUserAgent;
|
|
14
16
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* user-agent / etc. headers configured per variation must reach the origin
|
|
18
|
-
* so that the response matches what the variation is supposed to represent
|
|
19
|
-
* (e.g. a logged-in user, a specific store, a mobile device).
|
|
17
|
+
* Builds the origin request exactly as the Baqend asset server builds it, so a locally cached
|
|
18
|
+
* document matches what production would cache for the same variation.
|
|
20
19
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* `variationParameter` first, then fall back to its Java-style
|
|
26
|
-
* `regex` (e.g. `(?i)desktop-fmarkt-variant-(?<storeId>.*)`). The
|
|
27
|
-
* first variation that matches by either means is applied — exact
|
|
28
|
-
* matches do NOT take priority over regex matches in earlier
|
|
29
|
-
* entries. When matched via regex, named capture groups are
|
|
30
|
-
* substituted into header values via `${name}` placeholders, so a
|
|
31
|
-
* cookie like `fmarktcookie=${storeId}` becomes
|
|
32
|
-
* `fmarktcookie=2879130`.
|
|
33
|
-
* 3. Origin-level basic auth fallback (only if no variation matched).
|
|
20
|
+
* The resolution itself lives in `services/origin-request` and is shared with
|
|
21
|
+
* `sk origin-request`, so there is one owner for it and one place the server's behaviour is
|
|
22
|
+
* pinned by specs. That covers the base headers, the wildcard-then-variation order, the regex
|
|
23
|
+
* templates with their `${group}` substitution, and the query params.
|
|
34
24
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
25
|
+
* Two things stay local: origin basic auth, which no variation carries, and tolerance for a
|
|
26
|
+
* variation the config does not define — the server rejects that request, but failing a whole
|
|
27
|
+
* onboarding session over it would be unhelpful, so it falls back to the base headers and warns.
|
|
37
28
|
*/
|
|
38
29
|
private prepareOriginRequest;
|
|
39
|
-
/**
|
|
40
|
-
* Applies a matched variation: merges its headers into `init` and appends its
|
|
41
|
-
* `queryParams` onto `url`, substituting regex named groups (`${name}`) — so
|
|
42
|
-
* `Sortierung=${sort}` with `{ sort: "3" }` becomes `Sortierung=3`.
|
|
43
|
-
*/
|
|
44
|
-
private applyVariation;
|
|
45
|
-
/**
|
|
46
|
-
* Replaces `${name}` placeholders in each header value with the matching
|
|
47
|
-
* named capture group from a regex match. Used by `prepareOriginRequest`
|
|
48
|
-
* to materialize parameterized variation headers — e.g. given:
|
|
49
|
-
*
|
|
50
|
-
* headers = { cookie: "fmarktcookie=${storeId}" }
|
|
51
|
-
* groups = { storeId: "2879130" }
|
|
52
|
-
*
|
|
53
|
-
* the result is `{ cookie: "fmarktcookie=2879130" }`.
|
|
54
|
-
*
|
|
55
|
-
* Placeholders that don't match a captured group are left intact rather
|
|
56
|
-
* than replaced with `undefined`, so a misconfigured variation still
|
|
57
|
-
* produces a recognisable header value instead of a silent corruption.
|
|
58
|
-
* Non-string header values are passed through unchanged.
|
|
59
|
-
*/
|
|
60
|
-
private substituteNamedGroups;
|
|
61
30
|
private prepareAuthRequest;
|
|
62
31
|
}
|
|
@@ -1,26 +1,27 @@
|
|
|
1
|
-
import { USER_AGENT } from "../onboarding-model.js";
|
|
2
1
|
import { safe } from "../../../helpers/safe.js";
|
|
3
|
-
import {
|
|
2
|
+
import { resolveRequest, UndefinedVariationError, } from "../../origin-request/index.js";
|
|
4
3
|
import { randomUUID } from "node:crypto";
|
|
5
4
|
export class Crawler {
|
|
6
5
|
cli;
|
|
7
6
|
agentConfig;
|
|
8
7
|
speedKitServerConfig;
|
|
8
|
+
/** Warnings already shown, so a per-fetch condition is reported once per session. */
|
|
9
|
+
warned = new Set();
|
|
9
10
|
constructor(cli, agentConfig, speedKitServerConfig) {
|
|
10
11
|
this.cli = cli;
|
|
11
12
|
this.agentConfig = agentConfig;
|
|
12
13
|
this.speedKitServerConfig = speedKitServerConfig;
|
|
13
14
|
}
|
|
15
|
+
warnOnce(key, message) {
|
|
16
|
+
if (this.warned.has(key))
|
|
17
|
+
return;
|
|
18
|
+
this.warned.add(key);
|
|
19
|
+
this.cli.writeWarning(message);
|
|
20
|
+
}
|
|
14
21
|
async fetchRemote(originUrl, variation) {
|
|
15
|
-
const userAgent = this.setDefaultUserAgent(variation);
|
|
16
|
-
const headers = {
|
|
17
|
-
"User-Agent": userAgent,
|
|
18
|
-
};
|
|
19
22
|
// origin URL with the variation's query params applied — returned so callers compare it
|
|
20
23
|
// (not the bare originUrl) against response.url for redirect detection
|
|
21
|
-
const { url, init } = this.prepareOriginRequest(originUrl, variation
|
|
22
|
-
headers,
|
|
23
|
-
});
|
|
24
|
+
const { url, init } = this.prepareOriginRequest(originUrl, variation);
|
|
24
25
|
const requestInit = {
|
|
25
26
|
...init,
|
|
26
27
|
agent: this.agentConfig,
|
|
@@ -36,138 +37,46 @@ export class Crawler {
|
|
|
36
37
|
this.cli.writeError(fetchResponse.error);
|
|
37
38
|
this.cli.writeError(JSON.stringify(fetchResponse.errorObj));
|
|
38
39
|
}
|
|
39
|
-
setDefaultUserAgent(variation) {
|
|
40
|
-
if (/^mobile$/is.test(variation)) {
|
|
41
|
-
return USER_AGENT.MOBILE;
|
|
42
|
-
}
|
|
43
|
-
if (/^tablet$/is.test(variation)) {
|
|
44
|
-
return USER_AGENT.TABLET;
|
|
45
|
-
}
|
|
46
|
-
return USER_AGENT.DESKTOP;
|
|
47
|
-
}
|
|
48
40
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
* user-agent / etc. headers configured per variation must reach the origin
|
|
52
|
-
* so that the response matches what the variation is supposed to represent
|
|
53
|
-
* (e.g. a logged-in user, a specific store, a mobile device).
|
|
41
|
+
* Builds the origin request exactly as the Baqend asset server builds it, so a locally cached
|
|
42
|
+
* document matches what production would cache for the same variation.
|
|
54
43
|
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
* `variationParameter` first, then fall back to its Java-style
|
|
60
|
-
* `regex` (e.g. `(?i)desktop-fmarkt-variant-(?<storeId>.*)`). The
|
|
61
|
-
* first variation that matches by either means is applied — exact
|
|
62
|
-
* matches do NOT take priority over regex matches in earlier
|
|
63
|
-
* entries. When matched via regex, named capture groups are
|
|
64
|
-
* substituted into header values via `${name}` placeholders, so a
|
|
65
|
-
* cookie like `fmarktcookie=${storeId}` becomes
|
|
66
|
-
* `fmarktcookie=2879130`.
|
|
67
|
-
* 3. Origin-level basic auth fallback (only if no variation matched).
|
|
44
|
+
* The resolution itself lives in `services/origin-request` and is shared with
|
|
45
|
+
* `sk origin-request`, so there is one owner for it and one place the server's behaviour is
|
|
46
|
+
* pinned by specs. That covers the base headers, the wildcard-then-variation order, the regex
|
|
47
|
+
* templates with their `${group}` substitution, and the query params.
|
|
68
48
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
|
|
72
|
-
prepareOriginRequest(originUrl, variationParameter, init) {
|
|
73
|
-
const url = new URL(originUrl);
|
|
74
|
-
if (!this.speedKitServerConfig) {
|
|
75
|
-
return { url: url.toString(), init };
|
|
76
|
-
}
|
|
77
|
-
const variations = this.speedKitServerConfig.speedKit.variations;
|
|
78
|
-
// Wildcard variation applies to every request, regardless of which
|
|
79
|
-
// specific variation (if any) is requested.
|
|
80
|
-
if (variations?.["*"]) {
|
|
81
|
-
this.applyVariation(url, init, variations["*"], undefined);
|
|
82
|
-
}
|
|
83
|
-
if (!variationParameter) {
|
|
84
|
-
return { url: url.toString(), init };
|
|
85
|
-
}
|
|
86
|
-
// First-match-wins over the variations in declaration order. Each entry
|
|
87
|
-
// can match by exact key or by its Java-style `regex` field; whichever
|
|
88
|
-
// entry matches first applies its headers + query params and short-circuits.
|
|
89
|
-
for (const variationKey in variations) {
|
|
90
|
-
if (variationKey === "*") {
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
const variationConfig = variations[variationKey];
|
|
94
|
-
if (variationParameter === variationKey) {
|
|
95
|
-
this.applyVariation(url, init, variationConfig, undefined);
|
|
96
|
-
return { url: url.toString(), init };
|
|
97
|
-
}
|
|
98
|
-
if (!variationConfig.regex) {
|
|
99
|
-
continue;
|
|
100
|
-
}
|
|
101
|
-
const regex = javaRegexToJs(variationConfig.regex);
|
|
102
|
-
if (!regex) {
|
|
103
|
-
// Unsupported Java regex construct — skip rather than crash so later
|
|
104
|
-
// variations still get a chance to match.
|
|
105
|
-
continue;
|
|
106
|
-
}
|
|
107
|
-
const match = variationParameter.match(regex);
|
|
108
|
-
if (!match) {
|
|
109
|
-
continue;
|
|
110
|
-
}
|
|
111
|
-
this.applyVariation(url, init, variationConfig, match.groups);
|
|
112
|
-
return { url: url.toString(), init };
|
|
113
|
-
}
|
|
114
|
-
// No variation matched: fall back to origin basic auth if configured.
|
|
115
|
-
const authentication = this.prepareAuthRequest(originUrl);
|
|
116
|
-
if (authentication) {
|
|
117
|
-
init.headers["Authorization"] = authentication;
|
|
118
|
-
}
|
|
119
|
-
return { url: url.toString(), init };
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Applies a matched variation: merges its headers into `init` and appends its
|
|
123
|
-
* `queryParams` onto `url`, substituting regex named groups (`${name}`) — so
|
|
124
|
-
* `Sortierung=${sort}` with `{ sort: "3" }` becomes `Sortierung=3`.
|
|
49
|
+
* Two things stay local: origin basic auth, which no variation carries, and tolerance for a
|
|
50
|
+
* variation the config does not define — the server rejects that request, but failing a whole
|
|
51
|
+
* onboarding session over it would be unhelpful, so it falls back to the base headers and warns.
|
|
125
52
|
*/
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
const substituted = groups
|
|
133
|
-
? queryParam.replace(/\$\{(\w+)}/g, (match, name) => groups[name] ?? match)
|
|
134
|
-
: queryParam;
|
|
135
|
-
const eq = substituted.indexOf("=");
|
|
136
|
-
const key = eq === -1 ? substituted : substituted.slice(0, eq);
|
|
137
|
-
const value = eq === -1 ? "" : substituted.slice(eq + 1);
|
|
138
|
-
url.searchParams.set(key, value);
|
|
53
|
+
prepareOriginRequest(originUrl, variationParameter) {
|
|
54
|
+
const variations = this.speedKitServerConfig?.speedKit?.variations ?? {};
|
|
55
|
+
// An empty map means the runtime config never loaded, so no fetch can carry variation headers.
|
|
56
|
+
// Say that once, rather than blaming each variation for being undefined.
|
|
57
|
+
if (!Object.keys(variations).length) {
|
|
58
|
+
this.warnOnce("no-runtime-config", "no variations available from the runtime config, so origin fetches carry the base headers only — the cached document may differ from production. Are you logged in?");
|
|
139
59
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
* named capture group from a regex match. Used by `prepareOriginRequest`
|
|
144
|
-
* to materialize parameterized variation headers — e.g. given:
|
|
145
|
-
*
|
|
146
|
-
* headers = { cookie: "fmarktcookie=${storeId}" }
|
|
147
|
-
* groups = { storeId: "2879130" }
|
|
148
|
-
*
|
|
149
|
-
* the result is `{ cookie: "fmarktcookie=2879130" }`.
|
|
150
|
-
*
|
|
151
|
-
* Placeholders that don't match a captured group are left intact rather
|
|
152
|
-
* than replaced with `undefined`, so a misconfigured variation still
|
|
153
|
-
* produces a recognisable header value instead of a silent corruption.
|
|
154
|
-
* Non-string header values are passed through unchanged.
|
|
155
|
-
*/
|
|
156
|
-
substituteNamedGroups(headers, groups) {
|
|
157
|
-
if (!headers) {
|
|
158
|
-
return {};
|
|
60
|
+
let resolved;
|
|
61
|
+
try {
|
|
62
|
+
resolved = resolveRequest(variations, originUrl, variationParameter);
|
|
159
63
|
}
|
|
160
|
-
|
|
161
|
-
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (!(error instanceof UndefinedVariationError))
|
|
66
|
+
throw error;
|
|
67
|
+
this.warnOnce(`undefined-variation:${variationParameter}`, `${error.message} Falling back to the base headers.`);
|
|
68
|
+
resolved = resolveRequest({}, originUrl);
|
|
162
69
|
}
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
70
|
+
const init = { headers: { ...resolved.headers } };
|
|
71
|
+
// Only a request that matched no specific variation falls back to origin basic auth. The
|
|
72
|
+
// wildcard applies to every request, so it does not count as a match.
|
|
73
|
+
const matchedSpecificVariation = resolved.appliedKeys.some((key) => key !== "*" && !key.startsWith("device-default:"));
|
|
74
|
+
if (!matchedSpecificVariation) {
|
|
75
|
+
const authentication = this.prepareAuthRequest(originUrl);
|
|
76
|
+
if (authentication)
|
|
77
|
+
init.headers["Authorization"] = authentication;
|
|
169
78
|
}
|
|
170
|
-
return
|
|
79
|
+
return { url: resolved.url, init };
|
|
171
80
|
}
|
|
172
81
|
prepareAuthRequest(originUrl) {
|
|
173
82
|
if (!this.speedKitServerConfig ||
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Port of the server's variation resolution, so a request can be reproduced locally with the exact
|
|
3
|
+
* headers Speed Kit would send.
|
|
4
|
+
*
|
|
5
|
+
* Mirrors two pieces of server behaviour:
|
|
6
|
+
* - `AssetVariations.getVariations()` — which variation configs apply and in which order
|
|
7
|
+
* (orestes: `orestes-common/.../typesystem/AssetVariations.kt`)
|
|
8
|
+
* - `AssetClient.newRequest()` / `createRequest()` — base headers, last-writer-wins merging,
|
|
9
|
+
* urlPrefix and queryParams (orestes: `orestes-server/.../asset/AssetClient.kt`)
|
|
10
|
+
*
|
|
11
|
+
* Keep this file free of I/O so the resolution stays unit-testable.
|
|
12
|
+
*/
|
|
13
|
+
/** A single variation entry of the runtime config's `speedKit.variations`. */
|
|
14
|
+
export interface AssetVariation {
|
|
15
|
+
headers?: Record<string, string | null>;
|
|
16
|
+
urlPrefix?: string;
|
|
17
|
+
encodePrefixedUrl?: boolean;
|
|
18
|
+
queryParams?: string[];
|
|
19
|
+
regex?: string | null;
|
|
20
|
+
/** Which config key produced this entry. Set during resolution, for reporting. */
|
|
21
|
+
matchedKey?: string;
|
|
22
|
+
}
|
|
23
|
+
export type AssetVariationMap = Record<string, AssetVariation>;
|
|
24
|
+
export interface ResolvedRequest {
|
|
25
|
+
/** Keys of the applied configs, in the order the server applies them. */
|
|
26
|
+
appliedKeys: string[];
|
|
27
|
+
/** `template` when a regex entry matched, otherwise `named`. */
|
|
28
|
+
resolutionPath: "template" | "named";
|
|
29
|
+
url: string;
|
|
30
|
+
headers: Record<string, string>;
|
|
31
|
+
}
|
|
32
|
+
export declare const DEFAULT_TYPE = "default";
|
|
33
|
+
/** User agents the server falls back to. Keep in sync with AssetVariations.kt. */
|
|
34
|
+
export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15 (compatible; SpeedKit/1.0)";
|
|
35
|
+
/** Thrown where the server would throw UndefinedVariationException. */
|
|
36
|
+
export declare class UndefinedVariationError extends Error {
|
|
37
|
+
constructor(variationType: string);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Resolves which variation configs apply, in server order. The template path returns
|
|
41
|
+
* `[wildcard, template]` and skips the device defaults; otherwise `[default, wildcard, named]`.
|
|
42
|
+
*/
|
|
43
|
+
export declare function getVariations(custom: AssetVariationMap, variationType?: string): AssetVariation[];
|
|
44
|
+
/** Resolves a variation into the concrete request the server would send. */
|
|
45
|
+
export declare function resolveRequest(custom: AssetVariationMap, url: string, variationType?: string, hostOverride?: string): ResolvedRequest;
|
|
46
|
+
/** Renders the resolved request as a curl command, for pasting into a bug report. */
|
|
47
|
+
export declare function toCurl(url: string, headers: Record<string, string>): string;
|