@speedkit/cli 4.25.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +192 -19
- package/dist/commands/auto-prewarm.d.ts +8 -1
- package/dist/commands/auto-prewarm.js +18 -7
- package/dist/commands/generate-pop-config.d.ts +1 -0
- package/dist/commands/generate-pop-config.js +9 -3
- package/dist/commands/prewarm.js +3 -1
- package/dist/commands/query/parameter.d.ts +26 -0
- package/dist/commands/query/parameter.js +27 -0
- package/dist/commands/query/prewarm.d.ts +31 -0
- package/dist/commands/query/prewarm.js +31 -0
- package/dist/commands/revalidate.js +8 -6
- package/dist/helpers/clipboard.d.ts +19 -0
- package/dist/helpers/clipboard.js +70 -0
- package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
- package/dist/helpers/evaluate-speed-kit-config.js +8 -6
- package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
- package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
- package/dist/models/cli-parameters.d.ts +73 -0
- package/dist/models/cli-parameters.js +151 -0
- package/dist/models/cli-parameters.spec.d.ts +1 -0
- package/dist/models/cli-parameters.spec.js +47 -0
- package/dist/services/athena/athena-service.d.ts +2 -0
- package/dist/services/athena/athena-service.js +15 -4
- package/dist/services/athena/athena-service.spec.d.ts +1 -0
- package/dist/services/athena/athena-service.spec.js +74 -0
- package/dist/services/deploy/handler/install-resource-handler.js +3 -4
- package/dist/services/onboarding/onboarding-service-factory.js +2 -2
- package/dist/services/pop-config/pop-config-service.d.ts +8 -1
- package/dist/services/pop-config/pop-config-service.js +16 -28
- package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
- package/dist/services/prewarm/assets/asset-api-client.js +5 -4
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +12 -2
- package/dist/services/prewarm/auto-pre-warm-factory.js +20 -6
- package/dist/services/prewarm/csv-reader.d.ts +21 -2
- package/dist/services/prewarm/csv-reader.js +71 -9
- package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
- package/dist/services/prewarm/csv-reader.spec.js +75 -0
- package/dist/services/prewarm/index.d.ts +1 -0
- package/dist/services/prewarm/index.js +1 -0
- package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
- package/dist/services/prewarm/pre-warm-factory.js +8 -11
- package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
- package/dist/services/prewarm/pre-warm-model.js +0 -1
- package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
- package/dist/services/prewarm/pre-warm-service.js +18 -14
- package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
- package/dist/services/prewarm/prewarm-targets.js +18 -0
- package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
- package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
- package/dist/services/query-builder/queries/device.d.ts +21 -0
- package/dist/services/query-builder/queries/device.js +26 -0
- package/dist/services/query-builder/queries/index.d.ts +6 -0
- package/dist/services/query-builder/queries/index.js +6 -0
- package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
- package/dist/services/query-builder/queries/page-filter.js +135 -0
- package/dist/services/query-builder/queries/page-source.d.ts +30 -0
- package/dist/services/query-builder/queries/page-source.js +79 -0
- package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
- package/dist/services/query-builder/queries/parameter-query.js +142 -0
- package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
- package/dist/services/query-builder/queries/pops-query.js +65 -0
- package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
- package/dist/services/query-builder/queries/prewarm-query.js +227 -0
- package/dist/services/query-builder/queries/query-model.d.ts +97 -0
- package/dist/services/query-builder/queries/query-model.js +19 -0
- package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
- package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
- package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
- package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
- package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
- package/dist/services/query-builder/query-builder-factory.js +35 -7
- package/dist/services/query-builder/query-builder-model.d.ts +31 -19
- package/dist/services/query-builder/query-builder-model.js +5 -5
- package/dist/services/query-builder/query-builder-service.d.ts +26 -5
- package/dist/services/query-builder/query-builder-service.js +157 -44
- package/dist/services/query-builder/query-command.d.ts +26 -0
- package/dist/services/query-builder/query-command.js +35 -0
- package/dist/services/query-builder/rules/index.d.ts +8 -0
- package/dist/services/query-builder/rules/index.js +8 -0
- package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
- package/dist/services/query-builder/rules/match-pattern.js +141 -0
- package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
- package/dist/services/query-builder/rules/pattern-language.js +159 -0
- package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
- package/dist/services/query-builder/rules/rule-model.js +54 -0
- package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
- package/dist/services/query-builder/rules/rule-parser.js +189 -0
- package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
- package/dist/services/query-builder/rules/rule-reducer.js +383 -0
- package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
- package/dist/services/query-builder/rules/rule-relevance.js +69 -0
- package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
- package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
- package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
- package/dist/services/query-builder/rules/strip-parameters.js +197 -0
- package/dist/services/query-builder/sql/expression.d.ts +73 -0
- package/dist/services/query-builder/sql/expression.js +133 -0
- package/dist/services/query-builder/sql/format.d.ts +14 -0
- package/dist/services/query-builder/sql/format.js +48 -0
- package/dist/services/query-builder/sql/index.d.ts +4 -0
- package/dist/services/query-builder/sql/index.js +4 -0
- package/dist/services/query-builder/sql/predicate.d.ts +66 -0
- package/dist/services/query-builder/sql/predicate.js +124 -0
- package/dist/services/query-builder/sql/regex.d.ts +28 -0
- package/dist/services/query-builder/sql/regex.js +81 -0
- package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
- package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
- package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
- package/dist/services/revalidate/revalidate-model.d.ts +9 -3
- package/dist/services/revalidate/revalidate-model.js +12 -3
- package/oclif.manifest.json +354 -104
- package/package.json +11 -2
- package/dist/commands/build-parameter-query.d.ts +0 -14
- package/dist/commands/build-parameter-query.js +0 -25
- package/dist/commands/build-prewarm-query.d.ts +0 -16
- package/dist/commands/build-prewarm-query.js +0 -37
- package/dist/helpers/build-query-helper.d.ts +0 -32
- package/dist/helpers/build-query-helper.js +0 -223
- package/dist/helpers/get-parsed-config.d.ts +0 -15
- package/dist/helpers/get-parsed-config.js +0 -62
- package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
- package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
import { CliServiceInterface } from "../cli/index.js";
|
|
2
|
-
import { AssetItemInterface } from "./index.js";
|
|
2
|
+
import { AssetItemInterface, PrewarmTarget } from "./index.js";
|
|
3
3
|
import { AssetApiClient } from "./assets/asset-api-client.js";
|
|
4
4
|
export declare class PreWarmService {
|
|
5
5
|
private assetClient;
|
|
6
6
|
private cli;
|
|
7
|
-
private
|
|
8
|
-
private variations;
|
|
7
|
+
private targets;
|
|
9
8
|
private verbose;
|
|
10
9
|
private requestsSend;
|
|
11
|
-
constructor(assetClient: AssetApiClient, cli: CliServiceInterface,
|
|
10
|
+
constructor(assetClient: AssetApiClient, cli: CliServiceInterface, targets: readonly PrewarmTarget[], verbose: boolean);
|
|
12
11
|
handleAsset(assetClient: AssetApiClient, entry: AssetItemInterface): Promise<void>;
|
|
13
12
|
run(): Promise<void>;
|
|
14
13
|
private fetchAssetList;
|
|
15
14
|
private handleProgress;
|
|
16
15
|
private fetchUrls;
|
|
16
|
+
/**
|
|
17
|
+
* The request list, in the order the targets were given. The order is the
|
|
18
|
+
* caller's to decide — a list built by device interleaves the variations, so
|
|
19
|
+
* that a run cut short has warmed the busiest entries of each rather than
|
|
20
|
+
* all of one and none of the next.
|
|
21
|
+
*/
|
|
17
22
|
private generateAssetRequestList;
|
|
18
23
|
private printSummary;
|
|
19
24
|
private startProgressBar;
|
|
@@ -5,15 +5,13 @@ import { AssetList } from "./assets/asset-list.js";
|
|
|
5
5
|
export class PreWarmService {
|
|
6
6
|
assetClient;
|
|
7
7
|
cli;
|
|
8
|
-
|
|
9
|
-
variations;
|
|
8
|
+
targets;
|
|
10
9
|
verbose;
|
|
11
10
|
requestsSend = 0;
|
|
12
|
-
constructor(assetClient, cli,
|
|
11
|
+
constructor(assetClient, cli, targets, verbose) {
|
|
13
12
|
this.assetClient = assetClient;
|
|
14
13
|
this.cli = cli;
|
|
15
|
-
this.
|
|
16
|
-
this.variations = variations;
|
|
14
|
+
this.targets = targets;
|
|
17
15
|
this.verbose = verbose;
|
|
18
16
|
}
|
|
19
17
|
async handleAsset(assetClient, entry) {
|
|
@@ -49,7 +47,7 @@ export class PreWarmService {
|
|
|
49
47
|
entry.setError(new Error(`status: ${response.status}, sk-error: ${serverTimingErrorCode}`));
|
|
50
48
|
}
|
|
51
49
|
async run() {
|
|
52
|
-
const assetList = this.generateAssetRequestList(this.
|
|
50
|
+
const assetList = this.generateAssetRequestList(this.targets);
|
|
53
51
|
await this.fetchUrls(assetList);
|
|
54
52
|
await this.printSummary(assetList);
|
|
55
53
|
if (assetList.getRetryCount() === 0) {
|
|
@@ -58,10 +56,12 @@ export class PreWarmService {
|
|
|
58
56
|
this.cli.write("Retry rateLimited requests.");
|
|
59
57
|
// reset for minimal output
|
|
60
58
|
this.requestsSend = 0;
|
|
61
|
-
|
|
59
|
+
// Rebuilt from the items themselves: they already carry the variation they
|
|
60
|
+
// were rate-limited in, and crossing their urls with anything else would
|
|
61
|
+
// both re-expand the list and lose that pairing.
|
|
62
|
+
const retryList = this.generateAssetRequestList(assetList
|
|
62
63
|
.getByStatus(ItemStatus.RETRY)
|
|
63
|
-
.map((
|
|
64
|
-
const retryList = this.generateAssetRequestList(urlsToRetry);
|
|
64
|
+
.map(({ url, variation }) => ({ url, variation })));
|
|
65
65
|
this.assetClient.setIsRetry();
|
|
66
66
|
await this.fetchUrls(retryList);
|
|
67
67
|
await this.printSummary(retryList);
|
|
@@ -109,12 +109,16 @@ export class PreWarmService {
|
|
|
109
109
|
await this.fetchAssetList(assetList);
|
|
110
110
|
this.stopProgressBar();
|
|
111
111
|
}
|
|
112
|
-
|
|
112
|
+
/**
|
|
113
|
+
* The request list, in the order the targets were given. The order is the
|
|
114
|
+
* caller's to decide — a list built by device interleaves the variations, so
|
|
115
|
+
* that a run cut short has warmed the busiest entries of each rather than
|
|
116
|
+
* all of one and none of the next.
|
|
117
|
+
*/
|
|
118
|
+
generateAssetRequestList(targets) {
|
|
113
119
|
const assetList = new AssetList();
|
|
114
|
-
for (const variation of
|
|
115
|
-
|
|
116
|
-
assetList.addItem(new AssetItem(url, variation));
|
|
117
|
-
}
|
|
120
|
+
for (const { url, variation } of targets) {
|
|
121
|
+
assetList.addItem(new AssetItem(url, variation));
|
|
118
122
|
}
|
|
119
123
|
return assetList;
|
|
120
124
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { PrewarmRow, PrewarmTarget } from "./pre-warm-model.js";
|
|
2
|
+
/**
|
|
3
|
+
* The (url, variation) pairs to warm.
|
|
4
|
+
*
|
|
5
|
+
* A list that already names a variation per url is taken as it stands — its
|
|
6
|
+
* order is the one whoever built it chose, and every pair in it was actually
|
|
7
|
+
* requested. A list of bare urls is crossed with the variations the caller
|
|
8
|
+
* named, url-major: each url is warmed in every variation before the next url
|
|
9
|
+
* is touched, so a run cut short has warmed whole pages rather than one
|
|
10
|
+
* variation of all of them.
|
|
11
|
+
*/
|
|
12
|
+
export declare function toPrewarmTargets(rows: readonly PrewarmRow[], variations: readonly string[]): PrewarmTarget[];
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The (url, variation) pairs to warm.
|
|
3
|
+
*
|
|
4
|
+
* A list that already names a variation per url is taken as it stands — its
|
|
5
|
+
* order is the one whoever built it chose, and every pair in it was actually
|
|
6
|
+
* requested. A list of bare urls is crossed with the variations the caller
|
|
7
|
+
* named, url-major: each url is warmed in every variation before the next url
|
|
8
|
+
* is touched, so a run cut short has warmed whole pages rather than one
|
|
9
|
+
* variation of all of them.
|
|
10
|
+
*/
|
|
11
|
+
export function toPrewarmTargets(rows, variations) {
|
|
12
|
+
if (rows.some((row) => row.variation)) {
|
|
13
|
+
return rows
|
|
14
|
+
.filter((row) => row.variation)
|
|
15
|
+
.map(({ url, variation }) => ({ url, variation }));
|
|
16
|
+
}
|
|
17
|
+
return rows.flatMap(({ url }) => variations.map((variation) => ({ url, variation })));
|
|
18
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { describe, it } from "mocha";
|
|
3
|
+
import { toPrewarmTargets } from "./prewarm-targets.js";
|
|
4
|
+
describe("toPrewarmTargets", () => {
|
|
5
|
+
it("should keep a paired list in the order it was given", () => {
|
|
6
|
+
// The order is the query's interleaving of the variations by hit count:
|
|
7
|
+
// grouping or sorting it here would undo the point of the split.
|
|
8
|
+
const rows = [
|
|
9
|
+
{ url: "https://a.de/", variation: "mobile" },
|
|
10
|
+
{ url: "https://b.de/", variation: "desktop" },
|
|
11
|
+
{ url: "https://a.de/", variation: "desktop" },
|
|
12
|
+
];
|
|
13
|
+
expect(toPrewarmTargets(rows, ["default"])).to.deep.equal(rows);
|
|
14
|
+
});
|
|
15
|
+
it("should not cross a paired list with the variations", () => {
|
|
16
|
+
expect(toPrewarmTargets([{ url: "https://a.de/", variation: "mobile" }], ["men", "women"])).to.have.lengthOf(1);
|
|
17
|
+
});
|
|
18
|
+
it("should cross a bare list url-major", () => {
|
|
19
|
+
expect(toPrewarmTargets([{ url: "https://a.de/" }, { url: "https://b.de/" }], ["desktop", "mobile"])).to.deep.equal([
|
|
20
|
+
{ url: "https://a.de/", variation: "desktop" },
|
|
21
|
+
{ url: "https://a.de/", variation: "mobile" },
|
|
22
|
+
{ url: "https://b.de/", variation: "desktop" },
|
|
23
|
+
{ url: "https://b.de/", variation: "mobile" },
|
|
24
|
+
]);
|
|
25
|
+
});
|
|
26
|
+
it("should warm nothing when there is no variation to warm in", () => {
|
|
27
|
+
expect(toPrewarmTargets([{ url: "https://a.de/" }], [])).to.deep.equal([]);
|
|
28
|
+
});
|
|
29
|
+
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import ApplicationError from "../../error-handling/error/application-error.js";
|
|
2
|
+
/**
|
|
3
|
+
* A `--match` pattern the config can never let through.
|
|
4
|
+
*
|
|
5
|
+
* The query would be valid and return nothing, after a full Athena scan. This
|
|
6
|
+
* stops before the query is sent.
|
|
7
|
+
*/
|
|
8
|
+
export declare class MatchMatchesNothingError extends ApplicationError {
|
|
9
|
+
readonly code = "MATCH_MATCHES_NOTHING";
|
|
10
|
+
constructor(reason: string);
|
|
11
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import ApplicationError from "../../error-handling/error/application-error.js";
|
|
2
|
+
/**
|
|
3
|
+
* A `--match` pattern the config can never let through.
|
|
4
|
+
*
|
|
5
|
+
* The query would be valid and return nothing, after a full Athena scan. This
|
|
6
|
+
* stops before the query is sent.
|
|
7
|
+
*/
|
|
8
|
+
export class MatchMatchesNothingError extends ApplicationError {
|
|
9
|
+
code = "MATCH_MATCHES_NOTHING";
|
|
10
|
+
constructor(reason) {
|
|
11
|
+
super(`--match ${reason}`);
|
|
12
|
+
this.suggestions = [
|
|
13
|
+
"the pattern is matched against the url without its scheme, e.g. www.shop.de/p/1",
|
|
14
|
+
"check the pattern against the config's enabledSites, disabledSites and blacklist",
|
|
15
|
+
];
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The device classes a pre-warm list can be split by.
|
|
3
|
+
*
|
|
4
|
+
* These are the variation names Speed Kit computes from the user agent, not
|
|
5
|
+
* the values `rum.pi` records — see {@link DEVICE_COLUMN_VALUES} for the
|
|
6
|
+
* translation. `Robot` is not one of them: a query never counts bots.
|
|
7
|
+
*
|
|
8
|
+
* A custom variation (`desktop-pdp-a`, `abtest_b`, `men`) can never be derived
|
|
9
|
+
* from a device, so it cannot be a value here — `--variation` puts those into
|
|
10
|
+
* the list by hand.
|
|
11
|
+
*/
|
|
12
|
+
export declare const DEVICE_VARIATIONS: readonly ["desktop", "mobile", "tablet"];
|
|
13
|
+
export type DeviceVariation = (typeof DEVICE_VARIATIONS)[number];
|
|
14
|
+
/** How `rum.pi` spells each of them in its `device` column. */
|
|
15
|
+
export declare const DEVICE_COLUMN_VALUES: Record<DeviceVariation, string>;
|
|
16
|
+
/**
|
|
17
|
+
* The requested devices in the canonical order, without duplicates — so the
|
|
18
|
+
* same request always produces the same SQL, whatever order the flag was
|
|
19
|
+
* given in.
|
|
20
|
+
*/
|
|
21
|
+
export declare function normalizeDevices(devices: readonly string[] | undefined): DeviceVariation[];
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The device classes a pre-warm list can be split by.
|
|
3
|
+
*
|
|
4
|
+
* These are the variation names Speed Kit computes from the user agent, not
|
|
5
|
+
* the values `rum.pi` records — see {@link DEVICE_COLUMN_VALUES} for the
|
|
6
|
+
* translation. `Robot` is not one of them: a query never counts bots.
|
|
7
|
+
*
|
|
8
|
+
* A custom variation (`desktop-pdp-a`, `abtest_b`, `men`) can never be derived
|
|
9
|
+
* from a device, so it cannot be a value here — `--variation` puts those into
|
|
10
|
+
* the list by hand.
|
|
11
|
+
*/
|
|
12
|
+
export const DEVICE_VARIATIONS = ["desktop", "mobile", "tablet"];
|
|
13
|
+
/** How `rum.pi` spells each of them in its `device` column. */
|
|
14
|
+
export const DEVICE_COLUMN_VALUES = {
|
|
15
|
+
desktop: "Desktop",
|
|
16
|
+
mobile: "Mobile",
|
|
17
|
+
tablet: "Tablet",
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* The requested devices in the canonical order, without duplicates — so the
|
|
21
|
+
* same request always produces the same SQL, whatever order the flag was
|
|
22
|
+
* given in.
|
|
23
|
+
*/
|
|
24
|
+
export function normalizeDevices(devices) {
|
|
25
|
+
return DEVICE_VARIATIONS.filter((device) => devices?.includes(device));
|
|
26
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { Predicate } from "../sql/index.js";
|
|
2
|
+
import { RuleNote, RuleSubjectExpressions } from "../rules/index.js";
|
|
3
|
+
import { QueryInput } from "./query-model.js";
|
|
4
|
+
export interface RuleFilter {
|
|
5
|
+
readonly approximated: RuleNote[];
|
|
6
|
+
readonly dropped: RuleNote[];
|
|
7
|
+
/** Findings about the config itself, worth reporting on their own. */
|
|
8
|
+
readonly findings: RuleNote[];
|
|
9
|
+
readonly predicate: Predicate;
|
|
10
|
+
/** Rules left out because another rule already decides their pages. */
|
|
11
|
+
readonly reduced: RuleNote[];
|
|
12
|
+
readonly warnings: string[];
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Which page impressions a query counts at all: the app's own traffic in the
|
|
16
|
+
* requested window, without the noise the caller asked to leave out.
|
|
17
|
+
*
|
|
18
|
+
* Bots are always excluded — a crawler requests pages no real user asks for,
|
|
19
|
+
* and neither pre-warming nor a parameter analysis should be driven by them.
|
|
20
|
+
* Naming the device classes to keep subsumes that: `Robot` is not one of them,
|
|
21
|
+
* and `in` leaves out a null device the same way `<> 'Robot'` does.
|
|
22
|
+
*/
|
|
23
|
+
export declare function buildRowFilter(input: QueryInput): Predicate;
|
|
24
|
+
/**
|
|
25
|
+
* Which of those pages Speed Kit accelerates.
|
|
26
|
+
*
|
|
27
|
+
* The rule sets are combined the way the runtime evaluates them:
|
|
28
|
+
* `enabledSites` decides first and an empty set means every page, then
|
|
29
|
+
* `disabledSites` takes pages away; `whitelist` and `blacklist` do the same
|
|
30
|
+
* for the document as a resource. `disabledScopes` takes away whatever the
|
|
31
|
+
* service worker never claims in the first place.
|
|
32
|
+
*
|
|
33
|
+
* Before that, the sets are ruled against each other. Each set loses the rules
|
|
34
|
+
* a sibling already covers, and the two excluding sets additionally lose every
|
|
35
|
+
* rule that excludes pages the including sets never let through — which is
|
|
36
|
+
* most of a real blacklist, since it guards paths `enabledSites` never
|
|
37
|
+
* allowed. What remains is the part of the config the data can actually see.
|
|
38
|
+
*/
|
|
39
|
+
export declare function buildRuleFilter(input: QueryInput, subjects: RuleSubjectExpressions): RuleFilter;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { and, call, column, equals, FALSE, greaterOrEqual, inValues, leaf, literal, not, notEquals, TRUE, } from "../sql/index.js";
|
|
2
|
+
import { absorbRules, asPathWithSearchRule, buildMatchRule, dropUnreachableRules, findShadowedRules, keepRelevantRules, reductionBudget, translateRules, } from "../rules/index.js";
|
|
3
|
+
import { MatchMatchesNothingError } from "../error/match-matches-nothing-error.js";
|
|
4
|
+
import { DEVICE_COLUMN_VALUES, normalizeDevices } from "./device.js";
|
|
5
|
+
import { sinceDate } from "./query-model.js";
|
|
6
|
+
/** The navigation types that are a full document load. */
|
|
7
|
+
const HARD_NAVIGATION_TYPES = ["NAVIGATE", "RELOAD", "BACK_FORWARD"];
|
|
8
|
+
/**
|
|
9
|
+
* Which page impressions a query counts at all: the app's own traffic in the
|
|
10
|
+
* requested window, without the noise the caller asked to leave out.
|
|
11
|
+
*
|
|
12
|
+
* Bots are always excluded — a crawler requests pages no real user asks for,
|
|
13
|
+
* and neither pre-warming nor a parameter analysis should be driven by them.
|
|
14
|
+
* Naming the device classes to keep subsumes that: `Robot` is not one of them,
|
|
15
|
+
* and `in` leaves out a null device the same way `<> 'Robot'` does.
|
|
16
|
+
*/
|
|
17
|
+
export function buildRowFilter(input) {
|
|
18
|
+
const devices = normalizeDevices(input.devices);
|
|
19
|
+
return and(equals(column("app"), literal(input.app)), greaterOrEqual(column("date"), literal(sinceDate(input.days, input.now))), devices.length > 0
|
|
20
|
+
? inValues(column("device"), devices.map((device) => DEVICE_COLUMN_VALUES[device]))
|
|
21
|
+
: notEquals(column("device"), literal("Robot")), input.hardNavigationsOnly
|
|
22
|
+
? inValues(column("navigationtype"), HARD_NAVIGATION_TYPES)
|
|
23
|
+
: TRUE, input.excludeSuspicious ? not(isSet("suspicious")) : TRUE, input.excludeNoindex ? not(isSet("noindex")) : TRUE);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Which of those pages Speed Kit accelerates.
|
|
27
|
+
*
|
|
28
|
+
* The rule sets are combined the way the runtime evaluates them:
|
|
29
|
+
* `enabledSites` decides first and an empty set means every page, then
|
|
30
|
+
* `disabledSites` takes pages away; `whitelist` and `blacklist` do the same
|
|
31
|
+
* for the document as a resource. `disabledScopes` takes away whatever the
|
|
32
|
+
* service worker never claims in the first place.
|
|
33
|
+
*
|
|
34
|
+
* Before that, the sets are ruled against each other. Each set loses the rules
|
|
35
|
+
* a sibling already covers, and the two excluding sets additionally lose every
|
|
36
|
+
* rule that excludes pages the including sets never let through — which is
|
|
37
|
+
* most of a real blacklist, since it guards paths `enabledSites` never
|
|
38
|
+
* allowed. What remains is the part of the config the data can actually see.
|
|
39
|
+
*/
|
|
40
|
+
export function buildRuleFilter(input, subjects) {
|
|
41
|
+
const { ruleSets } = input;
|
|
42
|
+
const approximated = [];
|
|
43
|
+
const dropped = [];
|
|
44
|
+
const reduced = [];
|
|
45
|
+
const warnings = [];
|
|
46
|
+
// One budget for the whole build, so the reductions cannot add up.
|
|
47
|
+
const budget = reductionBudget();
|
|
48
|
+
/** The rules of a set that can apply to a page impression at all. */
|
|
49
|
+
const relevant = (rules) => {
|
|
50
|
+
const result = keepRelevantRules(rules, { hosts: input.hosts });
|
|
51
|
+
approximated.push(...result.approximated);
|
|
52
|
+
dropped.push(...result.dropped);
|
|
53
|
+
return result.rules;
|
|
54
|
+
};
|
|
55
|
+
/** What a sibling rule already covers, dropped. */
|
|
56
|
+
const absorb = (rules) => {
|
|
57
|
+
const absorbed = absorbRules(rules, budget);
|
|
58
|
+
reduced.push(...absorbed.notes);
|
|
59
|
+
return absorbed.rules;
|
|
60
|
+
};
|
|
61
|
+
const translate = (rules, whenEmpty) => {
|
|
62
|
+
const translation = translateRules(rules, subjects, { whenEmpty });
|
|
63
|
+
warnings.push(...translation.warnings);
|
|
64
|
+
return translation.predicate;
|
|
65
|
+
};
|
|
66
|
+
const whitelist = absorb(relevant(ruleSets.whitelist));
|
|
67
|
+
const enabledSites = absorb(relevant(ruleSets.enabledSites));
|
|
68
|
+
// An empty including set lets everything through, so it constrains nothing.
|
|
69
|
+
const reachable = [enabledSites, whitelist].filter((set) => set.length > 0);
|
|
70
|
+
/**
|
|
71
|
+
* Reachability first, absorption second.
|
|
72
|
+
*
|
|
73
|
+
* Both shorten the query, but dropping what can never fire is the reduction
|
|
74
|
+
* that carries a real config — most of a blacklist guards pages
|
|
75
|
+
* `enabledSites` never allowed — while absorbing a sibling is cosmetic and
|
|
76
|
+
* costs a comparison for every pair of entries. Spending the budget in that
|
|
77
|
+
* order means a config that runs out of it still got the reduction worth
|
|
78
|
+
* having, and absorption then works on fewer entries.
|
|
79
|
+
*/
|
|
80
|
+
const exclude = (rules) => {
|
|
81
|
+
const reachableOnly = dropUnreachableRules(rules, reachable, budget);
|
|
82
|
+
reduced.push(...reachableOnly.notes);
|
|
83
|
+
return absorb(reachableOnly.rules);
|
|
84
|
+
};
|
|
85
|
+
const excluding = [
|
|
86
|
+
relevant(ruleSets.blacklist),
|
|
87
|
+
relevant(ruleSets.disabledSites),
|
|
88
|
+
relevant(ruleSets.disabledScopes),
|
|
89
|
+
];
|
|
90
|
+
/**
|
|
91
|
+
* The narrowing the caller asked for, held against what the config takes
|
|
92
|
+
* away.
|
|
93
|
+
*
|
|
94
|
+
* A pattern the excluding rules already cover would build a valid query,
|
|
95
|
+
* scan the whole window and return nothing, so it is refused before the
|
|
96
|
+
* query is handed over. The rules are the ones the query frame leaves, not
|
|
97
|
+
* the ones the reduction leaves: an entry dropped below because
|
|
98
|
+
* `enabledSites` never lets its pages through still says that a pattern
|
|
99
|
+
* aimed at them has nothing to warm.
|
|
100
|
+
*
|
|
101
|
+
* The comparison gets a budget of its own. It costs one comparison per
|
|
102
|
+
* excluding rule, and sharing the build's budget would mean a config large
|
|
103
|
+
* enough to spend it first is silently not checked at all.
|
|
104
|
+
*/
|
|
105
|
+
const match = buildMatchRule(input.match);
|
|
106
|
+
if (match) {
|
|
107
|
+
const [shadowed] = findShadowedRules([match], excluding.flat().map(asPathWithSearchRule).filter(Boolean), reductionBudget());
|
|
108
|
+
if (shadowed) {
|
|
109
|
+
throw new MatchMatchesNothingError(shadowed.reason);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const [blacklist, disabledSites, disabledScopes] = excluding.map(exclude);
|
|
113
|
+
const predicate = and(translate(whitelist, TRUE), not(translate(blacklist, FALSE)), translate(enabledSites, TRUE), not(translate(disabledSites, FALSE)),
|
|
114
|
+
// A page inside a disabled scope is served by an empty service worker, so
|
|
115
|
+
// Speed Kit never sees it at all.
|
|
116
|
+
not(translate(disabledScopes, FALSE)),
|
|
117
|
+
// Last, so the query reads as the config's filter with the caller's
|
|
118
|
+
// narrowing applied to it.
|
|
119
|
+
match ? translate([match], TRUE) : TRUE);
|
|
120
|
+
return {
|
|
121
|
+
approximated,
|
|
122
|
+
dropped,
|
|
123
|
+
findings: findShadowedRules(enabledSites, [...disabledScopes, ...disabledSites, ...blacklist], budget),
|
|
124
|
+
predicate,
|
|
125
|
+
reduced,
|
|
126
|
+
warnings,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* A boolean column that may be null. `not column` would drop the null rows
|
|
131
|
+
* along with the true ones, which is not what excluding a flag means.
|
|
132
|
+
*/
|
|
133
|
+
function isSet(name) {
|
|
134
|
+
return leaf(call("coalesce", column(name), literal(false)), true);
|
|
135
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { RuleSubjectExpressions } from "../rules/index.js";
|
|
2
|
+
import { QueryInput } from "./query-model.js";
|
|
3
|
+
/** The relation every query reads from. */
|
|
4
|
+
export declare const PAGE_IMPRESSIONS = "live.rum.pi";
|
|
5
|
+
/** The name of the common table expression the queries select from. */
|
|
6
|
+
export declare const PAGE_SOURCE = "pages";
|
|
7
|
+
export interface PageSource {
|
|
8
|
+
/** The `pages as (…)` common table expression, without the `with`. */
|
|
9
|
+
readonly cte: string;
|
|
10
|
+
/** The expressions rule conditions are matched against. */
|
|
11
|
+
readonly subjects: RuleSubjectExpressions;
|
|
12
|
+
}
|
|
13
|
+
export interface PageSourceOptions {
|
|
14
|
+
/**
|
|
15
|
+
* Columns of the relation to carry through, beyond `url`. Each query asks
|
|
16
|
+
* for what it reads: `createdat` is only needed where example urls are
|
|
17
|
+
* picked, `responsecause` only where 404s are filtered out.
|
|
18
|
+
*/
|
|
19
|
+
readonly columns?: readonly string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The rows a query works on: the app's page impressions in the requested
|
|
23
|
+
* window, with everything a rule matches against projected once.
|
|
24
|
+
*
|
|
25
|
+
* Deriving host and path per condition would repeat the same
|
|
26
|
+
* `url_extract_path(url)` a dozen times in a single `where` clause. Naming
|
|
27
|
+
* them here keeps the query readable — which matters, because these queries
|
|
28
|
+
* are read and adjusted by hand in the Athena console.
|
|
29
|
+
*/
|
|
30
|
+
export declare function buildPageSource(input: QueryInput, options?: PageSourceOptions): PageSource;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { column, formatPredicate } from "../sql/index.js";
|
|
2
|
+
import { resolvedSubject, urlSubjects, } from "../rules/index.js";
|
|
3
|
+
import { normalizeDevices } from "./device.js";
|
|
4
|
+
import { buildRowFilter } from "./page-filter.js";
|
|
5
|
+
/** The relation every query reads from. */
|
|
6
|
+
export const PAGE_IMPRESSIONS = "live.rum.pi";
|
|
7
|
+
/** The name of the common table expression the queries select from. */
|
|
8
|
+
export const PAGE_SOURCE = "pages";
|
|
9
|
+
const COLUMN_NAMES = {
|
|
10
|
+
host: "host",
|
|
11
|
+
path: "path",
|
|
12
|
+
pathWithSearch: "pathWithSearch",
|
|
13
|
+
url: "matchUrl",
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* The rows a query works on: the app's page impressions in the requested
|
|
17
|
+
* window, with everything a rule matches against projected once.
|
|
18
|
+
*
|
|
19
|
+
* Deriving host and path per condition would repeat the same
|
|
20
|
+
* `url_extract_path(url)` a dozen times in a single `where` clause. Naming
|
|
21
|
+
* them here keeps the query readable — which matters, because these queries
|
|
22
|
+
* are read and adjusted by hand in the Athena console.
|
|
23
|
+
*/
|
|
24
|
+
export function buildPageSource(input, options = {}) {
|
|
25
|
+
const inline = urlSubjects(column("url"));
|
|
26
|
+
const used = usedSubjects(input);
|
|
27
|
+
const projections = [
|
|
28
|
+
"url",
|
|
29
|
+
// The device is the variation a split list is keyed by, so it is read
|
|
30
|
+
// exactly where one was asked for.
|
|
31
|
+
...(normalizeDevices(input.devices).length > 0 ? [column("device")] : []),
|
|
32
|
+
...(options.columns ?? []).map((name) => column(name)),
|
|
33
|
+
...Object.keys(COLUMN_NAMES)
|
|
34
|
+
.filter((subject) => used.has(subject))
|
|
35
|
+
.map((subject) => `${inline[subject]} as ${COLUMN_NAMES[subject]}`),
|
|
36
|
+
];
|
|
37
|
+
const cte = `${PAGE_SOURCE} as (
|
|
38
|
+
select
|
|
39
|
+
${projections.join(",\n ")}
|
|
40
|
+
from ${PAGE_IMPRESSIONS}
|
|
41
|
+
where ${formatPredicate(buildRowFilter(input), { level: 2 })}
|
|
42
|
+
)`;
|
|
43
|
+
// An unused subject keeps its inline expression: nothing refers to it, and
|
|
44
|
+
// that way a caller cannot accidentally read a column that is not there.
|
|
45
|
+
const subjects = Object.fromEntries(Object.keys(COLUMN_NAMES).map((subject) => [
|
|
46
|
+
subject,
|
|
47
|
+
used.has(subject)
|
|
48
|
+
? column(COLUMN_NAMES[subject])
|
|
49
|
+
: inline[subject],
|
|
50
|
+
]));
|
|
51
|
+
return { cte, subjects };
|
|
52
|
+
}
|
|
53
|
+
/** The subjects the config's rules actually match against. */
|
|
54
|
+
function usedSubjects(input) {
|
|
55
|
+
const { ruleSets } = input;
|
|
56
|
+
const used = new Set();
|
|
57
|
+
// A `--match` pattern is a version 1 pathname condition, which reads the
|
|
58
|
+
// query string along with the path.
|
|
59
|
+
if (input.match && input.match.length > 0) {
|
|
60
|
+
used.add("pathWithSearch");
|
|
61
|
+
}
|
|
62
|
+
const collect = (rules) => {
|
|
63
|
+
for (const rule of rules) {
|
|
64
|
+
for (const { subject } of rule.conditions) {
|
|
65
|
+
used.add(resolvedSubject(subject, rule.ruleSetVersion));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
collect(ruleSets.blacklist);
|
|
70
|
+
collect(ruleSets.disabledScopes);
|
|
71
|
+
collect(ruleSets.disabledSites);
|
|
72
|
+
collect(ruleSets.enabledSites);
|
|
73
|
+
collect(ruleSets.whitelist);
|
|
74
|
+
for (const rule of ruleSets.stripQueryParams) {
|
|
75
|
+
collect(rule.enabledSites ?? []);
|
|
76
|
+
collect(rule.rules ?? []);
|
|
77
|
+
}
|
|
78
|
+
return used;
|
|
79
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { BuiltQuery, QueryInput } from "./query-model.js";
|
|
2
|
+
/**
|
|
3
|
+
* The query parameters that appear on accelerated pages, ranked by how many
|
|
4
|
+
* cache entries stripping them would collapse.
|
|
5
|
+
*
|
|
6
|
+
* Ranked *per parameter*: a parameter travels in many different combinations —
|
|
7
|
+
* on a large shop a single one shows up in over a thousand of them — so a
|
|
8
|
+
* ranking over whole combinations never states what one parameter is worth,
|
|
9
|
+
* which is the question being asked. `bySet` brings the combination view back.
|
|
10
|
+
*
|
|
11
|
+
* Parameters the config already strips are filtered out, so the result only
|
|
12
|
+
* lists candidates that still need a decision.
|
|
13
|
+
*/
|
|
14
|
+
export declare function buildParameterQuery(input: QueryInput): BuiltQuery;
|