@speedkit/cli 4.26.0 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +203 -15
- package/dist/commands/auto-prewarm.d.ts +12 -1
- package/dist/commands/auto-prewarm.js +26 -7
- package/dist/commands/generate-pop-config.d.ts +1 -0
- package/dist/commands/generate-pop-config.js +9 -3
- package/dist/commands/prewarm.js +3 -1
- package/dist/commands/query/parameter.d.ts +27 -0
- package/dist/commands/query/parameter.js +28 -0
- package/dist/commands/query/prewarm.d.ts +32 -0
- package/dist/commands/query/prewarm.js +32 -0
- package/dist/helpers/clipboard.d.ts +19 -0
- package/dist/helpers/clipboard.js +70 -0
- package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
- package/dist/helpers/evaluate-speed-kit-config.js +8 -6
- package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
- package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
- package/dist/models/cli-parameters.d.ts +106 -0
- package/dist/models/cli-parameters.js +183 -0
- package/dist/models/cli-parameters.spec.d.ts +1 -0
- package/dist/models/cli-parameters.spec.js +47 -0
- package/dist/services/athena/athena-service-factory.d.ts +4 -1
- package/dist/services/athena/athena-service-factory.js +9 -3
- package/dist/services/athena/athena-service.d.ts +13 -1
- package/dist/services/athena/athena-service.js +27 -5
- package/dist/services/athena/athena-service.spec.d.ts +1 -0
- package/dist/services/athena/athena-service.spec.js +93 -0
- package/dist/services/deploy/handler/install-resource-handler.js +3 -4
- package/dist/services/onboarding/onboarding-service-factory.js +2 -2
- package/dist/services/pop-config/pop-config-service.d.ts +8 -1
- package/dist/services/pop-config/pop-config-service.js +16 -28
- package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
- package/dist/services/prewarm/assets/asset-api-client.js +5 -4
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +20 -2
- package/dist/services/prewarm/auto-pre-warm-factory.js +24 -6
- package/dist/services/prewarm/csv-reader.d.ts +21 -2
- package/dist/services/prewarm/csv-reader.js +71 -9
- package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
- package/dist/services/prewarm/csv-reader.spec.js +75 -0
- package/dist/services/prewarm/index.d.ts +1 -0
- package/dist/services/prewarm/index.js +1 -0
- package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
- package/dist/services/prewarm/pre-warm-factory.js +8 -11
- package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
- package/dist/services/prewarm/pre-warm-model.js +0 -1
- package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
- package/dist/services/prewarm/pre-warm-service.js +18 -14
- package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
- package/dist/services/prewarm/prewarm-targets.js +18 -0
- package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
- package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
- package/dist/services/query-builder/queries/device.d.ts +21 -0
- package/dist/services/query-builder/queries/device.js +26 -0
- package/dist/services/query-builder/queries/index.d.ts +6 -0
- package/dist/services/query-builder/queries/index.js +6 -0
- package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
- package/dist/services/query-builder/queries/page-filter.js +135 -0
- package/dist/services/query-builder/queries/page-source.d.ts +30 -0
- package/dist/services/query-builder/queries/page-source.js +79 -0
- package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
- package/dist/services/query-builder/queries/parameter-query.js +142 -0
- package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
- package/dist/services/query-builder/queries/pops-query.js +65 -0
- package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
- package/dist/services/query-builder/queries/prewarm-query.js +227 -0
- package/dist/services/query-builder/queries/query-model.d.ts +97 -0
- package/dist/services/query-builder/queries/query-model.js +19 -0
- package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
- package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
- package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
- package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
- package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
- package/dist/services/query-builder/query-builder-factory.js +36 -8
- package/dist/services/query-builder/query-builder-model.d.ts +33 -19
- package/dist/services/query-builder/query-builder-model.js +5 -5
- package/dist/services/query-builder/query-builder-service.d.ts +26 -5
- package/dist/services/query-builder/query-builder-service.js +157 -44
- package/dist/services/query-builder/query-command.d.ts +27 -0
- package/dist/services/query-builder/query-command.js +36 -0
- package/dist/services/query-builder/rules/index.d.ts +8 -0
- package/dist/services/query-builder/rules/index.js +8 -0
- package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
- package/dist/services/query-builder/rules/match-pattern.js +141 -0
- package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
- package/dist/services/query-builder/rules/pattern-language.js +159 -0
- package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
- package/dist/services/query-builder/rules/rule-model.js +54 -0
- package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
- package/dist/services/query-builder/rules/rule-parser.js +189 -0
- package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
- package/dist/services/query-builder/rules/rule-reducer.js +383 -0
- package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
- package/dist/services/query-builder/rules/rule-relevance.js +69 -0
- package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
- package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
- package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
- package/dist/services/query-builder/rules/strip-parameters.js +197 -0
- package/dist/services/query-builder/sql/expression.d.ts +73 -0
- package/dist/services/query-builder/sql/expression.js +133 -0
- package/dist/services/query-builder/sql/format.d.ts +14 -0
- package/dist/services/query-builder/sql/format.js +48 -0
- package/dist/services/query-builder/sql/index.d.ts +4 -0
- package/dist/services/query-builder/sql/index.js +4 -0
- package/dist/services/query-builder/sql/predicate.d.ts +66 -0
- package/dist/services/query-builder/sql/predicate.js +124 -0
- package/dist/services/query-builder/sql/regex.d.ts +28 -0
- package/dist/services/query-builder/sql/regex.js +81 -0
- package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
- package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
- package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
- package/oclif.manifest.json +401 -100
- package/package.json +11 -2
- package/dist/commands/build-parameter-query.d.ts +0 -14
- package/dist/commands/build-parameter-query.js +0 -25
- package/dist/commands/build-prewarm-query.d.ts +0 -16
- package/dist/commands/build-prewarm-query.js +0 -37
- package/dist/helpers/build-query-helper.d.ts +0 -32
- package/dist/helpers/build-query-helper.js +0 -223
- package/dist/helpers/get-parsed-config.d.ts +0 -15
- package/dist/helpers/get-parsed-config.js +0 -62
- package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
- package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
|
@@ -1,22 +1,37 @@
|
|
|
1
|
-
import { getSpeedKitConfigVersion } from "../../helpers/get-parsed-config.js";
|
|
2
|
-
import { getBlacklistedFilter, getParameterFilter, getParameterReplaceFilter, getStrippedParameters, getWhitelistedFilter, } from "../../helpers/build-query-helper.js";
|
|
3
1
|
import { writeFile } from "node:fs/promises";
|
|
4
2
|
import { join } from "node:path";
|
|
5
|
-
import clipboardy from "clipboardy";
|
|
6
3
|
import json2Csv from "json2csv";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { Parameter } from "./query/parameter.js";
|
|
4
|
+
import { QueryType } from "./query-builder-model.js";
|
|
5
|
+
import { buildParameterQuery, buildPrewarmQuery, DEFAULT_QUERY_DAYS, } from "./queries/index.js";
|
|
10
6
|
import { AthenaBucket } from "../athena/index.js";
|
|
7
|
+
import { copyToClipboard } from "../../helpers/clipboard.js";
|
|
11
8
|
import { safe } from "../../helpers/safe.js";
|
|
9
|
+
/**
|
|
10
|
+
* The notes as unique lines. Each phase of the reduction reports on its own,
|
|
11
|
+
* so the same note — the budget running out, above all — can arrive more than
|
|
12
|
+
* once.
|
|
13
|
+
*/
|
|
14
|
+
function distinct(notes) {
|
|
15
|
+
return [
|
|
16
|
+
...new Set(notes.map(({ reason, source }) => `Reduced ${source}: ${reason}`)),
|
|
17
|
+
];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* A count as it is read: grouped in thousands. Athena hands numbers over as
|
|
21
|
+
* strings, so anything that is not one is passed through untouched.
|
|
22
|
+
*/
|
|
23
|
+
function readable(value) {
|
|
24
|
+
const text = String(value ?? "");
|
|
25
|
+
return /^\d+$/.test(text) ? Number(text).toLocaleString("en-US") : text;
|
|
26
|
+
}
|
|
12
27
|
export class QueryBuilderService {
|
|
13
28
|
context;
|
|
14
|
-
|
|
29
|
+
ruleSets;
|
|
15
30
|
cli;
|
|
16
31
|
athenaService;
|
|
17
|
-
constructor(context,
|
|
32
|
+
constructor(context, ruleSets, cli, athenaService) {
|
|
18
33
|
this.context = context;
|
|
19
|
-
this.
|
|
34
|
+
this.ruleSets = ruleSets;
|
|
20
35
|
this.cli = cli;
|
|
21
36
|
this.athenaService = athenaService;
|
|
22
37
|
}
|
|
@@ -25,37 +40,49 @@ export class QueryBuilderService {
|
|
|
25
40
|
return this.context.app;
|
|
26
41
|
}
|
|
27
42
|
async run() {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
|
|
32
|
-
const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
|
|
33
|
-
const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
|
|
34
|
-
const parameterFilter = getParameterFilter(parameters, skConfigVersion);
|
|
35
|
-
const result = safe(() => this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter));
|
|
36
|
-
this.cli.successAction(`QUERY:BUILD:${this.context.queryType}`);
|
|
43
|
+
const action = `QUERY:BUILD:${this.context.queryType}`;
|
|
44
|
+
this.cli.startAction(action, `build ${this.context.queryType}-query`);
|
|
45
|
+
const result = safe(() => this.build());
|
|
37
46
|
if (result.success !== true) {
|
|
38
|
-
this.cli.failAction(
|
|
47
|
+
this.cli.failAction(action, result.error);
|
|
39
48
|
this.cli.writeError(result.error);
|
|
40
49
|
console.error(result.errorObj);
|
|
41
50
|
return;
|
|
42
51
|
}
|
|
43
|
-
this.cli.successAction(
|
|
52
|
+
this.cli.successAction(action);
|
|
53
|
+
const query = result.data;
|
|
54
|
+
if (this.context.count) {
|
|
55
|
+
if (this.context.explain) {
|
|
56
|
+
this.report(query);
|
|
57
|
+
}
|
|
58
|
+
await this.executeAndReportCounts(query.sql);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
44
61
|
if (this.context.execute) {
|
|
45
|
-
await this.executeAndExport(
|
|
62
|
+
await this.executeAndExport(query.sql);
|
|
46
63
|
return;
|
|
47
64
|
}
|
|
48
65
|
if (this.context.quiet) {
|
|
49
|
-
|
|
66
|
+
// The query goes to stdout so it can be piped; an accounting the caller
|
|
67
|
+
// asked for goes next to it, on stderr.
|
|
68
|
+
if (this.context.explain) {
|
|
69
|
+
process.stderr.write(`${this.reportLines(query).join("\n")}\n`);
|
|
70
|
+
}
|
|
71
|
+
process.stdout.write(query.sql);
|
|
50
72
|
return;
|
|
51
73
|
}
|
|
52
|
-
|
|
53
|
-
|
|
74
|
+
this.report(query);
|
|
75
|
+
// The query is the result, the clipboard is a convenience: write it out
|
|
76
|
+
// first so nothing about the clipboard can delay or hide it.
|
|
77
|
+
this.cli.write(this.cli.style.dim.italic(query.sql));
|
|
78
|
+
if (this.context.isWindows) {
|
|
79
|
+
return;
|
|
54
80
|
}
|
|
55
|
-
|
|
56
|
-
if (!this.context.isWindows) {
|
|
81
|
+
if (await copyToClipboard(query.sql)) {
|
|
57
82
|
this.cli.writeSuccess("Result has been copied to the clipboard 📋");
|
|
83
|
+
return;
|
|
58
84
|
}
|
|
85
|
+
this.cli.write(this.cli.style.dim("The clipboard could not be reached 📋"));
|
|
59
86
|
}
|
|
60
87
|
/**
|
|
61
88
|
* Build and execute the query against Athena (Baqend native path, AWS
|
|
@@ -65,12 +92,115 @@ export class QueryBuilderService {
|
|
|
65
92
|
async execute() {
|
|
66
93
|
return this.runQuery(this.query());
|
|
67
94
|
}
|
|
95
|
+
/** The query as SQL. */
|
|
96
|
+
query() {
|
|
97
|
+
return this.build().sql;
|
|
98
|
+
}
|
|
99
|
+
build() {
|
|
100
|
+
const input = {
|
|
101
|
+
app: this.context.app,
|
|
102
|
+
bySet: this.context.bySet,
|
|
103
|
+
count: this.context.count,
|
|
104
|
+
days: this.context.days ?? DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
105
|
+
devices: this.context.devices,
|
|
106
|
+
excludeNotFound: this.context.excludeNotFound,
|
|
107
|
+
excludeNoindex: this.context.excludeNoindex,
|
|
108
|
+
excludeSuspicious: this.context.excludeSuspicious,
|
|
109
|
+
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
110
|
+
hosts: this.context.hosts,
|
|
111
|
+
limit: this.context.limit,
|
|
112
|
+
match: this.context.match,
|
|
113
|
+
minHits: this.context.minHits,
|
|
114
|
+
minImpressions: this.context.minImpressions,
|
|
115
|
+
ruleSets: this.ruleSets,
|
|
116
|
+
variations: this.context.variations,
|
|
117
|
+
};
|
|
118
|
+
return this.context.queryType === QueryType.prewarm
|
|
119
|
+
? buildPrewarmQuery(input)
|
|
120
|
+
: buildParameterQuery(input);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Points out where the query is not exactly what the config says — a rule
|
|
124
|
+
* left out, a rule the query can only approximate, a pattern Athena reads
|
|
125
|
+
* differently than the browser — plus what the config itself gets wrong.
|
|
126
|
+
*
|
|
127
|
+
* Reductions are silent unless asked for: a rule another rule already
|
|
128
|
+
* decides is not a divergence, it is arithmetic.
|
|
129
|
+
*/
|
|
130
|
+
report(query) {
|
|
131
|
+
for (const { reason, source } of query.findings) {
|
|
132
|
+
this.cli.writeWarning(`${source} ${reason}`);
|
|
133
|
+
}
|
|
134
|
+
for (const { reason, source } of query.dropped) {
|
|
135
|
+
this.cli.writeWarning(`Left out ${source}: ${reason}`);
|
|
136
|
+
}
|
|
137
|
+
for (const { reason, source } of query.approximated) {
|
|
138
|
+
this.cli.writeWarning(`Approximated ${source}: ${reason}`);
|
|
139
|
+
}
|
|
140
|
+
for (const warning of query.warnings) {
|
|
141
|
+
this.cli.writeWarning(warning);
|
|
142
|
+
}
|
|
143
|
+
const reduced = distinct(query.reduced);
|
|
144
|
+
if (this.context.explain) {
|
|
145
|
+
for (const line of reduced) {
|
|
146
|
+
this.cli.write(this.cli.style.dim(line));
|
|
147
|
+
}
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (reduced.length > 0) {
|
|
151
|
+
this.cli.write(this.cli.style.dim(`${reduced.length} rule(s) reduced away — run with --explain to see them`));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** The same accounting as {@link report}, as plain lines. */
|
|
155
|
+
reportLines(query) {
|
|
156
|
+
return [
|
|
157
|
+
...query.findings.map(({ reason, source }) => `${source} ${reason}`),
|
|
158
|
+
...query.dropped.map(({ reason, source }) => `Left out ${source}: ${reason}`),
|
|
159
|
+
...query.approximated.map(({ reason, source }) => `Approximated ${source}: ${reason}`),
|
|
160
|
+
...query.warnings,
|
|
161
|
+
...distinct(query.reduced),
|
|
162
|
+
];
|
|
163
|
+
}
|
|
68
164
|
async runQuery(query) {
|
|
69
165
|
if (!this.athenaService) {
|
|
70
166
|
throw new Error("No Athena service available to execute the query");
|
|
71
167
|
}
|
|
72
168
|
return this.athenaService.getResult(query, [], undefined, AthenaBucket.RumPi);
|
|
73
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Execute the count query and put its handful of rows on the terminal — a
|
|
172
|
+
* count is read, not filed, so nothing is written to disk.
|
|
173
|
+
*
|
|
174
|
+
* Under `--quiet` the rows go to stdout as tab separated values, so a count
|
|
175
|
+
* can be read by something other than a person.
|
|
176
|
+
*/
|
|
177
|
+
async executeAndReportCounts(query) {
|
|
178
|
+
this.cli.startAction("QUERY:COUNT", "count the pre-warm list");
|
|
179
|
+
const result = await safe(this.runQuery(query));
|
|
180
|
+
if (result.success !== true) {
|
|
181
|
+
this.cli.failAction("QUERY:COUNT");
|
|
182
|
+
this.cli.writeError(result.error);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
this.cli.successAction("QUERY:COUNT");
|
|
186
|
+
const rows = result.data;
|
|
187
|
+
if (rows.length === 0) {
|
|
188
|
+
this.cli.writeWarning("No page impressions in the window — the list would be empty");
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
if (this.context.quiet) {
|
|
192
|
+
const columns = Object.keys(rows[0]);
|
|
193
|
+
process.stdout.write([
|
|
194
|
+
columns.join("\t"),
|
|
195
|
+
...rows.map((row) => columns.map((name) => row[name]).join("\t")),
|
|
196
|
+
].join("\n") + "\n");
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
this.cli.table(rows, Object.fromEntries(Object.keys(rows[0]).map((name) => [
|
|
200
|
+
name,
|
|
201
|
+
{ get: (row) => readable(row[name]) },
|
|
202
|
+
])));
|
|
203
|
+
}
|
|
74
204
|
/**
|
|
75
205
|
* Execute the built query against Athena (Baqend native path, AWS fallback)
|
|
76
206
|
* and write the rows to a `<app>-<YYYY-MM-DD>-<queryType>.csv` file in the
|
|
@@ -80,7 +210,7 @@ export class QueryBuilderService {
|
|
|
80
210
|
this.cli.startAction("QUERY:EXECUTE", "execute query and export csv");
|
|
81
211
|
const result = await safe(this.runQuery(query));
|
|
82
212
|
if (result.success !== true) {
|
|
83
|
-
this.cli.failAction("QUERY:EXECUTE"
|
|
213
|
+
this.cli.failAction("QUERY:EXECUTE");
|
|
84
214
|
this.cli.writeError(result.error);
|
|
85
215
|
return;
|
|
86
216
|
}
|
|
@@ -93,21 +223,4 @@ export class QueryBuilderService {
|
|
|
93
223
|
this.cli.successAction("QUERY:EXECUTE");
|
|
94
224
|
this.cli.writeSuccess(`Wrote ${rows.length} rows to ${filePath} 📄`);
|
|
95
225
|
}
|
|
96
|
-
query() {
|
|
97
|
-
const parameters = getStrippedParameters(this.speedKitConfig);
|
|
98
|
-
const parameterReplaceFilter = getParameterReplaceFilter(parameters, getSpeedKitConfigVersion(this.speedKitConfig));
|
|
99
|
-
const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
|
|
100
|
-
const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
|
|
101
|
-
const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
|
|
102
|
-
const parameterFilter = getParameterFilter(parameters, skConfigVersion);
|
|
103
|
-
return this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter);
|
|
104
|
-
}
|
|
105
|
-
buildQuery(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter) {
|
|
106
|
-
if (this.context.queryType === QueryTpe.prewarm) {
|
|
107
|
-
return new Prewarm(app, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
|
|
108
|
-
}
|
|
109
|
-
if (this.context.queryType === QueryTpe.parameter) {
|
|
110
|
-
return new Parameter(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
226
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Config } from "@oclif/core";
|
|
2
|
+
import { ParameterOptions, PrewarmOptions } from "./queries/query-model.js";
|
|
3
|
+
import { QueryFilterOptions, QueryType } from "./query-builder-model.js";
|
|
4
|
+
/**
|
|
5
|
+
* The flags a query command parses; each takes the ones it offers.
|
|
6
|
+
*
|
|
7
|
+
* `--device` and `--variation` are named as the flags are, singular, and are
|
|
8
|
+
* the two the caller gives rather than the list the builder works on.
|
|
9
|
+
*/
|
|
10
|
+
export interface QueryCommandFlags extends ParameterOptions, Omit<PrewarmOptions, "devices" | "variations">, QueryFilterOptions {
|
|
11
|
+
readonly configName: string;
|
|
12
|
+
readonly device?: readonly string[];
|
|
13
|
+
readonly execute?: boolean;
|
|
14
|
+
readonly explain?: boolean;
|
|
15
|
+
readonly quiet?: boolean;
|
|
16
|
+
readonly variation?: readonly string[];
|
|
17
|
+
readonly workgroup?: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Builds one of the config-derived queries and hands it over.
|
|
21
|
+
*
|
|
22
|
+
* Both `sk query` commands differ only in which query they ask for, so the
|
|
23
|
+
* wiring lives here rather than twice in the command layer — the two of them
|
|
24
|
+
* drifting apart is what left a diverged copy of the parameter query behind
|
|
25
|
+
* once already.
|
|
26
|
+
*/
|
|
27
|
+
export declare function runQuery(queryType: QueryType, customerPath: string, flags: QueryCommandFlags, config: Config): Promise<void>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { CliConfig } from "../../helpers/cli-config.js";
|
|
2
|
+
import { QueryBuilderFactory } from "./query-builder-factory.js";
|
|
3
|
+
import { normalizeDevices } from "./queries/device.js";
|
|
4
|
+
/**
|
|
5
|
+
* Builds one of the config-derived queries and hands it over.
|
|
6
|
+
*
|
|
7
|
+
* Both `sk query` commands differ only in which query they ask for, so the
|
|
8
|
+
* wiring lives here rather than twice in the command layer — the two of them
|
|
9
|
+
* drifting apart is what left a diverged copy of the parameter query behind
|
|
10
|
+
* once already.
|
|
11
|
+
*/
|
|
12
|
+
export async function runQuery(queryType, customerPath, flags, config) {
|
|
13
|
+
const service = await new QueryBuilderFactory({
|
|
14
|
+
bySet: flags.bySet,
|
|
15
|
+
configName: flags.configName,
|
|
16
|
+
count: flags.count,
|
|
17
|
+
customerPath,
|
|
18
|
+
days: flags.days,
|
|
19
|
+
devices: normalizeDevices(flags.device),
|
|
20
|
+
excludeNotFound: flags.excludeNotFound,
|
|
21
|
+
excludeNoindex: flags.excludeNoindex,
|
|
22
|
+
excludeSuspicious: flags.excludeSuspicious,
|
|
23
|
+
execute: flags.execute,
|
|
24
|
+
explain: flags.explain,
|
|
25
|
+
hardNavigationsOnly: flags.hardNavigationsOnly,
|
|
26
|
+
isWindows: config.windows,
|
|
27
|
+
limit: flags.limit,
|
|
28
|
+
match: flags.match,
|
|
29
|
+
minHits: flags.minHits,
|
|
30
|
+
minImpressions: flags.minImpressions,
|
|
31
|
+
quiet: flags.quiet,
|
|
32
|
+
variations: flags.variation,
|
|
33
|
+
workgroup: flags.workgroup,
|
|
34
|
+
}, new CliConfig(config).load()).buildService(queryType);
|
|
35
|
+
await service.run();
|
|
36
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from "./match-pattern.js";
|
|
2
|
+
export * from "./pattern-language.js";
|
|
3
|
+
export * from "./rule-model.js";
|
|
4
|
+
export * from "./rule-parser.js";
|
|
5
|
+
export * from "./rule-reducer.js";
|
|
6
|
+
export * from "./rule-relevance.js";
|
|
7
|
+
export * from "./rule-to-sql.js";
|
|
8
|
+
export * from "./strip-parameters.js";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from "./match-pattern.js";
|
|
2
|
+
export * from "./pattern-language.js";
|
|
3
|
+
export * from "./rule-model.js";
|
|
4
|
+
export * from "./rule-parser.js";
|
|
5
|
+
export * from "./rule-reducer.js";
|
|
6
|
+
export * from "./rule-relevance.js";
|
|
7
|
+
export * from "./rule-to-sql.js";
|
|
8
|
+
export * from "./strip-parameters.js";
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { NormalizedRule } from "./rule-model.js";
|
|
2
|
+
/**
|
|
3
|
+
* An ad-hoc narrowing of a query, given on the command line as a regular
|
|
4
|
+
* expression over the path.
|
|
5
|
+
*
|
|
6
|
+
* It is compiled into the same shape a config rule has, so everything that
|
|
7
|
+
* follows — the check against the config, the translation to SQL, the
|
|
8
|
+
* warnings about constructs Athena reads differently — is the code that
|
|
9
|
+
* already runs for the config's own rules, and the pattern cannot come to
|
|
10
|
+
* mean something else here than it would in a config.
|
|
11
|
+
*
|
|
12
|
+
* The imports here are deliberately kept to the leaves of the rule layer: the
|
|
13
|
+
* flag definitions load this module, and pulling the automata library into
|
|
14
|
+
* every `sk` invocation to validate a pattern would be a poor trade.
|
|
15
|
+
*/
|
|
16
|
+
/** Where a `--match` pattern shows up in reports and errors. */
|
|
17
|
+
export declare const MATCH_SOURCE = "--match";
|
|
18
|
+
/**
|
|
19
|
+
* Compiles one `--match` value into a regular expression.
|
|
20
|
+
*
|
|
21
|
+
* Athena carries flags inside the pattern and JavaScript carries them beside
|
|
22
|
+
* it. The operator writes the Athena form, so a leading `(?i)` is lifted into
|
|
23
|
+
* real flags here — `new RegExp('(?i)…')` would throw — and `toAthenaRegex`
|
|
24
|
+
* puts it back when the pattern is emitted.
|
|
25
|
+
*
|
|
26
|
+
* @throws SyntaxError when the value is not a regular expression.
|
|
27
|
+
*/
|
|
28
|
+
export declare function parseMatchPattern(value: string): RegExp;
|
|
29
|
+
/**
|
|
30
|
+
* The `--match` patterns as a rule: one condition on the pathname, its
|
|
31
|
+
* entries OR'd, so `--match a --match b` keeps the pages matching either.
|
|
32
|
+
*/
|
|
33
|
+
export declare function buildMatchRule(values: readonly string[] | undefined): NormalizedRule | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* An excluding rule restated over `path + search`, so that it can be held
|
|
36
|
+
* against a `--match` pattern — or `undefined` when it cannot be restated
|
|
37
|
+
* exactly.
|
|
38
|
+
*
|
|
39
|
+
* A version 2 `pathname` condition is matched against the path alone, a
|
|
40
|
+
* version 1 one against the path with its query string, and the reduction
|
|
41
|
+
* never compares patterns across those two subjects. Since a `--match`
|
|
42
|
+
* pattern is a version 1 condition, a version 2 rule has to be said again in
|
|
43
|
+
* that subject before the two can be compared at all.
|
|
44
|
+
*
|
|
45
|
+
* This one is used to prove that a pattern is **entirely excluded**, so the
|
|
46
|
+
* safe direction is the opposite of the usual: a rule restated wider than it
|
|
47
|
+
* is would refuse a pattern that does have pages. Anything that cannot be
|
|
48
|
+
* restated exactly is therefore left out of the comparison altogether, and
|
|
49
|
+
* missing an exclusion only costs the early break.
|
|
50
|
+
*/
|
|
51
|
+
export declare function asPathWithSearchRule(rule: NormalizedRule): NormalizedRule | undefined;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { escapeRegexLiteral } from "../sql/regex.js";
|
|
2
|
+
import { matchesExactly, } from "./rule-model.js";
|
|
3
|
+
/**
|
|
4
|
+
* An ad-hoc narrowing of a query, given on the command line as a regular
|
|
5
|
+
* expression over the path.
|
|
6
|
+
*
|
|
7
|
+
* It is compiled into the same shape a config rule has, so everything that
|
|
8
|
+
* follows — the check against the config, the translation to SQL, the
|
|
9
|
+
* warnings about constructs Athena reads differently — is the code that
|
|
10
|
+
* already runs for the config's own rules, and the pattern cannot come to
|
|
11
|
+
* mean something else here than it would in a config.
|
|
12
|
+
*
|
|
13
|
+
* The imports here are deliberately kept to the leaves of the rule layer: the
|
|
14
|
+
* flag definitions load this module, and pulling the automata library into
|
|
15
|
+
* every `sk` invocation to validate a pattern would be a poor trade.
|
|
16
|
+
*/
|
|
17
|
+
/** Where a `--match` pattern shows up in reports and errors. */
|
|
18
|
+
export const MATCH_SOURCE = "--match";
|
|
19
|
+
/**
|
|
20
|
+
* A `--match` pattern is a version 1 `pathname` condition, whatever version
|
|
21
|
+
* the config runs under: version 1 matches a pathname against the path *and*
|
|
22
|
+
* the query string, which is the subject an operator narrowing a list wants —
|
|
23
|
+
* `^/p/` anchors at the start of the path and `[?&]size=` reaches a
|
|
24
|
+
* parameter.
|
|
25
|
+
*/
|
|
26
|
+
const MATCH_RULE_SET_VERSION = 1;
|
|
27
|
+
/** Inline flags, the way Athena takes them: `(?i)`, `(?is)`, … */
|
|
28
|
+
const INLINE_FLAGS = /^\(\?([ims]+)\)/;
|
|
29
|
+
/**
|
|
30
|
+
* Compiles one `--match` value into a regular expression.
|
|
31
|
+
*
|
|
32
|
+
* Athena carries flags inside the pattern and JavaScript carries them beside
|
|
33
|
+
* it. The operator writes the Athena form, so a leading `(?i)` is lifted into
|
|
34
|
+
* real flags here — `new RegExp('(?i)…')` would throw — and `toAthenaRegex`
|
|
35
|
+
* puts it back when the pattern is emitted.
|
|
36
|
+
*
|
|
37
|
+
* @throws SyntaxError when the value is not a regular expression.
|
|
38
|
+
*/
|
|
39
|
+
export function parseMatchPattern(value) {
|
|
40
|
+
const inline = INLINE_FLAGS.exec(value);
|
|
41
|
+
return inline
|
|
42
|
+
? new RegExp(value.slice(inline[0].length), inline[1])
|
|
43
|
+
: new RegExp(value);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The `--match` patterns as a rule: one condition on the pathname, its
|
|
47
|
+
* entries OR'd, so `--match a --match b` keeps the pages matching either.
|
|
48
|
+
*/
|
|
49
|
+
export function buildMatchRule(values) {
|
|
50
|
+
if (!values || values.length === 0) {
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
const patterns = values.map((value) => ({
|
|
54
|
+
kind: "regexp",
|
|
55
|
+
value: parseMatchPattern(value),
|
|
56
|
+
}));
|
|
57
|
+
const condition = { patterns, subject: "pathname" };
|
|
58
|
+
return {
|
|
59
|
+
conditions: [condition],
|
|
60
|
+
criteriaOutsideTheUrl: [],
|
|
61
|
+
matchesNothing: false,
|
|
62
|
+
ruleSetVersion: MATCH_RULE_SET_VERSION,
|
|
63
|
+
source: MATCH_SOURCE,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* An excluding rule restated over `path + search`, so that it can be held
|
|
68
|
+
* against a `--match` pattern — or `undefined` when it cannot be restated
|
|
69
|
+
* exactly.
|
|
70
|
+
*
|
|
71
|
+
* A version 2 `pathname` condition is matched against the path alone, a
|
|
72
|
+
* version 1 one against the path with its query string, and the reduction
|
|
73
|
+
* never compares patterns across those two subjects. Since a `--match`
|
|
74
|
+
* pattern is a version 1 condition, a version 2 rule has to be said again in
|
|
75
|
+
* that subject before the two can be compared at all.
|
|
76
|
+
*
|
|
77
|
+
* This one is used to prove that a pattern is **entirely excluded**, so the
|
|
78
|
+
* safe direction is the opposite of the usual: a rule restated wider than it
|
|
79
|
+
* is would refuse a pattern that does have pages. Anything that cannot be
|
|
80
|
+
* restated exactly is therefore left out of the comparison altogether, and
|
|
81
|
+
* missing an exclusion only costs the early break.
|
|
82
|
+
*/
|
|
83
|
+
export function asPathWithSearchRule(rule) {
|
|
84
|
+
// A version 1 rule already matches the subject the pattern is written for.
|
|
85
|
+
if (!matchesExactly(rule.ruleSetVersion)) {
|
|
86
|
+
return rule;
|
|
87
|
+
}
|
|
88
|
+
const conditions = [];
|
|
89
|
+
for (const condition of rule.conditions) {
|
|
90
|
+
// A rule constraining anything but the path cannot cover the pattern in
|
|
91
|
+
// the first place — the reduction checks that before it builds automata.
|
|
92
|
+
if (condition.subject !== "pathname") {
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
const patterns = [];
|
|
96
|
+
for (const pattern of condition.patterns) {
|
|
97
|
+
const restated = withSearch(pattern);
|
|
98
|
+
if (!restated) {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
patterns.push(restated);
|
|
102
|
+
}
|
|
103
|
+
conditions.push({ patterns, subject: "pathname" });
|
|
104
|
+
}
|
|
105
|
+
return { ...rule, conditions, ruleSetVersion: MATCH_RULE_SET_VERSION };
|
|
106
|
+
}
|
|
107
|
+
/** What may follow the path in `path + search`. */
|
|
108
|
+
const SEARCH = String.raw `(?:\?[\s\S]*)?`;
|
|
109
|
+
/** Anything but a single `$` at the very end. */
|
|
110
|
+
const INNER_END_ANCHOR = /(?<!\\)\$(?!$)/;
|
|
111
|
+
/**
|
|
112
|
+
* One entry of a version 2 path condition, said over `path + search`.
|
|
113
|
+
*
|
|
114
|
+
* Only the shapes whose meaning is unchanged by the query string are
|
|
115
|
+
* restated. A pattern anchored at the start says the same thing about both
|
|
116
|
+
* subjects; where it ends at the end of the path, the query string is spelled
|
|
117
|
+
* out instead. An unanchored pattern does not survive: it would match inside
|
|
118
|
+
* a query string the path never contained.
|
|
119
|
+
*/
|
|
120
|
+
function withSearch(pattern) {
|
|
121
|
+
if (pattern.kind === "string") {
|
|
122
|
+
// A version 2 string is the whole path, compared case-insensitively.
|
|
123
|
+
const value = escapeRegexLiteral(pattern.value);
|
|
124
|
+
return { kind: "regexp", value: new RegExp(`^${value}${SEARCH}$`, "i") };
|
|
125
|
+
}
|
|
126
|
+
const { flags, source } = pattern.value;
|
|
127
|
+
// An alternation may hide an anchor in a branch, which the reasoning below
|
|
128
|
+
// does not see.
|
|
129
|
+
if (!source.startsWith("^") || source.includes("|")) {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
if (INNER_END_ANCHOR.test(source)) {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
kind: "regexp",
|
|
137
|
+
value: /(?<!\\)\$$/.test(source)
|
|
138
|
+
? new RegExp(`${source.slice(0, -1)}${SEARCH}$`, flags)
|
|
139
|
+
: pattern.value,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { DFA, NFA } from "refa";
|
|
2
|
+
import { RuleCondition } from "./rule-model.js";
|
|
3
|
+
/**
|
|
4
|
+
* The set of subjects a condition accepts, as a finite automaton.
|
|
5
|
+
*
|
|
6
|
+
* Deciding whether one rule makes another redundant means deciding whether one
|
|
7
|
+
* pattern's language contains or avoids another's — a question about regular
|
|
8
|
+
* languages, not about strings, so it is answered with automata rather than
|
|
9
|
+
* guessed. `refa` builds them from the very same JavaScript patterns the config
|
|
10
|
+
* uses.
|
|
11
|
+
*/
|
|
12
|
+
export type ConditionLanguage = NFA;
|
|
13
|
+
/**
|
|
14
|
+
* Which side of the true language an automaton is allowed to fall on.
|
|
15
|
+
*
|
|
16
|
+
* Automata cannot express an assertion — a word boundary, or the `$` inside
|
|
17
|
+
* `/^\/shop(\/|$)/` — so a pattern using one is approximated. Approximating in
|
|
18
|
+
* an arbitrary direction would let the reducer drop a rule that does matter,
|
|
19
|
+
* so each question is asked with the bound that makes its answer sound:
|
|
20
|
+
*
|
|
21
|
+
* - `atLeast` accepts everything the pattern accepts, and possibly more. Two
|
|
22
|
+
* of these sharing nothing proves the patterns share nothing.
|
|
23
|
+
* - `atMost` accepts only what the pattern accepts, and possibly less. An
|
|
24
|
+
* `atLeast` language inside an `atMost` one proves containment.
|
|
25
|
+
*
|
|
26
|
+
* A pattern without assertions has one exact language, and both bounds are it.
|
|
27
|
+
*/
|
|
28
|
+
export type LanguageBound = "atLeast" | "atMost";
|
|
29
|
+
/**
|
|
30
|
+
* The language of one condition — the union of its entries, since one matching
|
|
31
|
+
* entry satisfies the condition.
|
|
32
|
+
*
|
|
33
|
+
* Returns `undefined` when an entry has no automaton at all, and the caller
|
|
34
|
+
* then leaves the rule out of every reduction.
|
|
35
|
+
*/
|
|
36
|
+
export declare function conditionLanguage(condition: RuleCondition, ruleSetVersion: number, bound: LanguageBound): ConditionLanguage | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* Everything a language does *not* accept.
|
|
39
|
+
*
|
|
40
|
+
* Determinising a language to complement it is the expensive step of the whole
|
|
41
|
+
* reduction, and one rule is compared with many others, so a caller that asks
|
|
42
|
+
* repeatedly should build this once and reuse it.
|
|
43
|
+
*/
|
|
44
|
+
export type LanguageOutside = DFA;
|
|
45
|
+
/** The complement of an `atMost` language. */
|
|
46
|
+
export declare function outsideOf(language: ConditionLanguage): LanguageOutside | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* True when the language stays inside the one `outside` was built from —
|
|
49
|
+
* `language ⊆ superset ⟺ language ∩ ¬superset = ∅`.
|
|
50
|
+
*
|
|
51
|
+
* @param language - has to be an `atLeast` language.
|
|
52
|
+
* @param outside - has to come from {@link outsideOf} on an `atMost` language.
|
|
53
|
+
*/
|
|
54
|
+
export declare function isInside(language: ConditionLanguage, outside: LanguageOutside): boolean | undefined;
|
|
55
|
+
/**
|
|
56
|
+
* True when no subject is accepted by both languages. Both have to be
|
|
57
|
+
* `atLeast` languages.
|
|
58
|
+
*/
|
|
59
|
+
export declare function languagesDisjoint(left: ConditionLanguage, right: ConditionLanguage): boolean | undefined;
|