@speedkit/cli 4.26.0 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +186 -15
- package/dist/commands/auto-prewarm.d.ts +8 -1
- package/dist/commands/auto-prewarm.js +18 -7
- package/dist/commands/generate-pop-config.d.ts +1 -0
- package/dist/commands/generate-pop-config.js +9 -3
- package/dist/commands/prewarm.js +3 -1
- package/dist/commands/query/parameter.d.ts +26 -0
- package/dist/commands/query/parameter.js +27 -0
- package/dist/commands/query/prewarm.d.ts +31 -0
- package/dist/commands/query/prewarm.js +31 -0
- package/dist/helpers/clipboard.d.ts +19 -0
- package/dist/helpers/clipboard.js +70 -0
- package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
- package/dist/helpers/evaluate-speed-kit-config.js +8 -6
- package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
- package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
- package/dist/models/cli-parameters.d.ts +73 -0
- package/dist/models/cli-parameters.js +151 -0
- package/dist/models/cli-parameters.spec.d.ts +1 -0
- package/dist/models/cli-parameters.spec.js +47 -0
- package/dist/services/athena/athena-service.d.ts +2 -0
- package/dist/services/athena/athena-service.js +15 -4
- package/dist/services/athena/athena-service.spec.d.ts +1 -0
- package/dist/services/athena/athena-service.spec.js +74 -0
- package/dist/services/deploy/handler/install-resource-handler.js +3 -4
- package/dist/services/onboarding/onboarding-service-factory.js +2 -2
- package/dist/services/pop-config/pop-config-service.d.ts +8 -1
- package/dist/services/pop-config/pop-config-service.js +16 -28
- package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
- package/dist/services/prewarm/assets/asset-api-client.js +5 -4
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +12 -2
- package/dist/services/prewarm/auto-pre-warm-factory.js +20 -6
- package/dist/services/prewarm/csv-reader.d.ts +21 -2
- package/dist/services/prewarm/csv-reader.js +71 -9
- package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
- package/dist/services/prewarm/csv-reader.spec.js +75 -0
- package/dist/services/prewarm/index.d.ts +1 -0
- package/dist/services/prewarm/index.js +1 -0
- package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
- package/dist/services/prewarm/pre-warm-factory.js +8 -11
- package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
- package/dist/services/prewarm/pre-warm-model.js +0 -1
- package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
- package/dist/services/prewarm/pre-warm-service.js +18 -14
- package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
- package/dist/services/prewarm/prewarm-targets.js +18 -0
- package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
- package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
- package/dist/services/query-builder/queries/device.d.ts +21 -0
- package/dist/services/query-builder/queries/device.js +26 -0
- package/dist/services/query-builder/queries/index.d.ts +6 -0
- package/dist/services/query-builder/queries/index.js +6 -0
- package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
- package/dist/services/query-builder/queries/page-filter.js +135 -0
- package/dist/services/query-builder/queries/page-source.d.ts +30 -0
- package/dist/services/query-builder/queries/page-source.js +79 -0
- package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
- package/dist/services/query-builder/queries/parameter-query.js +142 -0
- package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
- package/dist/services/query-builder/queries/pops-query.js +65 -0
- package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
- package/dist/services/query-builder/queries/prewarm-query.js +227 -0
- package/dist/services/query-builder/queries/query-model.d.ts +97 -0
- package/dist/services/query-builder/queries/query-model.js +19 -0
- package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
- package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
- package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
- package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
- package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
- package/dist/services/query-builder/query-builder-factory.js +35 -7
- package/dist/services/query-builder/query-builder-model.d.ts +31 -19
- package/dist/services/query-builder/query-builder-model.js +5 -5
- package/dist/services/query-builder/query-builder-service.d.ts +26 -5
- package/dist/services/query-builder/query-builder-service.js +157 -44
- package/dist/services/query-builder/query-command.d.ts +26 -0
- package/dist/services/query-builder/query-command.js +35 -0
- package/dist/services/query-builder/rules/index.d.ts +8 -0
- package/dist/services/query-builder/rules/index.js +8 -0
- package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
- package/dist/services/query-builder/rules/match-pattern.js +141 -0
- package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
- package/dist/services/query-builder/rules/pattern-language.js +159 -0
- package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
- package/dist/services/query-builder/rules/rule-model.js +54 -0
- package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
- package/dist/services/query-builder/rules/rule-parser.js +189 -0
- package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
- package/dist/services/query-builder/rules/rule-reducer.js +383 -0
- package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
- package/dist/services/query-builder/rules/rule-relevance.js +69 -0
- package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
- package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
- package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
- package/dist/services/query-builder/rules/strip-parameters.js +197 -0
- package/dist/services/query-builder/sql/expression.d.ts +73 -0
- package/dist/services/query-builder/sql/expression.js +133 -0
- package/dist/services/query-builder/sql/format.d.ts +14 -0
- package/dist/services/query-builder/sql/format.js +48 -0
- package/dist/services/query-builder/sql/index.d.ts +4 -0
- package/dist/services/query-builder/sql/index.js +4 -0
- package/dist/services/query-builder/sql/predicate.d.ts +66 -0
- package/dist/services/query-builder/sql/predicate.js +124 -0
- package/dist/services/query-builder/sql/regex.d.ts +28 -0
- package/dist/services/query-builder/sql/regex.js +81 -0
- package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
- package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
- package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
- package/oclif.manifest.json +350 -100
- package/package.json +11 -2
- package/dist/commands/build-parameter-query.d.ts +0 -14
- package/dist/commands/build-parameter-query.js +0 -25
- package/dist/commands/build-prewarm-query.d.ts +0 -16
- package/dist/commands/build-prewarm-query.js +0 -37
- package/dist/helpers/build-query-helper.d.ts +0 -32
- package/dist/helpers/build-query-helper.js +0 -223
- package/dist/helpers/get-parsed-config.d.ts +0 -15
- package/dist/helpers/get-parsed-config.js +0 -62
- package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
- package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { describe, it } from "mocha";
|
|
3
|
+
import { parseRuleSets } from "../../rules/index.js";
|
|
4
|
+
import { buildParameterQuery } from "../parameter-query.js";
|
|
5
|
+
import { buildPrewarmQuery } from "../prewarm-query.js";
|
|
6
|
+
/**
|
|
7
|
+
* The bugs that prompted the rework, each with the config that reported it.
|
|
8
|
+
*
|
|
9
|
+
* A golden query shows the whole output; these name the defect, so a
|
|
10
|
+
* regression fails with the ticket it belongs to rather than as an
|
|
11
|
+
* unexplained diff.
|
|
12
|
+
*/
|
|
13
|
+
function inputFor(config, hosts) {
|
|
14
|
+
return {
|
|
15
|
+
app: "shop",
|
|
16
|
+
days: 30,
|
|
17
|
+
hosts,
|
|
18
|
+
now: new Date("2026-09-04T09:30:00Z"),
|
|
19
|
+
ruleSets: parseRuleSets(config),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
describe("ED-277: a query from whitelisted pathnames", () => {
|
|
23
|
+
// The whitelist carried the page scope, and only its host was ever read, so
|
|
24
|
+
// the pathnames never reached the query.
|
|
25
|
+
const config = {
|
|
26
|
+
whitelist: [
|
|
27
|
+
{
|
|
28
|
+
contentType: ["navigate"],
|
|
29
|
+
host: ["www.hugendubel.de"],
|
|
30
|
+
pathname: [/\/de$/, /\/category/, /\.html/],
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
};
|
|
34
|
+
const { sql } = buildPrewarmQuery(inputFor(config, ["www.hugendubel.de"]));
|
|
35
|
+
it("should filter on the whitelisted host", () => {
|
|
36
|
+
expect(sql).to.contain("lower(host) like 'www.hugendubel.de%'");
|
|
37
|
+
});
|
|
38
|
+
it("should filter on every whitelisted pathname", () => {
|
|
39
|
+
expect(sql).to.contain(String.raw `regexp_like(pathWithSearch, '/de$')`);
|
|
40
|
+
expect(sql).to.contain(String.raw `regexp_like(pathWithSearch, '/category')`);
|
|
41
|
+
expect(sql).to.contain(String.raw `regexp_like(pathWithSearch, '\.html')`);
|
|
42
|
+
});
|
|
43
|
+
it("should or the pathnames and and them with the host", () => {
|
|
44
|
+
const clause = sql.slice(sql.indexOf(" where lower(host)"));
|
|
45
|
+
expect(clause).to.match(/where lower\(host\) like[^\n]*\n {4}and \(/);
|
|
46
|
+
expect(clause).to.contain("\n or regexp_like(pathWithSearch");
|
|
47
|
+
});
|
|
48
|
+
it("should match the pathname the way the rule set version does", () => {
|
|
49
|
+
// No ruleSetConfig means version 1, where the runtime tests a pathname
|
|
50
|
+
// against `pathname + search` — not against the path alone.
|
|
51
|
+
expect(sql).to.not.contain("regexp_like(path,");
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
describe("ED-408: a pathname-scoped stripQueryParams rule", () => {
|
|
55
|
+
// `params: /.*/` was applied to every key of every page, so no key ever
|
|
56
|
+
// survived the filter and the analysis came back empty.
|
|
57
|
+
const config = {
|
|
58
|
+
ruleSetConfig: { version: 2 },
|
|
59
|
+
stripQueryParams: [
|
|
60
|
+
{ params: /.*/, rules: [{ pathname: "/hoteldetail/" }] },
|
|
61
|
+
{ params: ["_ga"] },
|
|
62
|
+
],
|
|
63
|
+
whitelist: [{ host: ["www.alltours.de"] }],
|
|
64
|
+
};
|
|
65
|
+
const { sql } = buildParameterQuery(inputFor(config, ["www.alltours.de"]));
|
|
66
|
+
it("should reject a key only on the scoped pathname", () => {
|
|
67
|
+
expect(sql).to.contain([
|
|
68
|
+
"(key) -> not (",
|
|
69
|
+
" lower(path) = '/hoteldetail/'",
|
|
70
|
+
" and regexp_like(key, '.*')",
|
|
71
|
+
" )",
|
|
72
|
+
].join("\n"));
|
|
73
|
+
});
|
|
74
|
+
it("should keep rejecting the unscoped parameters everywhere", () => {
|
|
75
|
+
expect(sql).to.contain("and not (lower(key) = '_ga')");
|
|
76
|
+
});
|
|
77
|
+
it("should scope the cache key the same way", () => {
|
|
78
|
+
// The cache key is built from the parameters that survive the very same
|
|
79
|
+
// filter, so the scope cannot apply to one and not the other.
|
|
80
|
+
expect(sql).to.contain("filter(pairs, (pair) -> contains(keptKeys, split(pair, '=')[1]))");
|
|
81
|
+
expect(sql).to.contain("path || '?' || array_join(pairs, '&') as cacheKey");
|
|
82
|
+
});
|
|
83
|
+
it("should never strip every parameter unconditionally", () => {
|
|
84
|
+
expect(sql).to.not.contain("(key) -> not regexp_like(key, '.*')");
|
|
85
|
+
});
|
|
86
|
+
});
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { QueryBuilderService } from "./query-builder-service.js";
|
|
2
|
-
import { QueryBuilderFactoryContext,
|
|
2
|
+
import { QueryBuilderFactoryContext, QueryType } from "./query-builder-model.js";
|
|
3
3
|
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
4
4
|
export declare class QueryBuilderFactory {
|
|
5
5
|
private context;
|
|
6
6
|
private userConfig;
|
|
7
7
|
constructor(context: QueryBuilderFactoryContext, userConfig: UserCliConfig);
|
|
8
|
-
buildService(queryType:
|
|
8
|
+
buildService(queryType: QueryType): Promise<QueryBuilderService>;
|
|
9
9
|
private getIntegrationFiles;
|
|
10
10
|
private getSpeedKitConfig;
|
|
11
11
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { QueryBuilderService } from "./query-builder-service.js";
|
|
2
|
+
import { parseRuleSets } from "./rules/index.js";
|
|
2
3
|
import { IntegrationApiContext, IntegrationApiFactory, } from "../integration-api/index.js";
|
|
3
4
|
import { INTEGRATION_FILES } from "../../models/files.js";
|
|
4
5
|
import { CliServiceFactory } from "../cli/index.js";
|
|
@@ -18,21 +19,37 @@ export class QueryBuilderFactory {
|
|
|
18
19
|
const files = await this.getIntegrationFiles();
|
|
19
20
|
const customerConfig = files.getCustomerConfig().config;
|
|
20
21
|
const app = customerConfig.app;
|
|
21
|
-
const
|
|
22
|
+
const origins = customerConfig.origins ?? [];
|
|
22
23
|
const speedKitConfigFile = files.getByName(INTEGRATION_FILES.CONFIG.SPEED_KIT);
|
|
23
|
-
const speedKitConfig = await this.getSpeedKitConfig(app,
|
|
24
|
+
const speedKitConfig = await this.getSpeedKitConfig(app, origins[0], speedKitConfigFile);
|
|
24
25
|
const serviceContext = {
|
|
25
26
|
app,
|
|
26
|
-
|
|
27
|
+
bySet: this.context.bySet,
|
|
28
|
+
count: this.context.count,
|
|
29
|
+
days: this.context.days,
|
|
30
|
+
devices: this.context.devices,
|
|
31
|
+
excludeNotFound: this.context.excludeNotFound,
|
|
32
|
+
excludeNoindex: this.context.excludeNoindex,
|
|
33
|
+
excludeSuspicious: this.context.excludeSuspicious,
|
|
34
|
+
execute: this.context.execute,
|
|
35
|
+
explain: this.context.explain,
|
|
36
|
+
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
37
|
+
hosts: toHosts(origins),
|
|
27
38
|
isWindows: this.context.isWindows,
|
|
39
|
+
limit: this.context.limit,
|
|
40
|
+
match: this.context.match,
|
|
41
|
+
minHits: this.context.minHits,
|
|
42
|
+
minImpressions: this.context.minImpressions,
|
|
43
|
+
queryType,
|
|
28
44
|
quiet: this.context.quiet,
|
|
29
|
-
|
|
45
|
+
variations: this.context.variations,
|
|
30
46
|
};
|
|
31
|
-
// The Athena service is only needed when the query is executed directly
|
|
32
|
-
|
|
47
|
+
// The Athena service is only needed when the query is executed directly —
|
|
48
|
+
// which `--count` is, a count query being of no use as text.
|
|
49
|
+
const athenaService = this.context.execute || this.context.count
|
|
33
50
|
? await new AthenaServiceFactory(app).getService()
|
|
34
51
|
: undefined;
|
|
35
|
-
return new QueryBuilderService(serviceContext, speedKitConfig, cli, athenaService);
|
|
52
|
+
return new QueryBuilderService(serviceContext, parseRuleSets(speedKitConfig), cli, athenaService);
|
|
36
53
|
}
|
|
37
54
|
async getIntegrationFiles() {
|
|
38
55
|
const integrationApiContext = new IntegrationApiContext(this.context.customerPath, this.context.configName, ["config_SpeedKit"]);
|
|
@@ -49,3 +66,14 @@ export class QueryBuilderFactory {
|
|
|
49
66
|
return result.data;
|
|
50
67
|
}
|
|
51
68
|
}
|
|
69
|
+
/** The host of every origin, ignoring the ones that are not a URL. */
|
|
70
|
+
function toHosts(origins) {
|
|
71
|
+
const hosts = [];
|
|
72
|
+
for (const origin of origins) {
|
|
73
|
+
const result = safe(() => new URL(origin).host);
|
|
74
|
+
if (result.success === true) {
|
|
75
|
+
hosts.push(result.data);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return hosts;
|
|
79
|
+
}
|
|
@@ -1,24 +1,36 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
queryType: QueryTpe;
|
|
6
|
-
app: string;
|
|
1
|
+
import { ParameterOptions, PrewarmOptions } from "./queries/query-model.js";
|
|
2
|
+
export declare enum QueryType {
|
|
3
|
+
parameter = "parameter",
|
|
4
|
+
prewarm = "prewarm"
|
|
7
5
|
}
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
/** Options that narrow the rows a query looks at. */
|
|
7
|
+
export interface QueryFilterOptions {
|
|
8
|
+
/** Size of the window the query looks at, in days. */
|
|
9
|
+
readonly days?: number;
|
|
10
|
+
/** Drop page impressions of pages marked `noindex`. */
|
|
11
|
+
readonly excludeNoindex?: boolean;
|
|
12
|
+
/** Drop page impressions Speed Kit classified as suspicious. */
|
|
13
|
+
readonly excludeSuspicious?: boolean;
|
|
14
|
+
/** Keep hard navigations only, dropping soft (SPA) navigations. */
|
|
15
|
+
readonly hardNavigationsOnly?: boolean;
|
|
16
|
+
}
|
|
17
|
+
export interface QueryBuilderContext extends ParameterOptions, PrewarmOptions, QueryFilterOptions {
|
|
18
|
+
readonly app: string;
|
|
19
|
+
readonly execute?: boolean;
|
|
20
|
+
/** Report every rule the query leaves out, reduces or approximates. */
|
|
21
|
+
readonly explain?: boolean;
|
|
22
|
+
/** Hosts of the app's origins. */
|
|
23
|
+
readonly hosts?: readonly string[];
|
|
11
24
|
readonly isWindows?: boolean;
|
|
25
|
+
readonly queryType: QueryType;
|
|
12
26
|
readonly quiet?: boolean;
|
|
13
|
-
readonly execute?: boolean;
|
|
14
|
-
}
|
|
15
|
-
export declare enum QueryTpe {
|
|
16
|
-
prewarm = "prewarm",
|
|
17
|
-
parameter = "parameter"
|
|
18
27
|
}
|
|
19
|
-
export interface
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
28
|
+
export interface QueryBuilderFactoryContext extends ParameterOptions, PrewarmOptions, QueryFilterOptions {
|
|
29
|
+
readonly configName: string;
|
|
30
|
+
readonly customerPath: string;
|
|
31
|
+
readonly execute?: boolean;
|
|
32
|
+
/** Report every rule the query leaves out, reduces or approximates. */
|
|
33
|
+
readonly explain?: boolean;
|
|
34
|
+
readonly isWindows?: boolean;
|
|
35
|
+
readonly quiet?: boolean;
|
|
24
36
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export var
|
|
2
|
-
(function (
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
})(
|
|
1
|
+
export var QueryType;
|
|
2
|
+
(function (QueryType) {
|
|
3
|
+
QueryType["parameter"] = "parameter";
|
|
4
|
+
QueryType["prewarm"] = "prewarm";
|
|
5
|
+
})(QueryType || (QueryType = {}));
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { QueryBuilderContext
|
|
1
|
+
import { QueryBuilderContext } from "./query-builder-model.js";
|
|
2
|
+
import { ParsedRuleSets } from "./rules/index.js";
|
|
2
3
|
import { CliServiceInterface } from "../cli/index.js";
|
|
3
4
|
import { AthenaService } from "../athena/index.js";
|
|
4
5
|
export declare class QueryBuilderService {
|
|
5
6
|
private context;
|
|
6
|
-
private
|
|
7
|
+
private ruleSets;
|
|
7
8
|
private cli;
|
|
8
9
|
private athenaService?;
|
|
9
|
-
constructor(context: QueryBuilderContext,
|
|
10
|
+
constructor(context: QueryBuilderContext, ruleSets: ParsedRuleSets, cli: CliServiceInterface, athenaService?: AthenaService);
|
|
10
11
|
/** The app the query targets, as read from the customer config. */
|
|
11
12
|
get app(): string;
|
|
12
13
|
run(): Promise<void>;
|
|
@@ -16,13 +17,33 @@ export declare class QueryBuilderService {
|
|
|
16
17
|
* directly instead of exporting it (e.g. `auto-prewarm`).
|
|
17
18
|
*/
|
|
18
19
|
execute<T>(): Promise<T[]>;
|
|
20
|
+
/** The query as SQL. */
|
|
21
|
+
query(): string;
|
|
22
|
+
private build;
|
|
23
|
+
/**
|
|
24
|
+
* Points out where the query is not exactly what the config says — a rule
|
|
25
|
+
* left out, a rule the query can only approximate, a pattern Athena reads
|
|
26
|
+
* differently than the browser — plus what the config itself gets wrong.
|
|
27
|
+
*
|
|
28
|
+
* Reductions are silent unless asked for: a rule another rule already
|
|
29
|
+
* decides is not a divergence, it is arithmetic.
|
|
30
|
+
*/
|
|
31
|
+
private report;
|
|
32
|
+
/** The same accounting as {@link report}, as plain lines. */
|
|
33
|
+
private reportLines;
|
|
19
34
|
private runQuery;
|
|
35
|
+
/**
|
|
36
|
+
* Execute the count query and put its handful of rows on the terminal — a
|
|
37
|
+
* count is read, not filed, so nothing is written to disk.
|
|
38
|
+
*
|
|
39
|
+
* Under `--quiet` the rows go to stdout as tab separated values, so a count
|
|
40
|
+
* can be read by something other than a person.
|
|
41
|
+
*/
|
|
42
|
+
private executeAndReportCounts;
|
|
20
43
|
/**
|
|
21
44
|
* Execute the built query against Athena (Baqend native path, AWS fallback)
|
|
22
45
|
* and write the rows to a `<app>-<YYYY-MM-DD>-<queryType>.csv` file in the
|
|
23
46
|
* current working directory.
|
|
24
47
|
*/
|
|
25
48
|
private executeAndExport;
|
|
26
|
-
query(): string;
|
|
27
|
-
private buildQuery;
|
|
28
49
|
}
|
|
@@ -1,22 +1,37 @@
|
|
|
1
|
-
import { getSpeedKitConfigVersion } from "../../helpers/get-parsed-config.js";
|
|
2
|
-
import { getBlacklistedFilter, getParameterFilter, getParameterReplaceFilter, getStrippedParameters, getWhitelistedFilter, } from "../../helpers/build-query-helper.js";
|
|
3
1
|
import { writeFile } from "node:fs/promises";
|
|
4
2
|
import { join } from "node:path";
|
|
5
|
-
import clipboardy from "clipboardy";
|
|
6
3
|
import json2Csv from "json2csv";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { Parameter } from "./query/parameter.js";
|
|
4
|
+
import { QueryType } from "./query-builder-model.js";
|
|
5
|
+
import { buildParameterQuery, buildPrewarmQuery, DEFAULT_QUERY_DAYS, } from "./queries/index.js";
|
|
10
6
|
import { AthenaBucket } from "../athena/index.js";
|
|
7
|
+
import { copyToClipboard } from "../../helpers/clipboard.js";
|
|
11
8
|
import { safe } from "../../helpers/safe.js";
|
|
9
|
+
/**
|
|
10
|
+
* The notes as unique lines. Each phase of the reduction reports on its own,
|
|
11
|
+
* so the same note — the budget running out, above all — can arrive more than
|
|
12
|
+
* once.
|
|
13
|
+
*/
|
|
14
|
+
function distinct(notes) {
|
|
15
|
+
return [
|
|
16
|
+
...new Set(notes.map(({ reason, source }) => `Reduced ${source}: ${reason}`)),
|
|
17
|
+
];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* A count as it is read: grouped in thousands. Athena hands numbers over as
|
|
21
|
+
* strings, so anything that is not one is passed through untouched.
|
|
22
|
+
*/
|
|
23
|
+
function readable(value) {
|
|
24
|
+
const text = String(value ?? "");
|
|
25
|
+
return /^\d+$/.test(text) ? Number(text).toLocaleString("en-US") : text;
|
|
26
|
+
}
|
|
12
27
|
export class QueryBuilderService {
|
|
13
28
|
context;
|
|
14
|
-
|
|
29
|
+
ruleSets;
|
|
15
30
|
cli;
|
|
16
31
|
athenaService;
|
|
17
|
-
constructor(context,
|
|
32
|
+
constructor(context, ruleSets, cli, athenaService) {
|
|
18
33
|
this.context = context;
|
|
19
|
-
this.
|
|
34
|
+
this.ruleSets = ruleSets;
|
|
20
35
|
this.cli = cli;
|
|
21
36
|
this.athenaService = athenaService;
|
|
22
37
|
}
|
|
@@ -25,37 +40,49 @@ export class QueryBuilderService {
|
|
|
25
40
|
return this.context.app;
|
|
26
41
|
}
|
|
27
42
|
async run() {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
|
|
32
|
-
const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
|
|
33
|
-
const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
|
|
34
|
-
const parameterFilter = getParameterFilter(parameters, skConfigVersion);
|
|
35
|
-
const result = safe(() => this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter));
|
|
36
|
-
this.cli.successAction(`QUERY:BUILD:${this.context.queryType}`);
|
|
43
|
+
const action = `QUERY:BUILD:${this.context.queryType}`;
|
|
44
|
+
this.cli.startAction(action, `build ${this.context.queryType}-query`);
|
|
45
|
+
const result = safe(() => this.build());
|
|
37
46
|
if (result.success !== true) {
|
|
38
|
-
this.cli.failAction(
|
|
47
|
+
this.cli.failAction(action, result.error);
|
|
39
48
|
this.cli.writeError(result.error);
|
|
40
49
|
console.error(result.errorObj);
|
|
41
50
|
return;
|
|
42
51
|
}
|
|
43
|
-
this.cli.successAction(
|
|
52
|
+
this.cli.successAction(action);
|
|
53
|
+
const query = result.data;
|
|
54
|
+
if (this.context.count) {
|
|
55
|
+
if (this.context.explain) {
|
|
56
|
+
this.report(query);
|
|
57
|
+
}
|
|
58
|
+
await this.executeAndReportCounts(query.sql);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
44
61
|
if (this.context.execute) {
|
|
45
|
-
await this.executeAndExport(
|
|
62
|
+
await this.executeAndExport(query.sql);
|
|
46
63
|
return;
|
|
47
64
|
}
|
|
48
65
|
if (this.context.quiet) {
|
|
49
|
-
|
|
66
|
+
// The query goes to stdout so it can be piped; an accounting the caller
|
|
67
|
+
// asked for goes next to it, on stderr.
|
|
68
|
+
if (this.context.explain) {
|
|
69
|
+
process.stderr.write(`${this.reportLines(query).join("\n")}\n`);
|
|
70
|
+
}
|
|
71
|
+
process.stdout.write(query.sql);
|
|
50
72
|
return;
|
|
51
73
|
}
|
|
52
|
-
|
|
53
|
-
|
|
74
|
+
this.report(query);
|
|
75
|
+
// The query is the result, the clipboard is a convenience: write it out
|
|
76
|
+
// first so nothing about the clipboard can delay or hide it.
|
|
77
|
+
this.cli.write(this.cli.style.dim.italic(query.sql));
|
|
78
|
+
if (this.context.isWindows) {
|
|
79
|
+
return;
|
|
54
80
|
}
|
|
55
|
-
|
|
56
|
-
if (!this.context.isWindows) {
|
|
81
|
+
if (await copyToClipboard(query.sql)) {
|
|
57
82
|
this.cli.writeSuccess("Result has been copied to the clipboard 📋");
|
|
83
|
+
return;
|
|
58
84
|
}
|
|
85
|
+
this.cli.write(this.cli.style.dim("The clipboard could not be reached 📋"));
|
|
59
86
|
}
|
|
60
87
|
/**
|
|
61
88
|
* Build and execute the query against Athena (Baqend native path, AWS
|
|
@@ -65,12 +92,115 @@ export class QueryBuilderService {
|
|
|
65
92
|
async execute() {
|
|
66
93
|
return this.runQuery(this.query());
|
|
67
94
|
}
|
|
95
|
+
/** The query as SQL. */
|
|
96
|
+
query() {
|
|
97
|
+
return this.build().sql;
|
|
98
|
+
}
|
|
99
|
+
build() {
|
|
100
|
+
const input = {
|
|
101
|
+
app: this.context.app,
|
|
102
|
+
bySet: this.context.bySet,
|
|
103
|
+
count: this.context.count,
|
|
104
|
+
days: this.context.days ?? DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
105
|
+
devices: this.context.devices,
|
|
106
|
+
excludeNotFound: this.context.excludeNotFound,
|
|
107
|
+
excludeNoindex: this.context.excludeNoindex,
|
|
108
|
+
excludeSuspicious: this.context.excludeSuspicious,
|
|
109
|
+
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
110
|
+
hosts: this.context.hosts,
|
|
111
|
+
limit: this.context.limit,
|
|
112
|
+
match: this.context.match,
|
|
113
|
+
minHits: this.context.minHits,
|
|
114
|
+
minImpressions: this.context.minImpressions,
|
|
115
|
+
ruleSets: this.ruleSets,
|
|
116
|
+
variations: this.context.variations,
|
|
117
|
+
};
|
|
118
|
+
return this.context.queryType === QueryType.prewarm
|
|
119
|
+
? buildPrewarmQuery(input)
|
|
120
|
+
: buildParameterQuery(input);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Points out where the query is not exactly what the config says — a rule
|
|
124
|
+
* left out, a rule the query can only approximate, a pattern Athena reads
|
|
125
|
+
* differently than the browser — plus what the config itself gets wrong.
|
|
126
|
+
*
|
|
127
|
+
* Reductions are silent unless asked for: a rule another rule already
|
|
128
|
+
* decides is not a divergence, it is arithmetic.
|
|
129
|
+
*/
|
|
130
|
+
report(query) {
|
|
131
|
+
for (const { reason, source } of query.findings) {
|
|
132
|
+
this.cli.writeWarning(`${source} ${reason}`);
|
|
133
|
+
}
|
|
134
|
+
for (const { reason, source } of query.dropped) {
|
|
135
|
+
this.cli.writeWarning(`Left out ${source}: ${reason}`);
|
|
136
|
+
}
|
|
137
|
+
for (const { reason, source } of query.approximated) {
|
|
138
|
+
this.cli.writeWarning(`Approximated ${source}: ${reason}`);
|
|
139
|
+
}
|
|
140
|
+
for (const warning of query.warnings) {
|
|
141
|
+
this.cli.writeWarning(warning);
|
|
142
|
+
}
|
|
143
|
+
const reduced = distinct(query.reduced);
|
|
144
|
+
if (this.context.explain) {
|
|
145
|
+
for (const line of reduced) {
|
|
146
|
+
this.cli.write(this.cli.style.dim(line));
|
|
147
|
+
}
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (reduced.length > 0) {
|
|
151
|
+
this.cli.write(this.cli.style.dim(`${reduced.length} rule(s) reduced away — run with --explain to see them`));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** The same accounting as {@link report}, as plain lines. */
|
|
155
|
+
reportLines(query) {
|
|
156
|
+
return [
|
|
157
|
+
...query.findings.map(({ reason, source }) => `${source} ${reason}`),
|
|
158
|
+
...query.dropped.map(({ reason, source }) => `Left out ${source}: ${reason}`),
|
|
159
|
+
...query.approximated.map(({ reason, source }) => `Approximated ${source}: ${reason}`),
|
|
160
|
+
...query.warnings,
|
|
161
|
+
...distinct(query.reduced),
|
|
162
|
+
];
|
|
163
|
+
}
|
|
68
164
|
async runQuery(query) {
|
|
69
165
|
if (!this.athenaService) {
|
|
70
166
|
throw new Error("No Athena service available to execute the query");
|
|
71
167
|
}
|
|
72
168
|
return this.athenaService.getResult(query, [], undefined, AthenaBucket.RumPi);
|
|
73
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Execute the count query and put its handful of rows on the terminal — a
|
|
172
|
+
* count is read, not filed, so nothing is written to disk.
|
|
173
|
+
*
|
|
174
|
+
* Under `--quiet` the rows go to stdout as tab separated values, so a count
|
|
175
|
+
* can be read by something other than a person.
|
|
176
|
+
*/
|
|
177
|
+
async executeAndReportCounts(query) {
|
|
178
|
+
this.cli.startAction("QUERY:COUNT", "count the pre-warm list");
|
|
179
|
+
const result = await safe(this.runQuery(query));
|
|
180
|
+
if (result.success !== true) {
|
|
181
|
+
this.cli.failAction("QUERY:COUNT");
|
|
182
|
+
this.cli.writeError(result.error);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
this.cli.successAction("QUERY:COUNT");
|
|
186
|
+
const rows = result.data;
|
|
187
|
+
if (rows.length === 0) {
|
|
188
|
+
this.cli.writeWarning("No page impressions in the window — the list would be empty");
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
if (this.context.quiet) {
|
|
192
|
+
const columns = Object.keys(rows[0]);
|
|
193
|
+
process.stdout.write([
|
|
194
|
+
columns.join("\t"),
|
|
195
|
+
...rows.map((row) => columns.map((name) => row[name]).join("\t")),
|
|
196
|
+
].join("\n") + "\n");
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
this.cli.table(rows, Object.fromEntries(Object.keys(rows[0]).map((name) => [
|
|
200
|
+
name,
|
|
201
|
+
{ get: (row) => readable(row[name]) },
|
|
202
|
+
])));
|
|
203
|
+
}
|
|
74
204
|
/**
|
|
75
205
|
* Execute the built query against Athena (Baqend native path, AWS fallback)
|
|
76
206
|
* and write the rows to a `<app>-<YYYY-MM-DD>-<queryType>.csv` file in the
|
|
@@ -80,7 +210,7 @@ export class QueryBuilderService {
|
|
|
80
210
|
this.cli.startAction("QUERY:EXECUTE", "execute query and export csv");
|
|
81
211
|
const result = await safe(this.runQuery(query));
|
|
82
212
|
if (result.success !== true) {
|
|
83
|
-
this.cli.failAction("QUERY:EXECUTE"
|
|
213
|
+
this.cli.failAction("QUERY:EXECUTE");
|
|
84
214
|
this.cli.writeError(result.error);
|
|
85
215
|
return;
|
|
86
216
|
}
|
|
@@ -93,21 +223,4 @@ export class QueryBuilderService {
|
|
|
93
223
|
this.cli.successAction("QUERY:EXECUTE");
|
|
94
224
|
this.cli.writeSuccess(`Wrote ${rows.length} rows to ${filePath} 📄`);
|
|
95
225
|
}
|
|
96
|
-
query() {
|
|
97
|
-
const parameters = getStrippedParameters(this.speedKitConfig);
|
|
98
|
-
const parameterReplaceFilter = getParameterReplaceFilter(parameters, getSpeedKitConfigVersion(this.speedKitConfig));
|
|
99
|
-
const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
|
|
100
|
-
const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
|
|
101
|
-
const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
|
|
102
|
-
const parameterFilter = getParameterFilter(parameters, skConfigVersion);
|
|
103
|
-
return this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter);
|
|
104
|
-
}
|
|
105
|
-
buildQuery(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter) {
|
|
106
|
-
if (this.context.queryType === QueryTpe.prewarm) {
|
|
107
|
-
return new Prewarm(app, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
|
|
108
|
-
}
|
|
109
|
-
if (this.context.queryType === QueryTpe.parameter) {
|
|
110
|
-
return new Parameter(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
226
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { Config } from "@oclif/core";
|
|
2
|
+
import { ParameterOptions, PrewarmOptions } from "./queries/query-model.js";
|
|
3
|
+
import { QueryFilterOptions, QueryType } from "./query-builder-model.js";
|
|
4
|
+
/**
|
|
5
|
+
* The flags a query command parses; each takes the ones it offers.
|
|
6
|
+
*
|
|
7
|
+
* `--device` and `--variation` are named as the flags are, singular, and are
|
|
8
|
+
* the two the caller gives rather than the list the builder works on.
|
|
9
|
+
*/
|
|
10
|
+
export interface QueryCommandFlags extends ParameterOptions, Omit<PrewarmOptions, "devices" | "variations">, QueryFilterOptions {
|
|
11
|
+
readonly configName: string;
|
|
12
|
+
readonly device?: readonly string[];
|
|
13
|
+
readonly execute?: boolean;
|
|
14
|
+
readonly explain?: boolean;
|
|
15
|
+
readonly quiet?: boolean;
|
|
16
|
+
readonly variation?: readonly string[];
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Builds one of the config-derived queries and hands it over.
|
|
20
|
+
*
|
|
21
|
+
* Both `sk query` commands differ only in which query they ask for, so the
|
|
22
|
+
* wiring lives here rather than twice in the command layer — the two of them
|
|
23
|
+
* drifting apart is what left a diverged copy of the parameter query behind
|
|
24
|
+
* once already.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runQuery(queryType: QueryType, customerPath: string, flags: QueryCommandFlags, config: Config): Promise<void>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { CliConfig } from "../../helpers/cli-config.js";
|
|
2
|
+
import { QueryBuilderFactory } from "./query-builder-factory.js";
|
|
3
|
+
import { normalizeDevices } from "./queries/device.js";
|
|
4
|
+
/**
|
|
5
|
+
* Builds one of the config-derived queries and hands it over.
|
|
6
|
+
*
|
|
7
|
+
* Both `sk query` commands differ only in which query they ask for, so the
|
|
8
|
+
* wiring lives here rather than twice in the command layer — the two of them
|
|
9
|
+
* drifting apart is what left a diverged copy of the parameter query behind
|
|
10
|
+
* once already.
|
|
11
|
+
*/
|
|
12
|
+
export async function runQuery(queryType, customerPath, flags, config) {
|
|
13
|
+
const service = await new QueryBuilderFactory({
|
|
14
|
+
bySet: flags.bySet,
|
|
15
|
+
configName: flags.configName,
|
|
16
|
+
count: flags.count,
|
|
17
|
+
customerPath,
|
|
18
|
+
days: flags.days,
|
|
19
|
+
devices: normalizeDevices(flags.device),
|
|
20
|
+
excludeNotFound: flags.excludeNotFound,
|
|
21
|
+
excludeNoindex: flags.excludeNoindex,
|
|
22
|
+
excludeSuspicious: flags.excludeSuspicious,
|
|
23
|
+
execute: flags.execute,
|
|
24
|
+
explain: flags.explain,
|
|
25
|
+
hardNavigationsOnly: flags.hardNavigationsOnly,
|
|
26
|
+
isWindows: config.windows,
|
|
27
|
+
limit: flags.limit,
|
|
28
|
+
match: flags.match,
|
|
29
|
+
minHits: flags.minHits,
|
|
30
|
+
minImpressions: flags.minImpressions,
|
|
31
|
+
quiet: flags.quiet,
|
|
32
|
+
variations: flags.variation,
|
|
33
|
+
}, new CliConfig(config).load()).buildService(queryType);
|
|
34
|
+
await service.run();
|
|
35
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from "./match-pattern.js";
|
|
2
|
+
export * from "./pattern-language.js";
|
|
3
|
+
export * from "./rule-model.js";
|
|
4
|
+
export * from "./rule-parser.js";
|
|
5
|
+
export * from "./rule-reducer.js";
|
|
6
|
+
export * from "./rule-relevance.js";
|
|
7
|
+
export * from "./rule-to-sql.js";
|
|
8
|
+
export * from "./strip-parameters.js";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from "./match-pattern.js";
|
|
2
|
+
export * from "./pattern-language.js";
|
|
3
|
+
export * from "./rule-model.js";
|
|
4
|
+
export * from "./rule-parser.js";
|
|
5
|
+
export * from "./rule-reducer.js";
|
|
6
|
+
export * from "./rule-relevance.js";
|
|
7
|
+
export * from "./rule-to-sql.js";
|
|
8
|
+
export * from "./strip-parameters.js";
|