@speedkit/cli 4.26.0 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +203 -15
  3. package/dist/commands/auto-prewarm.d.ts +12 -1
  4. package/dist/commands/auto-prewarm.js +26 -7
  5. package/dist/commands/generate-pop-config.d.ts +1 -0
  6. package/dist/commands/generate-pop-config.js +9 -3
  7. package/dist/commands/prewarm.js +3 -1
  8. package/dist/commands/query/parameter.d.ts +27 -0
  9. package/dist/commands/query/parameter.js +28 -0
  10. package/dist/commands/query/prewarm.d.ts +32 -0
  11. package/dist/commands/query/prewarm.js +32 -0
  12. package/dist/helpers/clipboard.d.ts +19 -0
  13. package/dist/helpers/clipboard.js +70 -0
  14. package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
  15. package/dist/helpers/evaluate-speed-kit-config.js +8 -6
  16. package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
  17. package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
  18. package/dist/models/cli-parameters.d.ts +106 -0
  19. package/dist/models/cli-parameters.js +183 -0
  20. package/dist/models/cli-parameters.spec.d.ts +1 -0
  21. package/dist/models/cli-parameters.spec.js +47 -0
  22. package/dist/services/athena/athena-service-factory.d.ts +4 -1
  23. package/dist/services/athena/athena-service-factory.js +9 -3
  24. package/dist/services/athena/athena-service.d.ts +13 -1
  25. package/dist/services/athena/athena-service.js +27 -5
  26. package/dist/services/athena/athena-service.spec.d.ts +1 -0
  27. package/dist/services/athena/athena-service.spec.js +93 -0
  28. package/dist/services/deploy/handler/install-resource-handler.js +3 -4
  29. package/dist/services/onboarding/onboarding-service-factory.js +2 -2
  30. package/dist/services/pop-config/pop-config-service.d.ts +8 -1
  31. package/dist/services/pop-config/pop-config-service.js +16 -28
  32. package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
  33. package/dist/services/prewarm/assets/asset-api-client.js +5 -4
  34. package/dist/services/prewarm/auto-pre-warm-factory.d.ts +20 -2
  35. package/dist/services/prewarm/auto-pre-warm-factory.js +24 -6
  36. package/dist/services/prewarm/csv-reader.d.ts +21 -2
  37. package/dist/services/prewarm/csv-reader.js +71 -9
  38. package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
  39. package/dist/services/prewarm/csv-reader.spec.js +75 -0
  40. package/dist/services/prewarm/index.d.ts +1 -0
  41. package/dist/services/prewarm/index.js +1 -0
  42. package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
  43. package/dist/services/prewarm/pre-warm-factory.js +8 -11
  44. package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
  45. package/dist/services/prewarm/pre-warm-model.js +0 -1
  46. package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
  47. package/dist/services/prewarm/pre-warm-service.js +18 -14
  48. package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
  49. package/dist/services/prewarm/prewarm-targets.js +18 -0
  50. package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
  51. package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
  52. package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
  53. package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
  54. package/dist/services/query-builder/queries/device.d.ts +21 -0
  55. package/dist/services/query-builder/queries/device.js +26 -0
  56. package/dist/services/query-builder/queries/index.d.ts +6 -0
  57. package/dist/services/query-builder/queries/index.js +6 -0
  58. package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
  59. package/dist/services/query-builder/queries/page-filter.js +135 -0
  60. package/dist/services/query-builder/queries/page-source.d.ts +30 -0
  61. package/dist/services/query-builder/queries/page-source.js +79 -0
  62. package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
  63. package/dist/services/query-builder/queries/parameter-query.js +142 -0
  64. package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
  65. package/dist/services/query-builder/queries/pops-query.js +65 -0
  66. package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
  67. package/dist/services/query-builder/queries/prewarm-query.js +227 -0
  68. package/dist/services/query-builder/queries/query-model.d.ts +97 -0
  69. package/dist/services/query-builder/queries/query-model.js +19 -0
  70. package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
  71. package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
  72. package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
  73. package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
  74. package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
  75. package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
  76. package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
  77. package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
  78. package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
  79. package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
  80. package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
  81. package/dist/services/query-builder/query-builder-factory.js +36 -8
  82. package/dist/services/query-builder/query-builder-model.d.ts +33 -19
  83. package/dist/services/query-builder/query-builder-model.js +5 -5
  84. package/dist/services/query-builder/query-builder-service.d.ts +26 -5
  85. package/dist/services/query-builder/query-builder-service.js +157 -44
  86. package/dist/services/query-builder/query-command.d.ts +27 -0
  87. package/dist/services/query-builder/query-command.js +36 -0
  88. package/dist/services/query-builder/rules/index.d.ts +8 -0
  89. package/dist/services/query-builder/rules/index.js +8 -0
  90. package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
  91. package/dist/services/query-builder/rules/match-pattern.js +141 -0
  92. package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
  93. package/dist/services/query-builder/rules/pattern-language.js +159 -0
  94. package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
  95. package/dist/services/query-builder/rules/rule-model.js +54 -0
  96. package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
  97. package/dist/services/query-builder/rules/rule-parser.js +189 -0
  98. package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
  99. package/dist/services/query-builder/rules/rule-reducer.js +383 -0
  100. package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
  101. package/dist/services/query-builder/rules/rule-relevance.js +69 -0
  102. package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
  103. package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
  104. package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
  105. package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
  106. package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
  107. package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
  108. package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
  109. package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
  110. package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
  111. package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
  112. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
  113. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
  114. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
  115. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
  116. package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
  117. package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
  118. package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
  119. package/dist/services/query-builder/rules/strip-parameters.js +197 -0
  120. package/dist/services/query-builder/sql/expression.d.ts +73 -0
  121. package/dist/services/query-builder/sql/expression.js +133 -0
  122. package/dist/services/query-builder/sql/format.d.ts +14 -0
  123. package/dist/services/query-builder/sql/format.js +48 -0
  124. package/dist/services/query-builder/sql/index.d.ts +4 -0
  125. package/dist/services/query-builder/sql/index.js +4 -0
  126. package/dist/services/query-builder/sql/predicate.d.ts +66 -0
  127. package/dist/services/query-builder/sql/predicate.js +124 -0
  128. package/dist/services/query-builder/sql/regex.d.ts +28 -0
  129. package/dist/services/query-builder/sql/regex.js +81 -0
  130. package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
  131. package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
  132. package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
  133. package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
  134. package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
  135. package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
  136. package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
  137. package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
  138. package/oclif.manifest.json +401 -100
  139. package/package.json +11 -2
  140. package/dist/commands/build-parameter-query.d.ts +0 -14
  141. package/dist/commands/build-parameter-query.js +0 -25
  142. package/dist/commands/build-prewarm-query.d.ts +0 -16
  143. package/dist/commands/build-prewarm-query.js +0 -37
  144. package/dist/helpers/build-query-helper.d.ts +0 -32
  145. package/dist/helpers/build-query-helper.js +0 -223
  146. package/dist/helpers/get-parsed-config.d.ts +0 -15
  147. package/dist/helpers/get-parsed-config.js +0 -62
  148. package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
  149. package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
@@ -1,22 +1,37 @@
1
- import { getSpeedKitConfigVersion } from "../../helpers/get-parsed-config.js";
2
- import { getBlacklistedFilter, getParameterFilter, getParameterReplaceFilter, getStrippedParameters, getWhitelistedFilter, } from "../../helpers/build-query-helper.js";
3
1
  import { writeFile } from "node:fs/promises";
4
2
  import { join } from "node:path";
5
- import clipboardy from "clipboardy";
6
3
  import json2Csv from "json2csv";
7
- import { QueryTpe, } from "./query-builder-model.js";
8
- import { Prewarm } from "./query/prewarm.js";
9
- import { Parameter } from "./query/parameter.js";
4
+ import { QueryType } from "./query-builder-model.js";
5
+ import { buildParameterQuery, buildPrewarmQuery, DEFAULT_QUERY_DAYS, } from "./queries/index.js";
10
6
  import { AthenaBucket } from "../athena/index.js";
7
+ import { copyToClipboard } from "../../helpers/clipboard.js";
11
8
  import { safe } from "../../helpers/safe.js";
9
+ /**
10
+ * The notes as unique lines. Each phase of the reduction reports on its own,
11
+ * so the same note — the budget running out, above all — can arrive more than
12
+ * once.
13
+ */
14
+ function distinct(notes) {
15
+ return [
16
+ ...new Set(notes.map(({ reason, source }) => `Reduced ${source}: ${reason}`)),
17
+ ];
18
+ }
19
+ /**
20
+ * A count as it is read: grouped in thousands. Athena hands numbers over as
21
+ * strings, so anything that is not one is passed through untouched.
22
+ */
23
+ function readable(value) {
24
+ const text = String(value ?? "");
25
+ return /^\d+$/.test(text) ? Number(text).toLocaleString("en-US") : text;
26
+ }
12
27
  export class QueryBuilderService {
13
28
  context;
14
- speedKitConfig;
29
+ ruleSets;
15
30
  cli;
16
31
  athenaService;
17
- constructor(context, speedKitConfig, cli, athenaService) {
32
+ constructor(context, ruleSets, cli, athenaService) {
18
33
  this.context = context;
19
- this.speedKitConfig = speedKitConfig;
34
+ this.ruleSets = ruleSets;
20
35
  this.cli = cli;
21
36
  this.athenaService = athenaService;
22
37
  }
@@ -25,37 +40,49 @@ export class QueryBuilderService {
25
40
  return this.context.app;
26
41
  }
27
42
  async run() {
28
- this.cli.startAction(`QUERY:BUILD:${this.context.queryType}`, `build ${this.context.queryType}-query`);
29
- const parameters = getStrippedParameters(this.speedKitConfig);
30
- const parameterReplaceFilter = getParameterReplaceFilter(parameters, getSpeedKitConfigVersion(this.speedKitConfig));
31
- const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
32
- const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
33
- const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
34
- const parameterFilter = getParameterFilter(parameters, skConfigVersion);
35
- const result = safe(() => this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter));
36
- this.cli.successAction(`QUERY:BUILD:${this.context.queryType}`);
43
+ const action = `QUERY:BUILD:${this.context.queryType}`;
44
+ this.cli.startAction(action, `build ${this.context.queryType}-query`);
45
+ const result = safe(() => this.build());
37
46
  if (result.success !== true) {
38
- this.cli.failAction(`QUERY:BUILD:${this.context.queryType}`, result.error);
47
+ this.cli.failAction(action, result.error);
39
48
  this.cli.writeError(result.error);
40
49
  console.error(result.errorObj);
41
50
  return;
42
51
  }
43
- this.cli.successAction(`QUERY:BUILD:${this.context.queryType}`);
52
+ this.cli.successAction(action);
53
+ const query = result.data;
54
+ if (this.context.count) {
55
+ if (this.context.explain) {
56
+ this.report(query);
57
+ }
58
+ await this.executeAndReportCounts(query.sql);
59
+ return;
60
+ }
44
61
  if (this.context.execute) {
45
- await this.executeAndExport(result.data);
62
+ await this.executeAndExport(query.sql);
46
63
  return;
47
64
  }
48
65
  if (this.context.quiet) {
49
- process.stdout.write(result.data);
66
+ // The query goes to stdout so it can be piped; an accounting the caller
67
+ // asked for goes next to it, on stderr.
68
+ if (this.context.explain) {
69
+ process.stderr.write(`${this.reportLines(query).join("\n")}\n`);
70
+ }
71
+ process.stdout.write(query.sql);
50
72
  return;
51
73
  }
52
- if (!this.context.isWindows) {
53
- clipboardy.writeSync(result.data);
74
+ this.report(query);
75
+ // The query is the result, the clipboard is a convenience: write it out
76
+ // first so nothing about the clipboard can delay or hide it.
77
+ this.cli.write(this.cli.style.dim.italic(query.sql));
78
+ if (this.context.isWindows) {
79
+ return;
54
80
  }
55
- this.cli.write(this.cli.style.dim.italic(result.data));
56
- if (!this.context.isWindows) {
81
+ if (await copyToClipboard(query.sql)) {
57
82
  this.cli.writeSuccess("Result has been copied to the clipboard 📋");
83
+ return;
58
84
  }
85
+ this.cli.write(this.cli.style.dim("The clipboard could not be reached 📋"));
59
86
  }
60
87
  /**
61
88
  * Build and execute the query against Athena (Baqend native path, AWS
@@ -65,12 +92,115 @@ export class QueryBuilderService {
65
92
  async execute() {
66
93
  return this.runQuery(this.query());
67
94
  }
95
+ /** The query as SQL. */
96
+ query() {
97
+ return this.build().sql;
98
+ }
99
+ build() {
100
+ const input = {
101
+ app: this.context.app,
102
+ bySet: this.context.bySet,
103
+ count: this.context.count,
104
+ days: this.context.days ?? DEFAULT_QUERY_DAYS[this.context.queryType],
105
+ devices: this.context.devices,
106
+ excludeNotFound: this.context.excludeNotFound,
107
+ excludeNoindex: this.context.excludeNoindex,
108
+ excludeSuspicious: this.context.excludeSuspicious,
109
+ hardNavigationsOnly: this.context.hardNavigationsOnly,
110
+ hosts: this.context.hosts,
111
+ limit: this.context.limit,
112
+ match: this.context.match,
113
+ minHits: this.context.minHits,
114
+ minImpressions: this.context.minImpressions,
115
+ ruleSets: this.ruleSets,
116
+ variations: this.context.variations,
117
+ };
118
+ return this.context.queryType === QueryType.prewarm
119
+ ? buildPrewarmQuery(input)
120
+ : buildParameterQuery(input);
121
+ }
122
+ /**
123
+ * Points out where the query is not exactly what the config says — a rule
124
+ * left out, a rule the query can only approximate, a pattern Athena reads
125
+ * differently than the browser — plus what the config itself gets wrong.
126
+ *
127
+ * Reductions are silent unless asked for: a rule another rule already
128
+ * decides is not a divergence, it is arithmetic.
129
+ */
130
+ report(query) {
131
+ for (const { reason, source } of query.findings) {
132
+ this.cli.writeWarning(`${source} ${reason}`);
133
+ }
134
+ for (const { reason, source } of query.dropped) {
135
+ this.cli.writeWarning(`Left out ${source}: ${reason}`);
136
+ }
137
+ for (const { reason, source } of query.approximated) {
138
+ this.cli.writeWarning(`Approximated ${source}: ${reason}`);
139
+ }
140
+ for (const warning of query.warnings) {
141
+ this.cli.writeWarning(warning);
142
+ }
143
+ const reduced = distinct(query.reduced);
144
+ if (this.context.explain) {
145
+ for (const line of reduced) {
146
+ this.cli.write(this.cli.style.dim(line));
147
+ }
148
+ return;
149
+ }
150
+ if (reduced.length > 0) {
151
+ this.cli.write(this.cli.style.dim(`${reduced.length} rule(s) reduced away — run with --explain to see them`));
152
+ }
153
+ }
154
+ /** The same accounting as {@link report}, as plain lines. */
155
+ reportLines(query) {
156
+ return [
157
+ ...query.findings.map(({ reason, source }) => `${source} ${reason}`),
158
+ ...query.dropped.map(({ reason, source }) => `Left out ${source}: ${reason}`),
159
+ ...query.approximated.map(({ reason, source }) => `Approximated ${source}: ${reason}`),
160
+ ...query.warnings,
161
+ ...distinct(query.reduced),
162
+ ];
163
+ }
68
164
  async runQuery(query) {
69
165
  if (!this.athenaService) {
70
166
  throw new Error("No Athena service available to execute the query");
71
167
  }
72
168
  return this.athenaService.getResult(query, [], undefined, AthenaBucket.RumPi);
73
169
  }
170
+ /**
171
+ * Execute the count query and put its handful of rows on the terminal — a
172
+ * count is read, not filed, so nothing is written to disk.
173
+ *
174
+ * Under `--quiet` the rows go to stdout as tab separated values, so a count
175
+ * can be read by something other than a person.
176
+ */
177
+ async executeAndReportCounts(query) {
178
+ this.cli.startAction("QUERY:COUNT", "count the pre-warm list");
179
+ const result = await safe(this.runQuery(query));
180
+ if (result.success !== true) {
181
+ this.cli.failAction("QUERY:COUNT");
182
+ this.cli.writeError(result.error);
183
+ return;
184
+ }
185
+ this.cli.successAction("QUERY:COUNT");
186
+ const rows = result.data;
187
+ if (rows.length === 0) {
188
+ this.cli.writeWarning("No page impressions in the window — the list would be empty");
189
+ return;
190
+ }
191
+ if (this.context.quiet) {
192
+ const columns = Object.keys(rows[0]);
193
+ process.stdout.write([
194
+ columns.join("\t"),
195
+ ...rows.map((row) => columns.map((name) => row[name]).join("\t")),
196
+ ].join("\n") + "\n");
197
+ return;
198
+ }
199
+ this.cli.table(rows, Object.fromEntries(Object.keys(rows[0]).map((name) => [
200
+ name,
201
+ { get: (row) => readable(row[name]) },
202
+ ])));
203
+ }
74
204
  /**
75
205
  * Execute the built query against Athena (Baqend native path, AWS fallback)
76
206
  * and write the rows to a `<app>-<YYYY-MM-DD>-<queryType>.csv` file in the
@@ -80,7 +210,7 @@ export class QueryBuilderService {
80
210
  this.cli.startAction("QUERY:EXECUTE", "execute query and export csv");
81
211
  const result = await safe(this.runQuery(query));
82
212
  if (result.success !== true) {
83
- this.cli.failAction("QUERY:EXECUTE", result.error);
213
+ this.cli.failAction("QUERY:EXECUTE");
84
214
  this.cli.writeError(result.error);
85
215
  return;
86
216
  }
@@ -93,21 +223,4 @@ export class QueryBuilderService {
93
223
  this.cli.successAction("QUERY:EXECUTE");
94
224
  this.cli.writeSuccess(`Wrote ${rows.length} rows to ${filePath} 📄`);
95
225
  }
96
- query() {
97
- const parameters = getStrippedParameters(this.speedKitConfig);
98
- const parameterReplaceFilter = getParameterReplaceFilter(parameters, getSpeedKitConfigVersion(this.speedKitConfig));
99
- const whitelistedFilter = getWhitelistedFilter(this.speedKitConfig);
100
- const blacklistedFilter = getBlacklistedFilter(this.speedKitConfig);
101
- const skConfigVersion = getSpeedKitConfigVersion(this.speedKitConfig);
102
- const parameterFilter = getParameterFilter(parameters, skConfigVersion);
103
- return this.buildQuery(this.context.app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter);
104
- }
105
- buildQuery(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter) {
106
- if (this.context.queryType === QueryTpe.prewarm) {
107
- return new Prewarm(app, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
108
- }
109
- if (this.context.queryType === QueryTpe.parameter) {
110
- return new Parameter(app, parameterFilter, whitelistedFilter, blacklistedFilter, parameterReplaceFilter).query;
111
- }
112
- }
113
226
  }
@@ -0,0 +1,27 @@
1
+ import { Config } from "@oclif/core";
2
+ import { ParameterOptions, PrewarmOptions } from "./queries/query-model.js";
3
+ import { QueryFilterOptions, QueryType } from "./query-builder-model.js";
4
+ /**
5
+ * The flags a query command parses; each takes the ones it offers.
6
+ *
7
+ * `--device` and `--variation` are named as the flags are, singular, and are
8
+ * the two the caller gives rather than the list the builder works on.
9
+ */
10
+ export interface QueryCommandFlags extends ParameterOptions, Omit<PrewarmOptions, "devices" | "variations">, QueryFilterOptions {
11
+ readonly configName: string;
12
+ readonly device?: readonly string[];
13
+ readonly execute?: boolean;
14
+ readonly explain?: boolean;
15
+ readonly quiet?: boolean;
16
+ readonly variation?: readonly string[];
17
+ readonly workgroup?: string;
18
+ }
19
+ /**
20
+ * Builds one of the config-derived queries and hands it over.
21
+ *
22
+ * Both `sk query` commands differ only in which query they ask for, so the
23
+ * wiring lives here rather than twice in the command layer — the two of them
24
+ * drifting apart is what left a diverged copy of the parameter query behind
25
+ * once already.
26
+ */
27
+ export declare function runQuery(queryType: QueryType, customerPath: string, flags: QueryCommandFlags, config: Config): Promise<void>;
@@ -0,0 +1,36 @@
1
+ import { CliConfig } from "../../helpers/cli-config.js";
2
+ import { QueryBuilderFactory } from "./query-builder-factory.js";
3
+ import { normalizeDevices } from "./queries/device.js";
4
+ /**
5
+ * Builds one of the config-derived queries and hands it over.
6
+ *
7
+ * Both `sk query` commands differ only in which query they ask for, so the
8
+ * wiring lives here rather than twice in the command layer — the two of them
9
+ * drifting apart is what left a diverged copy of the parameter query behind
10
+ * once already.
11
+ */
12
+ export async function runQuery(queryType, customerPath, flags, config) {
13
+ const service = await new QueryBuilderFactory({
14
+ bySet: flags.bySet,
15
+ configName: flags.configName,
16
+ count: flags.count,
17
+ customerPath,
18
+ days: flags.days,
19
+ devices: normalizeDevices(flags.device),
20
+ excludeNotFound: flags.excludeNotFound,
21
+ excludeNoindex: flags.excludeNoindex,
22
+ excludeSuspicious: flags.excludeSuspicious,
23
+ execute: flags.execute,
24
+ explain: flags.explain,
25
+ hardNavigationsOnly: flags.hardNavigationsOnly,
26
+ isWindows: config.windows,
27
+ limit: flags.limit,
28
+ match: flags.match,
29
+ minHits: flags.minHits,
30
+ minImpressions: flags.minImpressions,
31
+ quiet: flags.quiet,
32
+ variations: flags.variation,
33
+ workgroup: flags.workgroup,
34
+ }, new CliConfig(config).load()).buildService(queryType);
35
+ await service.run();
36
+ }
@@ -0,0 +1,8 @@
1
+ export * from "./match-pattern.js";
2
+ export * from "./pattern-language.js";
3
+ export * from "./rule-model.js";
4
+ export * from "./rule-parser.js";
5
+ export * from "./rule-reducer.js";
6
+ export * from "./rule-relevance.js";
7
+ export * from "./rule-to-sql.js";
8
+ export * from "./strip-parameters.js";
@@ -0,0 +1,8 @@
1
+ export * from "./match-pattern.js";
2
+ export * from "./pattern-language.js";
3
+ export * from "./rule-model.js";
4
+ export * from "./rule-parser.js";
5
+ export * from "./rule-reducer.js";
6
+ export * from "./rule-relevance.js";
7
+ export * from "./rule-to-sql.js";
8
+ export * from "./strip-parameters.js";
@@ -0,0 +1,51 @@
1
+ import { NormalizedRule } from "./rule-model.js";
2
+ /**
3
+ * An ad-hoc narrowing of a query, given on the command line as a regular
4
+ * expression over the path.
5
+ *
6
+ * It is compiled into the same shape a config rule has, so everything that
7
+ * follows — the check against the config, the translation to SQL, the
8
+ * warnings about constructs Athena reads differently — is the code that
9
+ * already runs for the config's own rules, and the pattern cannot come to
10
+ * mean something else here than it would in a config.
11
+ *
12
+ * The imports here are deliberately kept to the leaves of the rule layer: the
13
+ * flag definitions load this module, and pulling the automata library into
14
+ * every `sk` invocation to validate a pattern would be a poor trade.
15
+ */
16
+ /** Where a `--match` pattern shows up in reports and errors. */
17
+ export declare const MATCH_SOURCE = "--match";
18
+ /**
19
+ * Compiles one `--match` value into a regular expression.
20
+ *
21
+ * Athena carries flags inside the pattern and JavaScript carries them beside
22
+ * it. The operator writes the Athena form, so a leading `(?i)` is lifted into
23
+ * real flags here — `new RegExp('(?i)…')` would throw — and `toAthenaRegex`
24
+ * puts it back when the pattern is emitted.
25
+ *
26
+ * @throws SyntaxError when the value is not a regular expression.
27
+ */
28
+ export declare function parseMatchPattern(value: string): RegExp;
29
+ /**
30
+ * The `--match` patterns as a rule: one condition on the pathname, its
31
+ * entries OR'd, so `--match a --match b` keeps the pages matching either.
32
+ */
33
+ export declare function buildMatchRule(values: readonly string[] | undefined): NormalizedRule | undefined;
34
+ /**
35
+ * An excluding rule restated over `path + search`, so that it can be held
36
+ * against a `--match` pattern — or `undefined` when it cannot be restated
37
+ * exactly.
38
+ *
39
+ * A version 2 `pathname` condition is matched against the path alone, a
40
+ * version 1 one against the path with its query string, and the reduction
41
+ * never compares patterns across those two subjects. Since a `--match`
42
+ * pattern is a version 1 condition, a version 2 rule has to be said again in
43
+ * that subject before the two can be compared at all.
44
+ *
45
+ * This one is used to prove that a pattern is **entirely excluded**, so the
46
+ * safe direction is the opposite of the usual: a rule restated wider than it
47
+ * is would refuse a pattern that does have pages. Anything that cannot be
48
+ * restated exactly is therefore left out of the comparison altogether, and
49
+ * missing an exclusion only costs the early break.
50
+ */
51
+ export declare function asPathWithSearchRule(rule: NormalizedRule): NormalizedRule | undefined;
@@ -0,0 +1,141 @@
1
+ import { escapeRegexLiteral } from "../sql/regex.js";
2
+ import { matchesExactly, } from "./rule-model.js";
3
+ /**
4
+ * An ad-hoc narrowing of a query, given on the command line as a regular
5
+ * expression over the path.
6
+ *
7
+ * It is compiled into the same shape a config rule has, so everything that
8
+ * follows — the check against the config, the translation to SQL, the
9
+ * warnings about constructs Athena reads differently — is the code that
10
+ * already runs for the config's own rules, and the pattern cannot come to
11
+ * mean something else here than it would in a config.
12
+ *
13
+ * The imports here are deliberately kept to the leaves of the rule layer: the
14
+ * flag definitions load this module, and pulling the automata library into
15
+ * every `sk` invocation to validate a pattern would be a poor trade.
16
+ */
17
+ /** Where a `--match` pattern shows up in reports and errors. */
18
+ export const MATCH_SOURCE = "--match";
19
+ /**
20
+ * A `--match` pattern is a version 1 `pathname` condition, whatever version
21
+ * the config runs under: version 1 matches a pathname against the path *and*
22
+ * the query string, which is the subject an operator narrowing a list wants —
23
+ * `^/p/` anchors at the start of the path and `[?&]size=` reaches a
24
+ * parameter.
25
+ */
26
+ const MATCH_RULE_SET_VERSION = 1;
27
+ /** Inline flags, the way Athena takes them: `(?i)`, `(?is)`, … */
28
+ const INLINE_FLAGS = /^\(\?([ims]+)\)/;
29
+ /**
30
+ * Compiles one `--match` value into a regular expression.
31
+ *
32
+ * Athena carries flags inside the pattern and JavaScript carries them beside
33
+ * it. The operator writes the Athena form, so a leading `(?i)` is lifted into
34
+ * real flags here — `new RegExp('(?i)…')` would throw — and `toAthenaRegex`
35
+ * puts it back when the pattern is emitted.
36
+ *
37
+ * @throws SyntaxError when the value is not a regular expression.
38
+ */
39
+ export function parseMatchPattern(value) {
40
+ const inline = INLINE_FLAGS.exec(value);
41
+ return inline
42
+ ? new RegExp(value.slice(inline[0].length), inline[1])
43
+ : new RegExp(value);
44
+ }
45
+ /**
46
+ * The `--match` patterns as a rule: one condition on the pathname, its
47
+ * entries OR'd, so `--match a --match b` keeps the pages matching either.
48
+ */
49
+ export function buildMatchRule(values) {
50
+ if (!values || values.length === 0) {
51
+ return undefined;
52
+ }
53
+ const patterns = values.map((value) => ({
54
+ kind: "regexp",
55
+ value: parseMatchPattern(value),
56
+ }));
57
+ const condition = { patterns, subject: "pathname" };
58
+ return {
59
+ conditions: [condition],
60
+ criteriaOutsideTheUrl: [],
61
+ matchesNothing: false,
62
+ ruleSetVersion: MATCH_RULE_SET_VERSION,
63
+ source: MATCH_SOURCE,
64
+ };
65
+ }
66
+ /**
67
+ * An excluding rule restated over `path + search`, so that it can be held
68
+ * against a `--match` pattern — or `undefined` when it cannot be restated
69
+ * exactly.
70
+ *
71
+ * A version 2 `pathname` condition is matched against the path alone, a
72
+ * version 1 one against the path with its query string, and the reduction
73
+ * never compares patterns across those two subjects. Since a `--match`
74
+ * pattern is a version 1 condition, a version 2 rule has to be said again in
75
+ * that subject before the two can be compared at all.
76
+ *
77
+ * This one is used to prove that a pattern is **entirely excluded**, so the
78
+ * safe direction is the opposite of the usual: a rule restated wider than it
79
+ * is would refuse a pattern that does have pages. Anything that cannot be
80
+ * restated exactly is therefore left out of the comparison altogether, and
81
+ * missing an exclusion only costs the early break.
82
+ */
83
+ export function asPathWithSearchRule(rule) {
84
+ // A version 1 rule already matches the subject the pattern is written for.
85
+ if (!matchesExactly(rule.ruleSetVersion)) {
86
+ return rule;
87
+ }
88
+ const conditions = [];
89
+ for (const condition of rule.conditions) {
90
+ // A rule constraining anything but the path cannot cover the pattern in
91
+ // the first place — the reduction checks that before it builds automata.
92
+ if (condition.subject !== "pathname") {
93
+ return undefined;
94
+ }
95
+ const patterns = [];
96
+ for (const pattern of condition.patterns) {
97
+ const restated = withSearch(pattern);
98
+ if (!restated) {
99
+ return undefined;
100
+ }
101
+ patterns.push(restated);
102
+ }
103
+ conditions.push({ patterns, subject: "pathname" });
104
+ }
105
+ return { ...rule, conditions, ruleSetVersion: MATCH_RULE_SET_VERSION };
106
+ }
107
+ /** What may follow the path in `path + search`. */
108
+ const SEARCH = String.raw `(?:\?[\s\S]*)?`;
109
+ /** Anything but a single `$` at the very end. */
110
+ const INNER_END_ANCHOR = /(?<!\\)\$(?!$)/;
111
+ /**
112
+ * One entry of a version 2 path condition, said over `path + search`.
113
+ *
114
+ * Only the shapes whose meaning is unchanged by the query string are
115
+ * restated. A pattern anchored at the start says the same thing about both
116
+ * subjects; where it ends at the end of the path, the query string is spelled
117
+ * out instead. An unanchored pattern does not survive: it would match inside
118
+ * a query string the path never contained.
119
+ */
120
+ function withSearch(pattern) {
121
+ if (pattern.kind === "string") {
122
+ // A version 2 string is the whole path, compared case-insensitively.
123
+ const value = escapeRegexLiteral(pattern.value);
124
+ return { kind: "regexp", value: new RegExp(`^${value}${SEARCH}$`, "i") };
125
+ }
126
+ const { flags, source } = pattern.value;
127
+ // An alternation may hide an anchor in a branch, which the reasoning below
128
+ // does not see.
129
+ if (!source.startsWith("^") || source.includes("|")) {
130
+ return undefined;
131
+ }
132
+ if (INNER_END_ANCHOR.test(source)) {
133
+ return undefined;
134
+ }
135
+ return {
136
+ kind: "regexp",
137
+ value: /(?<!\\)\$$/.test(source)
138
+ ? new RegExp(`${source.slice(0, -1)}${SEARCH}$`, flags)
139
+ : pattern.value,
140
+ };
141
+ }
@@ -0,0 +1,59 @@
1
+ import { DFA, NFA } from "refa";
2
+ import { RuleCondition } from "./rule-model.js";
3
+ /**
4
+ * The set of subjects a condition accepts, as a finite automaton.
5
+ *
6
+ * Deciding whether one rule makes another redundant means deciding whether one
7
+ * pattern's language contains or avoids another's — a question about regular
8
+ * languages, not about strings, so it is answered with automata rather than
9
+ * guessed. `refa` builds them from the very same JavaScript patterns the config
10
+ * uses.
11
+ */
12
+ export type ConditionLanguage = NFA;
13
+ /**
14
+ * Which side of the true language an automaton is allowed to fall on.
15
+ *
16
+ * Automata cannot express an assertion — a word boundary, or the `$` inside
17
+ * `/^\/shop(\/|$)/` — so a pattern using one is approximated. Approximating in
18
+ * an arbitrary direction would let the reducer drop a rule that does matter,
19
+ * so each question is asked with the bound that makes its answer sound:
20
+ *
21
+ * - `atLeast` accepts everything the pattern accepts, and possibly more. Two
22
+ * of these sharing nothing proves the patterns share nothing.
23
+ * - `atMost` accepts only what the pattern accepts, and possibly less. An
24
+ * `atLeast` language inside an `atMost` one proves containment.
25
+ *
26
+ * A pattern without assertions has one exact language, and both bounds are it.
27
+ */
28
+ export type LanguageBound = "atLeast" | "atMost";
29
+ /**
30
+ * The language of one condition — the union of its entries, since one matching
31
+ * entry satisfies the condition.
32
+ *
33
+ * Returns `undefined` when an entry has no automaton at all, and the caller
34
+ * then leaves the rule out of every reduction.
35
+ */
36
+ export declare function conditionLanguage(condition: RuleCondition, ruleSetVersion: number, bound: LanguageBound): ConditionLanguage | undefined;
37
+ /**
38
+ * Everything a language does *not* accept.
39
+ *
40
+ * Determinising a language to complement it is the expensive step of the whole
41
+ * reduction, and one rule is compared with many others, so a caller that asks
42
+ * repeatedly should build this once and reuse it.
43
+ */
44
+ export type LanguageOutside = DFA;
45
+ /** The complement of an `atMost` language. */
46
+ export declare function outsideOf(language: ConditionLanguage): LanguageOutside | undefined;
47
+ /**
48
+ * True when the language stays inside the one `outside` was built from —
49
+ * `language ⊆ superset ⟺ language ∩ ¬superset = ∅`.
50
+ *
51
+ * @param language - has to be an `atLeast` language.
52
+ * @param outside - has to come from {@link outsideOf} on an `atMost` language.
53
+ */
54
+ export declare function isInside(language: ConditionLanguage, outside: LanguageOutside): boolean | undefined;
55
+ /**
56
+ * True when no subject is accepted by both languages. Both have to be
57
+ * `atLeast` languages.
58
+ */
59
+ export declare function languagesDisjoint(left: ConditionLanguage, right: ConditionLanguage): boolean | undefined;