@speedkit/cli 4.25.2 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +192 -19
  3. package/dist/commands/auto-prewarm.d.ts +8 -1
  4. package/dist/commands/auto-prewarm.js +18 -7
  5. package/dist/commands/generate-pop-config.d.ts +1 -0
  6. package/dist/commands/generate-pop-config.js +9 -3
  7. package/dist/commands/prewarm.js +3 -1
  8. package/dist/commands/query/parameter.d.ts +26 -0
  9. package/dist/commands/query/parameter.js +27 -0
  10. package/dist/commands/query/prewarm.d.ts +31 -0
  11. package/dist/commands/query/prewarm.js +31 -0
  12. package/dist/commands/revalidate.js +8 -6
  13. package/dist/helpers/clipboard.d.ts +19 -0
  14. package/dist/helpers/clipboard.js +70 -0
  15. package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
  16. package/dist/helpers/evaluate-speed-kit-config.js +8 -6
  17. package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
  18. package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
  19. package/dist/models/cli-parameters.d.ts +73 -0
  20. package/dist/models/cli-parameters.js +151 -0
  21. package/dist/models/cli-parameters.spec.d.ts +1 -0
  22. package/dist/models/cli-parameters.spec.js +47 -0
  23. package/dist/services/athena/athena-service.d.ts +2 -0
  24. package/dist/services/athena/athena-service.js +15 -4
  25. package/dist/services/athena/athena-service.spec.d.ts +1 -0
  26. package/dist/services/athena/athena-service.spec.js +74 -0
  27. package/dist/services/deploy/handler/install-resource-handler.js +3 -4
  28. package/dist/services/onboarding/onboarding-service-factory.js +2 -2
  29. package/dist/services/pop-config/pop-config-service.d.ts +8 -1
  30. package/dist/services/pop-config/pop-config-service.js +16 -28
  31. package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
  32. package/dist/services/prewarm/assets/asset-api-client.js +5 -4
  33. package/dist/services/prewarm/auto-pre-warm-factory.d.ts +12 -2
  34. package/dist/services/prewarm/auto-pre-warm-factory.js +20 -6
  35. package/dist/services/prewarm/csv-reader.d.ts +21 -2
  36. package/dist/services/prewarm/csv-reader.js +71 -9
  37. package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
  38. package/dist/services/prewarm/csv-reader.spec.js +75 -0
  39. package/dist/services/prewarm/index.d.ts +1 -0
  40. package/dist/services/prewarm/index.js +1 -0
  41. package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
  42. package/dist/services/prewarm/pre-warm-factory.js +8 -11
  43. package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
  44. package/dist/services/prewarm/pre-warm-model.js +0 -1
  45. package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
  46. package/dist/services/prewarm/pre-warm-service.js +18 -14
  47. package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
  48. package/dist/services/prewarm/prewarm-targets.js +18 -0
  49. package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
  50. package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
  51. package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
  52. package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
  53. package/dist/services/query-builder/queries/device.d.ts +21 -0
  54. package/dist/services/query-builder/queries/device.js +26 -0
  55. package/dist/services/query-builder/queries/index.d.ts +6 -0
  56. package/dist/services/query-builder/queries/index.js +6 -0
  57. package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
  58. package/dist/services/query-builder/queries/page-filter.js +135 -0
  59. package/dist/services/query-builder/queries/page-source.d.ts +30 -0
  60. package/dist/services/query-builder/queries/page-source.js +79 -0
  61. package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
  62. package/dist/services/query-builder/queries/parameter-query.js +142 -0
  63. package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
  64. package/dist/services/query-builder/queries/pops-query.js +65 -0
  65. package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
  66. package/dist/services/query-builder/queries/prewarm-query.js +227 -0
  67. package/dist/services/query-builder/queries/query-model.d.ts +97 -0
  68. package/dist/services/query-builder/queries/query-model.js +19 -0
  69. package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
  70. package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
  71. package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
  72. package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
  73. package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
  74. package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
  75. package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
  76. package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
  77. package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
  78. package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
  79. package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
  80. package/dist/services/query-builder/query-builder-factory.js +35 -7
  81. package/dist/services/query-builder/query-builder-model.d.ts +31 -19
  82. package/dist/services/query-builder/query-builder-model.js +5 -5
  83. package/dist/services/query-builder/query-builder-service.d.ts +26 -5
  84. package/dist/services/query-builder/query-builder-service.js +157 -44
  85. package/dist/services/query-builder/query-command.d.ts +26 -0
  86. package/dist/services/query-builder/query-command.js +35 -0
  87. package/dist/services/query-builder/rules/index.d.ts +8 -0
  88. package/dist/services/query-builder/rules/index.js +8 -0
  89. package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
  90. package/dist/services/query-builder/rules/match-pattern.js +141 -0
  91. package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
  92. package/dist/services/query-builder/rules/pattern-language.js +159 -0
  93. package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
  94. package/dist/services/query-builder/rules/rule-model.js +54 -0
  95. package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
  96. package/dist/services/query-builder/rules/rule-parser.js +189 -0
  97. package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
  98. package/dist/services/query-builder/rules/rule-reducer.js +383 -0
  99. package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
  100. package/dist/services/query-builder/rules/rule-relevance.js +69 -0
  101. package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
  102. package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
  103. package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
  104. package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
  105. package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
  106. package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
  107. package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
  108. package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
  109. package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
  110. package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
  111. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
  112. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
  113. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
  114. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
  115. package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
  116. package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
  117. package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
  118. package/dist/services/query-builder/rules/strip-parameters.js +197 -0
  119. package/dist/services/query-builder/sql/expression.d.ts +73 -0
  120. package/dist/services/query-builder/sql/expression.js +133 -0
  121. package/dist/services/query-builder/sql/format.d.ts +14 -0
  122. package/dist/services/query-builder/sql/format.js +48 -0
  123. package/dist/services/query-builder/sql/index.d.ts +4 -0
  124. package/dist/services/query-builder/sql/index.js +4 -0
  125. package/dist/services/query-builder/sql/predicate.d.ts +66 -0
  126. package/dist/services/query-builder/sql/predicate.js +124 -0
  127. package/dist/services/query-builder/sql/regex.d.ts +28 -0
  128. package/dist/services/query-builder/sql/regex.js +81 -0
  129. package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
  130. package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
  131. package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
  132. package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
  133. package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
  134. package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
  135. package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
  136. package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
  137. package/dist/services/revalidate/revalidate-model.d.ts +9 -3
  138. package/dist/services/revalidate/revalidate-model.js +12 -3
  139. package/oclif.manifest.json +354 -104
  140. package/package.json +11 -2
  141. package/dist/commands/build-parameter-query.d.ts +0 -14
  142. package/dist/commands/build-parameter-query.js +0 -25
  143. package/dist/commands/build-prewarm-query.d.ts +0 -16
  144. package/dist/commands/build-prewarm-query.js +0 -37
  145. package/dist/helpers/build-query-helper.d.ts +0 -32
  146. package/dist/helpers/build-query-helper.js +0 -223
  147. package/dist/helpers/get-parsed-config.d.ts +0 -15
  148. package/dist/helpers/get-parsed-config.js +0 -62
  149. package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
  150. package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
@@ -17,15 +17,26 @@ export class AwsAthenaService {
17
17
  }
18
18
  async getResult(query, parameters = [], maxAgeInMinutes = DEFAULT_MAX_AGE_IN_MINUTES) {
19
19
  this.cli.startAction("ATHENA:QUERY", "execute athena query");
20
- const queryId = await this.runQuery(query, parameters, maxAgeInMinutes);
21
- const result = await safe(this.fetchResult(queryId));
20
+ // Everything the action covers is inside the guard, running the query
21
+ // included: Athena reports a rejected query by failing the execution, and
22
+ // a throw that escapes here leaves the spinner running — which is a timer,
23
+ // so the process stays alive with nothing left to do and has to be killed
24
+ // by hand.
25
+ const result = await safe(this.runAndFetch(query, parameters, maxAgeInMinutes));
22
26
  if (result.success === true) {
23
27
  this.cli.successAction("ATHENA:QUERY");
24
28
  return result.data;
25
29
  }
30
+ // Rethrown rather than answered with an empty result: no rows and no rows
31
+ // found look the same to a caller, and a pre-warm list that silently came
32
+ // back empty is worse than one that failed.
26
33
  this.cli.failAction("ATHENA:QUERY");
27
- this.cli.writeError(result.error);
28
- return [];
34
+ throw result.errorObj ?? new Error(result.error);
35
+ }
36
+ /** The query, from sending it off to the rows it produced. */
37
+ async runAndFetch(query, parameters, maxAgeInMinutes) {
38
+ const queryId = await this.runQuery(query, parameters, maxAgeInMinutes);
39
+ return this.fetchResult(queryId);
29
40
  }
30
41
  castValueToType(value, type) {
31
42
  if (value === null)
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,74 @@
1
+ import { GetQueryResultsCommand, StartQueryExecutionCommand, } from "@aws-sdk/client-athena";
2
+ import { expect } from "chai";
3
+ import { describe, it } from "mocha";
4
+ import { AwsAthenaService } from "./athena-service.js";
5
+ /** The cli calls the service makes, in the order it made them. */
6
+ function recordingCli() {
7
+ const calls = [];
8
+ const cli = {
9
+ failAction: (name) => calls.push(`fail:${name}`),
10
+ startAction: (name) => calls.push(`start:${name}`),
11
+ successAction: (name) => calls.push(`success:${name}`),
12
+ writeError: (message) => calls.push(`error:${message}`),
13
+ };
14
+ return { calls, cli };
15
+ }
16
+ /** An Athena that answers every execution with the given state. */
17
+ function clientAnswering(state, reason) {
18
+ return {
19
+ send: async (command) => {
20
+ if (command instanceof StartQueryExecutionCommand) {
21
+ return { QueryExecutionId: "query-1" };
22
+ }
23
+ if (command instanceof GetQueryResultsCommand) {
24
+ return {
25
+ ResultSet: {
26
+ ResultSetMetadata: {
27
+ ColumnInfo: [{ Name: "urls", Type: "bigint" }],
28
+ },
29
+ Rows: [
30
+ { Data: [{ VarCharValue: "urls" }] },
31
+ { Data: [{ VarCharValue: "42" }] },
32
+ ],
33
+ },
34
+ };
35
+ }
36
+ return {
37
+ QueryExecution: { Status: { State: state, StateChangeReason: reason } },
38
+ };
39
+ },
40
+ };
41
+ }
42
+ describe("AwsAthenaService", () => {
43
+ it("should return the rows of a query that succeeded", async () => {
44
+ const { calls, cli } = recordingCli();
45
+ const service = new AwsAthenaService(clientAnswering("SUCCEEDED"), cli);
46
+ expect(await service.getResult("select 1", [], 0)).to.deep.equal([
47
+ { urls: 42 },
48
+ ]);
49
+ expect(calls).to.deep.equal(["start:ATHENA:QUERY", "success:ATHENA:QUERY"]);
50
+ });
51
+ it("should end the action when the query itself failed", async () => {
52
+ // A failed execution throws from inside the action. Left unhandled it
53
+ // leaves the spinner running, and the spinner is a timer: the process then
54
+ // never exits and has to be killed by hand.
55
+ const { calls, cli } = recordingCli();
56
+ const service = new AwsAthenaService(clientAnswering("FAILED", "INVALID_ARGUMENTS: nope"), cli);
57
+ const result = await service
58
+ .getResult("select 1", [], 0)
59
+ .then(() => undefined)
60
+ .catch((error) => error);
61
+ expect(result).to.be.an("error");
62
+ expect(result.message).to.contain("INVALID_ARGUMENTS: nope");
63
+ expect(calls).to.deep.equal(["start:ATHENA:QUERY", "fail:ATHENA:QUERY"]);
64
+ });
65
+ it("should not pass a failed query off as an empty result", async () => {
66
+ const { cli } = recordingCli();
67
+ const service = new AwsAthenaService(clientAnswering("CANCELLED"), cli);
68
+ const result = await service
69
+ .getResult("select 1", [], 0)
70
+ .then(() => "resolved")
71
+ .catch(() => "rejected");
72
+ expect(result).to.equal("rejected");
73
+ });
74
+ });
@@ -1,6 +1,6 @@
1
1
  import { FILE_TYPE, } from "../../integration-api/index.js";
2
+ import { copyToClipboard } from "../../../helpers/clipboard.js";
2
3
  import { safe } from "../../../helpers/safe.js";
3
- import clipboardy from "clipboardy";
4
4
  export default class InstallResourceHandler {
5
5
  customer;
6
6
  configApi;
@@ -67,11 +67,10 @@ export default class InstallResourceHandler {
67
67
  }
68
68
  this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST");
69
69
  const debugParameter = `?debugSpeedKit=${install.id.split("/").pop()}`;
70
- try {
71
- clipboardy.writeSync(debugParameter);
70
+ if (await copyToClipboard(debugParameter)) {
72
71
  this.cli.writeSuccess("Query parameter to activate the testing installation has been copied to the clipboard 📋");
73
72
  }
74
- catch {
73
+ else {
75
74
  this.cli.writeSuccess(`Use the following parameter to active the testing installation in your browser: ${debugParameter}`);
76
75
  }
77
76
  }
@@ -41,7 +41,7 @@ import { ExtensionDownloader } from "./browser/extension/extension-downloader.js
41
41
  import { ExecutableValidator } from "./browser/executable/executable-validator.js";
42
42
  import { applyOriginProxy, withOriginProxyFlag, } from "../../helpers/origin-proxy.js";
43
43
  import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
44
- import { QueryTpe } from "../query-builder/query-builder-model.js";
44
+ import { QueryType } from "../query-builder/query-builder-model.js";
45
45
  export class OnboardingServiceFactory {
46
46
  context;
47
47
  cliConfig;
@@ -155,7 +155,7 @@ export class OnboardingServiceFactory {
155
155
  configName: this.context.configName,
156
156
  customerPath: this.context.customerPath,
157
157
  isWindows: this.cliConfig.runtimeConfig.windows,
158
- }, this.cliConfig).buildService(QueryTpe.parameter);
158
+ }, this.cliConfig).buildService(QueryType.parameter);
159
159
  const DiffService = new DiffServiceFactory(new DiffContext(this.cliConfig.diffExec, this.cliConfig.diffExecTemplate)).getService();
160
160
  const requestDiffService = new RequestDiffService(DiffService, documentHandler);
161
161
  const handlers = [
@@ -3,5 +3,12 @@ import { AthenaService } from "../athena/index.js";
3
3
  export declare class PopConfigService {
4
4
  private athenaService;
5
5
  constructor(athenaService: AthenaService);
6
- getUsedPopsPerOrigin(app: string, useRum: boolean): Promise<UsedPopsPerOrigin[]>;
6
+ /**
7
+ * The PoPs serving the app's traffic, per origin and region.
8
+ *
9
+ * @param useRum - count page-impression beacons instead of Speed Kit's own
10
+ * asset requests, which is what an app without production traffic needs.
11
+ * @param days - size of the window to look at, in days.
12
+ */
13
+ getUsedPopsPerOrigin(app: string, useRum: boolean, days?: number): Promise<UsedPopsPerOrigin[]>;
7
14
  }
@@ -1,36 +1,24 @@
1
1
  import { AthenaBucket } from "../athena/index.js";
2
+ import { buildPopsQuery } from "../query-builder/queries/index.js";
3
+ /**
4
+ * How long a result may be reused. The PoP distribution of an app barely
5
+ * moves within a day, and the query scans a week of request logs.
6
+ */
7
+ const MAX_AGE_IN_MINUTES = 12 * 60;
2
8
  export class PopConfigService {
3
9
  athenaService;
4
10
  constructor(athenaService) {
5
11
  this.athenaService = athenaService;
6
12
  }
7
- async getUsedPopsPerOrigin(app, useRum) {
8
- const query = `
9
- with data as (select *,
10
- coalesce('https://' || url_extract_host(assetUrl) ||
11
- regexp_extract(url_extract_path(assetUrl), '(?i)((/\\w+)?/[a-z]{2}([-/_][a-z]{2})*/)', 0),
12
- 'Unknown') as originWithPrefix
13
- from fastly.logs
14
- where date between ? and ?
15
- and app = ?
16
- and ${useRum ? "ispibeacon" : "isassetrequest and isnavigaterequest"}
17
- and client.bot is null
18
- and origin not in ('Unknown', ''))
19
-
20
- select region,
21
- pop,
22
- coalesce(originWithPrefix, origin) as origin,
23
- count() as hits
24
- from data
25
- group by grouping sets (
26
- (region, pop, origin),
27
- (region, pop, origin, originWithPrefix)
28
- )
29
- having originWithPrefix is null or originWithPrefix <> 'Unknown'`;
30
- const endDate = new Date();
31
- const startDate = new Date(endDate.getFullYear(), endDate.getMonth(), endDate.getDate() - 7);
32
- const start = startDate.toISOString().slice(0, 10);
33
- const end = endDate.toISOString().slice(0, 10);
34
- return await this.athenaService.getResult(query, [start, end, app], 12 * 60, AthenaBucket.FastlyLogs);
13
+ /**
14
+ * The PoPs serving the app's traffic, per origin and region.
15
+ *
16
+ * @param useRum - count page-impression beacons instead of Speed Kit's own
17
+ * asset requests, which is what an app without production traffic needs.
18
+ * @param days - size of the window to look at, in days.
19
+ */
20
+ async getUsedPopsPerOrigin(app, useRum, days) {
21
+ const { parameters, sql } = buildPopsQuery({ app, days, useRum });
22
+ return await this.athenaService.getResult(sql, parameters, MAX_AGE_IN_MINUTES, AthenaBucket.FastlyLogs);
35
23
  }
36
24
  }
@@ -26,7 +26,7 @@ export declare class AssetApiClient {
26
26
  * handle parameter "bqvariation"
27
27
  * reflect behaviour of SpeedKit.
28
28
  * - do not send default variation
29
- * - send mobile/tablet/tv as lowercase
29
+ * - send mobile/tablet as lowercase
30
30
  * - send customVariations as they are
31
31
  *
32
32
  * As orestes will always make variations uppercase and does not care for caseSensitivity,
@@ -1,6 +1,7 @@
1
1
  import { URL } from "node:url";
2
2
  import http2 from "node:http2";
3
- import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR, DEFAULT_VARIATIONS, TIMING_HEADER_KEY, } from "../index.js";
3
+ import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR, TIMING_HEADER_KEY, } from "../index.js";
4
+ import { DEVICE_VARIATIONS, } from "../../query-builder/queries/device.js";
4
5
  import { normalize } from "../../../helpers/normalize.js";
5
6
  import { InvalidOriginError } from "../error/invalid-origin-error.js";
6
7
  export class AssetApiClient {
@@ -114,7 +115,7 @@ export class AssetApiClient {
114
115
  * handle parameter "bqvariation"
115
116
  * reflect behaviour of SpeedKit.
116
117
  * - do not send default variation
117
- * - send mobile/tablet/tv as lowercase
118
+ * - send mobile/tablet as lowercase
118
119
  * - send customVariations as they are
119
120
  *
120
121
  * As orestes will always make variations uppercase and does not care for caseSensitivity,
@@ -130,8 +131,8 @@ export class AssetApiClient {
130
131
  if (variation === "default" || variation === "desktop") {
131
132
  return;
132
133
  }
133
- // on tv|tablet|mobile set variation as lowerCase
134
- if (DEFAULT_VARIATIONS.includes(variation.toLowerCase())) {
134
+ // on tablet|mobile set variation as lowerCase
135
+ if (DEVICE_VARIATIONS.includes(variation.toLowerCase())) {
135
136
  assetUrl.searchParams.set("bqvariation", variation.toLowerCase());
136
137
  return;
137
138
  }
@@ -1,9 +1,15 @@
1
1
  import { UserCliConfig } from "../../helpers/cli-config.js";
2
+ import { QueryFilterOptions } from "../query-builder/query-builder-model.js";
2
3
  import { PreWarmService } from "./pre-warm-service.js";
3
- export interface AutoPreWarmContext {
4
+ export interface AutoPreWarmContext extends QueryFilterOptions {
4
5
  readonly customerPath: string;
5
6
  readonly configName: string;
6
- readonly variation?: string;
7
+ /** Device classes to warm each url in the variations it was requested in. */
8
+ readonly device?: readonly string[];
9
+ /** Regular expressions the pre-warmed urls have to match. */
10
+ readonly match?: readonly string[];
11
+ /** Variations to pair every url with, when no traffic can reveal them. */
12
+ readonly variation?: readonly string[];
7
13
  readonly variationPath?: string;
8
14
  readonly sortParameters?: boolean;
9
15
  readonly quiet?: boolean;
@@ -16,6 +22,10 @@ export interface AutoPreWarmContext {
16
22
  * config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
17
23
  * against Athena (Baqend native path, AWS fallback), and feeds the resulting
18
24
  * URLs straight into the prewarm job.
25
+ *
26
+ * `--device` and `--variation` are handed to the query, which returns the
27
+ * variation alongside every url; without them the query returns urls and the
28
+ * list is crossed with `--variationPath` or the default variation, as before.
19
29
  */
20
30
  export declare class AutoPreWarmFactory {
21
31
  private context;
@@ -1,17 +1,23 @@
1
1
  import { CliContext, CliServiceFactory } from "../cli/index.js";
2
2
  import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
3
- import { QueryTpe } from "../query-builder/query-builder-model.js";
3
+ import { QueryType, } from "../query-builder/query-builder-model.js";
4
+ import { normalizeDevices } from "../query-builder/queries/device.js";
4
5
  import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
5
6
  import { AssetApiClient } from "./assets/asset-api-client.js";
6
7
  import { CsvReader } from "./csv-reader.js";
7
8
  import { PreWarmService } from "./pre-warm-service.js";
8
9
  import { resolveVariations } from "./pre-warm-factory.js";
10
+ import { toPrewarmTargets } from "./prewarm-targets.js";
9
11
  /**
10
12
  * Wires a {@link PreWarmService} that gets its URLs from a freshly built and
11
13
  * executed prewarm query instead of a CSV file. Reads the app from the customer
12
14
  * config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
13
15
  * against Athena (Baqend native path, AWS fallback), and feeds the resulting
14
16
  * URLs straight into the prewarm job.
17
+ *
18
+ * `--device` and `--variation` are handed to the query, which returns the
19
+ * variation alongside every url; without them the query returns urls and the
20
+ * list is crossed with `--variationPath` or the default variation, as before.
15
21
  */
16
22
  export class AutoPreWarmFactory {
17
23
  context;
@@ -25,18 +31,26 @@ export class AutoPreWarmFactory {
25
31
  const queryService = await new QueryBuilderFactory({
26
32
  configName: this.context.configName,
27
33
  customerPath: this.context.customerPath,
34
+ days: this.context.days,
35
+ devices: normalizeDevices(this.context.device),
36
+ excludeNoindex: this.context.excludeNoindex,
37
+ excludeSuspicious: this.context.excludeSuspicious,
38
+ execute: true,
39
+ hardNavigationsOnly: this.context.hardNavigationsOnly,
28
40
  isWindows: this.context.isWindows,
41
+ match: this.context.match,
29
42
  quiet: this.context.quiet,
30
- execute: true,
31
- }, this.userConfig).buildService(QueryTpe.prewarm);
43
+ variations: this.context.variation,
44
+ }, this.userConfig).buildService(QueryType.prewarm);
32
45
  const app = queryService.app;
33
46
  const rows = await queryService.execute();
34
- const urls = rows.map((row) => row.url).filter(Boolean);
35
47
  const configApi = new ConfigApiServiceFactory(new ConfigApiContext(app)).getService();
36
48
  const entityManager = await configApi.client.getEntityManager();
37
49
  const assetApiClient = new AssetApiClient(entityManager.token, app, this.context.sortParameters);
38
50
  const csvReader = new CsvReader();
39
- const variations = resolveVariations(csvReader, this.context.variation, this.context.variationPath);
40
- return new PreWarmService(assetApiClient, cli, urls, variations, this.context.verbose);
51
+ return new PreWarmService(assetApiClient, cli, toPrewarmTargets(rows.filter((row) => row.url),
52
+ // `--variation` was answered by the query already; a variations file
53
+ // is still applied here.
54
+ resolveVariations(csvReader, undefined, this.context.variationPath)), this.context.verbose);
41
55
  }
42
56
  }
@@ -1,5 +1,24 @@
1
+ import { PrewarmRow } from "./pre-warm-model.js";
1
2
  export declare class CsvReader {
2
- readPath(filePath: string): string[];
3
- private cleanUpEntries;
3
+ /**
4
+ * The pre-warm list of a csv: one url per line, together with the variation
5
+ * to warm it in where the file names one.
6
+ *
7
+ * A second column only carries a variation when the header says so. The file
8
+ * `sk query prewarm --execute` writes for an unsplit list has the hit count
9
+ * there, and a list without a header at all is what it always was — bare
10
+ * urls, all of them to be warmed in whatever variations the caller asks for.
11
+ */
12
+ readList(filePath: string): PrewarmRow[];
13
+ /**
14
+ * The first column of a csv, for a file that is a plain list of values —
15
+ * the variations to warm every url in.
16
+ */
17
+ readValues(filePath: string): string[];
18
+ /**
19
+ * The fields of a csv line. A quoted field is taken as it stands, so a comma
20
+ * inside a url stays part of it, and its doubled quotes are one quote.
21
+ */
22
+ private fields;
4
23
  private getEntries;
5
24
  }
@@ -3,22 +3,84 @@ import path from "node:path";
3
3
  import { ReadCsvError } from "./error/read-csv-error.js";
4
4
  const ENCODING = "utf8";
5
5
  const LINE_BREAK = /\r?\n/;
6
+ /** The header the query command writes above a split pre-warm list. */
7
+ const VARIATION_COLUMN = "variation";
6
8
  export class CsvReader {
7
- readPath(filePath) {
9
+ /**
10
+ * The pre-warm list of a csv: one url per line, together with the variation
11
+ * to warm it in where the file names one.
12
+ *
13
+ * A second column only carries a variation when the header says so. The file
14
+ * `sk query prewarm --execute` writes for an unsplit list has the hit count
15
+ * there, and a list without a header at all is what it always was — bare
16
+ * urls, all of them to be warmed in whatever variations the caller asks for.
17
+ */
18
+ readList(filePath) {
8
19
  try {
9
- const entries = this.getEntries(filePath);
10
- return this.cleanUpEntries(entries);
20
+ const lines = this.getEntries(filePath).map((line) => this.fields(line));
21
+ const [header] = lines;
22
+ const hasHeader = header?.[0]?.toLowerCase() === "url";
23
+ const carriesVariation = hasHeader && header[1]?.toLowerCase() === VARIATION_COLUMN;
24
+ return (hasHeader ? lines.slice(1) : lines)
25
+ .filter(([url]) => url)
26
+ .map(([url, second]) => carriesVariation && second ? { url, variation: second } : { url });
11
27
  }
12
28
  catch (error) {
13
29
  throw new ReadCsvError(error.message);
14
30
  }
15
31
  }
16
- cleanUpEntries(entries) {
17
- return entries
18
- .map((entry) => {
19
- return /^"[^"]+"/.test(entry) ? entry.slice(1, -1) : entry;
20
- })
21
- .filter(Boolean);
32
+ /**
33
+ * The first column of a csv, for a file that is a plain list of values —
34
+ * the variations to warm every url in.
35
+ */
36
+ readValues(filePath) {
37
+ try {
38
+ return this.getEntries(filePath)
39
+ .map((line) => this.fields(line)[0])
40
+ .filter(Boolean);
41
+ }
42
+ catch (error) {
43
+ throw new ReadCsvError(error.message);
44
+ }
45
+ }
46
+ /**
47
+ * The fields of a csv line. A quoted field is taken as it stands, so a comma
48
+ * inside a url stays part of it, and its doubled quotes are one quote.
49
+ */
50
+ fields(line) {
51
+ const entry = line.trim();
52
+ const fields = [];
53
+ let field = "";
54
+ let quoted = false;
55
+ for (let index = 0; index < entry.length; index++) {
56
+ const character = entry[index];
57
+ if (quoted) {
58
+ if (character !== '"') {
59
+ field += character;
60
+ continue;
61
+ }
62
+ // A doubled quote is one quote; a single one ends the field.
63
+ if (entry[index + 1] === '"') {
64
+ field += '"';
65
+ index++;
66
+ continue;
67
+ }
68
+ quoted = false;
69
+ continue;
70
+ }
71
+ if (character === '"' && field === "") {
72
+ quoted = true;
73
+ continue;
74
+ }
75
+ if (character === ",") {
76
+ fields.push(field.trim());
77
+ field = "";
78
+ continue;
79
+ }
80
+ field += character;
81
+ }
82
+ fields.push(field.trim());
83
+ return fields;
22
84
  }
23
85
  getEntries(filePath) {
24
86
  const fileContent = fs.readFileSync(path.resolve(filePath), ENCODING);
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,75 @@
1
+ import { mkdtempSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { expect } from "chai";
5
+ import { describe, it } from "mocha";
6
+ import { CsvReader } from "./csv-reader.js";
7
+ function write(content) {
8
+ const file = join(mkdtempSync(join(tmpdir(), "sk-csv-")), "urls.csv");
9
+ writeFileSync(file, content, "utf8");
10
+ return file;
11
+ }
12
+ /** The urls of a list, for the cases that say nothing about variations. */
13
+ function read(content) {
14
+ return readList(content).map(({ url }) => url);
15
+ }
16
+ function readList(content) {
17
+ return new CsvReader().readList(write(content));
18
+ }
19
+ describe("CsvReader", () => {
20
+ it("should read a bare list of urls", () => {
21
+ expect(read("https://a.de/\nhttps://b.de/\n")).to.deep.equal([
22
+ "https://a.de/",
23
+ "https://b.de/",
24
+ ]);
25
+ });
26
+ it("should read quoted urls", () => {
27
+ expect(read('"https://a.de/"\n"https://b.de/"')).to.deep.equal([
28
+ "https://a.de/",
29
+ "https://b.de/",
30
+ ]);
31
+ });
32
+ it("should take only the first column", () => {
33
+ // What `sk query prewarm --execute` writes: url and hit count.
34
+ expect(read('"url","hits"\n"https://a.de/",12\n"https://b.de/",3')).to.deep.eq(["https://a.de/", "https://b.de/"]);
35
+ });
36
+ it("should skip a header row", () => {
37
+ expect(read("url\nhttps://a.de/")).to.deep.equal(["https://a.de/"]);
38
+ });
39
+ it("should keep a comma inside a quoted url", () => {
40
+ expect(read('"https://a.de/?list=1,2",7')).to.deep.equal([
41
+ "https://a.de/?list=1,2",
42
+ ]);
43
+ });
44
+ it("should read a doubled quote as one quote", () => {
45
+ expect(read('"https://a.de/?q=""x"""')).to.deep.equal([
46
+ 'https://a.de/?q="x"',
47
+ ]);
48
+ });
49
+ it("should drop empty lines", () => {
50
+ expect(read("https://a.de/\n\n\nhttps://b.de/\n")).to.have.lengthOf(2);
51
+ });
52
+ it("should read the variation of a split list", () => {
53
+ // What `sk query prewarm --device desktop,mobile --execute` writes.
54
+ expect(readList('"url","variation","hits"\n"https://a.de/","mobile",12\n"https://a.de/","desktop",3')).to.deep.equal([
55
+ { url: "https://a.de/", variation: "mobile" },
56
+ { url: "https://a.de/", variation: "desktop" },
57
+ ]);
58
+ });
59
+ it("should not read a hit count as a variation", () => {
60
+ expect(readList('"url","hits"\n"https://a.de/",12')).to.deep.equal([
61
+ { url: "https://a.de/" },
62
+ ]);
63
+ });
64
+ it("should not read a second column of a headerless list", () => {
65
+ expect(readList('"https://a.de/","mobile"')).to.deep.equal([
66
+ { url: "https://a.de/" },
67
+ ]);
68
+ });
69
+ it("should keep a comma inside a quoted url of a split list", () => {
70
+ expect(readList('url,variation\n"https://a.de/?list=1,2",mobile')).to.deep.equal([{ url: "https://a.de/?list=1,2", variation: "mobile" }]);
71
+ });
72
+ it("should read a plain list of values", () => {
73
+ expect(new CsvReader().readValues(write("desktop\nmobile\n"))).to.deep.equal(["desktop", "mobile"]);
74
+ });
75
+ });
@@ -2,3 +2,4 @@ export * from "./pre-warm-service.js";
2
2
  export * from "./pre-warm-factory.js";
3
3
  export * from "./auto-pre-warm-factory.js";
4
4
  export * from "./pre-warm-model.js";
5
+ export * from "./prewarm-targets.js";
@@ -2,3 +2,4 @@ export * from "./pre-warm-service.js";
2
2
  export * from "./pre-warm-factory.js";
3
3
  export * from "./auto-pre-warm-factory.js";
4
4
  export * from "./pre-warm-model.js";
5
+ export * from "./prewarm-targets.js";
@@ -4,8 +4,6 @@ export declare class PreWarmFactory {
4
4
  private context;
5
5
  constructor(context: PreWarmContext);
6
6
  getService(): Promise<PreWarmService>;
7
- private getUrls;
8
- private getVariations;
9
7
  }
10
8
  /**
11
9
  * Resolve the list of variations to prewarm: a `variationPath` CSV takes
@@ -1,5 +1,5 @@
1
1
  import { CliContext, CliServiceFactory } from "../cli/index.js";
2
- import { DEFAULT_VARIATION, PreWarmService, VARIATION_SEPARATOR, } from "./index.js";
2
+ import { DEFAULT_VARIATION, PreWarmService, toPrewarmTargets, VARIATION_SEPARATOR, } from "./index.js";
3
3
  import { AssetApiClient } from "./assets/asset-api-client.js";
4
4
  import { CsvReader } from "./csv-reader.js";
5
5
  import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
@@ -14,15 +14,12 @@ export class PreWarmFactory {
14
14
  const entityManager = await configApi.client.getEntityManager();
15
15
  const csvReader = new CsvReader();
16
16
  const assetApiClient = new AssetApiClient(entityManager.token, this.context.app, this.context.keepParameterSorting);
17
- const urls = this.getUrls(csvReader);
18
- const variations = this.getVariations(csvReader);
19
- return new PreWarmService(assetApiClient, cli, urls, variations, this.context.verbose);
20
- }
21
- getUrls(csvReader) {
22
- return csvReader.readPath(this.context.path);
23
- }
24
- getVariations(csvReader) {
25
- return resolveVariations(csvReader, this.context.variation, this.context.variationPath);
17
+ const rows = csvReader.readList(this.context.path);
18
+ const named = this.context.variation ?? this.context.variationPath;
19
+ if (named && rows.some((row) => row.variation)) {
20
+ cli.writeWarning(`The list names a variation for every url, so ${named} is not used`);
21
+ }
22
+ return new PreWarmService(assetApiClient, cli, toPrewarmTargets(rows, resolveVariations(csvReader, this.context.variation, this.context.variationPath)), this.context.verbose);
26
23
  }
27
24
  }
28
25
  /**
@@ -37,7 +34,7 @@ export function resolveVariations(csvReader, variation, variationPath) {
37
34
  variations = variation.split(VARIATION_SEPARATOR);
38
35
  }
39
36
  if (variationPath) {
40
- variations = csvReader.readPath(variationPath);
37
+ variations = csvReader.readValues(variationPath);
41
38
  }
42
39
  return variations;
43
40
  }
@@ -8,6 +8,20 @@ export declare class PreWarmContext {
8
8
  readonly verbose: boolean;
9
9
  constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean, verbose?: boolean);
10
10
  }
11
+ /**
12
+ * One url to warm, in one variation. A pre-warm list is a list of these: the
13
+ * variation is a fact about the url, not a dimension crossed over the whole
14
+ * list, so a url seen only on phones is warmed only as `mobile`.
15
+ */
16
+ export interface PrewarmTarget {
17
+ readonly url: string;
18
+ readonly variation: string;
19
+ }
20
+ /** A row of a pre-warm list, which may or may not name a variation yet. */
21
+ export interface PrewarmRow {
22
+ readonly url: string;
23
+ readonly variation?: string;
24
+ }
11
25
  export interface AssetItemInterface {
12
26
  done(): void;
13
27
  error?: Error;
@@ -36,4 +50,3 @@ export declare const TIMING_HEADER_KEY = "server-timing";
36
50
  export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
37
51
  export declare const VARIATION_SEPARATOR = ",";
38
52
  export declare const DEFAULT_VARIATION: string[];
39
- export declare const DEFAULT_VARIATIONS: string[];
@@ -36,4 +36,3 @@ export const TIMING_HEADER_KEY = "server-timing";
36
36
  export const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
37
37
  export const VARIATION_SEPARATOR = ",";
38
38
  export const DEFAULT_VARIATION = ["default"];
39
- export const DEFAULT_VARIATIONS = ["mobile", "tablet", "tv"];