@speedkit/cli 4.25.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +192 -19
- package/dist/commands/auto-prewarm.d.ts +8 -1
- package/dist/commands/auto-prewarm.js +18 -7
- package/dist/commands/generate-pop-config.d.ts +1 -0
- package/dist/commands/generate-pop-config.js +9 -3
- package/dist/commands/prewarm.js +3 -1
- package/dist/commands/query/parameter.d.ts +26 -0
- package/dist/commands/query/parameter.js +27 -0
- package/dist/commands/query/prewarm.d.ts +31 -0
- package/dist/commands/query/prewarm.js +31 -0
- package/dist/commands/revalidate.js +8 -6
- package/dist/helpers/clipboard.d.ts +19 -0
- package/dist/helpers/clipboard.js +70 -0
- package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
- package/dist/helpers/evaluate-speed-kit-config.js +8 -6
- package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
- package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
- package/dist/models/cli-parameters.d.ts +73 -0
- package/dist/models/cli-parameters.js +151 -0
- package/dist/models/cli-parameters.spec.d.ts +1 -0
- package/dist/models/cli-parameters.spec.js +47 -0
- package/dist/services/athena/athena-service.d.ts +2 -0
- package/dist/services/athena/athena-service.js +15 -4
- package/dist/services/athena/athena-service.spec.d.ts +1 -0
- package/dist/services/athena/athena-service.spec.js +74 -0
- package/dist/services/deploy/handler/install-resource-handler.js +3 -4
- package/dist/services/onboarding/onboarding-service-factory.js +2 -2
- package/dist/services/pop-config/pop-config-service.d.ts +8 -1
- package/dist/services/pop-config/pop-config-service.js +16 -28
- package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
- package/dist/services/prewarm/assets/asset-api-client.js +5 -4
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +12 -2
- package/dist/services/prewarm/auto-pre-warm-factory.js +20 -6
- package/dist/services/prewarm/csv-reader.d.ts +21 -2
- package/dist/services/prewarm/csv-reader.js +71 -9
- package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
- package/dist/services/prewarm/csv-reader.spec.js +75 -0
- package/dist/services/prewarm/index.d.ts +1 -0
- package/dist/services/prewarm/index.js +1 -0
- package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
- package/dist/services/prewarm/pre-warm-factory.js +8 -11
- package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
- package/dist/services/prewarm/pre-warm-model.js +0 -1
- package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
- package/dist/services/prewarm/pre-warm-service.js +18 -14
- package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
- package/dist/services/prewarm/prewarm-targets.js +18 -0
- package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
- package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
- package/dist/services/query-builder/queries/device.d.ts +21 -0
- package/dist/services/query-builder/queries/device.js +26 -0
- package/dist/services/query-builder/queries/index.d.ts +6 -0
- package/dist/services/query-builder/queries/index.js +6 -0
- package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
- package/dist/services/query-builder/queries/page-filter.js +135 -0
- package/dist/services/query-builder/queries/page-source.d.ts +30 -0
- package/dist/services/query-builder/queries/page-source.js +79 -0
- package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
- package/dist/services/query-builder/queries/parameter-query.js +142 -0
- package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
- package/dist/services/query-builder/queries/pops-query.js +65 -0
- package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
- package/dist/services/query-builder/queries/prewarm-query.js +227 -0
- package/dist/services/query-builder/queries/query-model.d.ts +97 -0
- package/dist/services/query-builder/queries/query-model.js +19 -0
- package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
- package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
- package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
- package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
- package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
- package/dist/services/query-builder/query-builder-factory.js +35 -7
- package/dist/services/query-builder/query-builder-model.d.ts +31 -19
- package/dist/services/query-builder/query-builder-model.js +5 -5
- package/dist/services/query-builder/query-builder-service.d.ts +26 -5
- package/dist/services/query-builder/query-builder-service.js +157 -44
- package/dist/services/query-builder/query-command.d.ts +26 -0
- package/dist/services/query-builder/query-command.js +35 -0
- package/dist/services/query-builder/rules/index.d.ts +8 -0
- package/dist/services/query-builder/rules/index.js +8 -0
- package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
- package/dist/services/query-builder/rules/match-pattern.js +141 -0
- package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
- package/dist/services/query-builder/rules/pattern-language.js +159 -0
- package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
- package/dist/services/query-builder/rules/rule-model.js +54 -0
- package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
- package/dist/services/query-builder/rules/rule-parser.js +189 -0
- package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
- package/dist/services/query-builder/rules/rule-reducer.js +383 -0
- package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
- package/dist/services/query-builder/rules/rule-relevance.js +69 -0
- package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
- package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
- package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
- package/dist/services/query-builder/rules/strip-parameters.js +197 -0
- package/dist/services/query-builder/sql/expression.d.ts +73 -0
- package/dist/services/query-builder/sql/expression.js +133 -0
- package/dist/services/query-builder/sql/format.d.ts +14 -0
- package/dist/services/query-builder/sql/format.js +48 -0
- package/dist/services/query-builder/sql/index.d.ts +4 -0
- package/dist/services/query-builder/sql/index.js +4 -0
- package/dist/services/query-builder/sql/predicate.d.ts +66 -0
- package/dist/services/query-builder/sql/predicate.js +124 -0
- package/dist/services/query-builder/sql/regex.d.ts +28 -0
- package/dist/services/query-builder/sql/regex.js +81 -0
- package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
- package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
- package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
- package/dist/services/revalidate/revalidate-model.d.ts +9 -3
- package/dist/services/revalidate/revalidate-model.js +12 -3
- package/oclif.manifest.json +354 -104
- package/package.json +11 -2
- package/dist/commands/build-parameter-query.d.ts +0 -14
- package/dist/commands/build-parameter-query.js +0 -25
- package/dist/commands/build-prewarm-query.d.ts +0 -16
- package/dist/commands/build-prewarm-query.js +0 -37
- package/dist/helpers/build-query-helper.d.ts +0 -32
- package/dist/helpers/build-query-helper.js +0 -223
- package/dist/helpers/get-parsed-config.d.ts +0 -15
- package/dist/helpers/get-parsed-config.js +0 -62
- package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
- package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
|
@@ -17,15 +17,26 @@ export class AwsAthenaService {
|
|
|
17
17
|
}
|
|
18
18
|
async getResult(query, parameters = [], maxAgeInMinutes = DEFAULT_MAX_AGE_IN_MINUTES) {
|
|
19
19
|
this.cli.startAction("ATHENA:QUERY", "execute athena query");
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
// Everything the action covers is inside the guard, running the query
|
|
21
|
+
// included: Athena reports a rejected query by failing the execution, and
|
|
22
|
+
// a throw that escapes here leaves the spinner running — which is a timer,
|
|
23
|
+
// so the process stays alive with nothing left to do and has to be killed
|
|
24
|
+
// by hand.
|
|
25
|
+
const result = await safe(this.runAndFetch(query, parameters, maxAgeInMinutes));
|
|
22
26
|
if (result.success === true) {
|
|
23
27
|
this.cli.successAction("ATHENA:QUERY");
|
|
24
28
|
return result.data;
|
|
25
29
|
}
|
|
30
|
+
// Rethrown rather than answered with an empty result: no rows and no rows
|
|
31
|
+
// found look the same to a caller, and a pre-warm list that silently came
|
|
32
|
+
// back empty is worse than one that failed.
|
|
26
33
|
this.cli.failAction("ATHENA:QUERY");
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
throw result.errorObj ?? new Error(result.error);
|
|
35
|
+
}
|
|
36
|
+
/** The query, from sending it off to the rows it produced. */
|
|
37
|
+
async runAndFetch(query, parameters, maxAgeInMinutes) {
|
|
38
|
+
const queryId = await this.runQuery(query, parameters, maxAgeInMinutes);
|
|
39
|
+
return this.fetchResult(queryId);
|
|
29
40
|
}
|
|
30
41
|
castValueToType(value, type) {
|
|
31
42
|
if (value === null)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { GetQueryResultsCommand, StartQueryExecutionCommand, } from "@aws-sdk/client-athena";
|
|
2
|
+
import { expect } from "chai";
|
|
3
|
+
import { describe, it } from "mocha";
|
|
4
|
+
import { AwsAthenaService } from "./athena-service.js";
|
|
5
|
+
/** The cli calls the service makes, in the order it made them. */
|
|
6
|
+
function recordingCli() {
|
|
7
|
+
const calls = [];
|
|
8
|
+
const cli = {
|
|
9
|
+
failAction: (name) => calls.push(`fail:${name}`),
|
|
10
|
+
startAction: (name) => calls.push(`start:${name}`),
|
|
11
|
+
successAction: (name) => calls.push(`success:${name}`),
|
|
12
|
+
writeError: (message) => calls.push(`error:${message}`),
|
|
13
|
+
};
|
|
14
|
+
return { calls, cli };
|
|
15
|
+
}
|
|
16
|
+
/** An Athena that answers every execution with the given state. */
|
|
17
|
+
function clientAnswering(state, reason) {
|
|
18
|
+
return {
|
|
19
|
+
send: async (command) => {
|
|
20
|
+
if (command instanceof StartQueryExecutionCommand) {
|
|
21
|
+
return { QueryExecutionId: "query-1" };
|
|
22
|
+
}
|
|
23
|
+
if (command instanceof GetQueryResultsCommand) {
|
|
24
|
+
return {
|
|
25
|
+
ResultSet: {
|
|
26
|
+
ResultSetMetadata: {
|
|
27
|
+
ColumnInfo: [{ Name: "urls", Type: "bigint" }],
|
|
28
|
+
},
|
|
29
|
+
Rows: [
|
|
30
|
+
{ Data: [{ VarCharValue: "urls" }] },
|
|
31
|
+
{ Data: [{ VarCharValue: "42" }] },
|
|
32
|
+
],
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
QueryExecution: { Status: { State: state, StateChangeReason: reason } },
|
|
38
|
+
};
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
describe("AwsAthenaService", () => {
|
|
43
|
+
it("should return the rows of a query that succeeded", async () => {
|
|
44
|
+
const { calls, cli } = recordingCli();
|
|
45
|
+
const service = new AwsAthenaService(clientAnswering("SUCCEEDED"), cli);
|
|
46
|
+
expect(await service.getResult("select 1", [], 0)).to.deep.equal([
|
|
47
|
+
{ urls: 42 },
|
|
48
|
+
]);
|
|
49
|
+
expect(calls).to.deep.equal(["start:ATHENA:QUERY", "success:ATHENA:QUERY"]);
|
|
50
|
+
});
|
|
51
|
+
it("should end the action when the query itself failed", async () => {
|
|
52
|
+
// A failed execution throws from inside the action. Left unhandled it
|
|
53
|
+
// leaves the spinner running, and the spinner is a timer: the process then
|
|
54
|
+
// never exits and has to be killed by hand.
|
|
55
|
+
const { calls, cli } = recordingCli();
|
|
56
|
+
const service = new AwsAthenaService(clientAnswering("FAILED", "INVALID_ARGUMENTS: nope"), cli);
|
|
57
|
+
const result = await service
|
|
58
|
+
.getResult("select 1", [], 0)
|
|
59
|
+
.then(() => undefined)
|
|
60
|
+
.catch((error) => error);
|
|
61
|
+
expect(result).to.be.an("error");
|
|
62
|
+
expect(result.message).to.contain("INVALID_ARGUMENTS: nope");
|
|
63
|
+
expect(calls).to.deep.equal(["start:ATHENA:QUERY", "fail:ATHENA:QUERY"]);
|
|
64
|
+
});
|
|
65
|
+
it("should not pass a failed query off as an empty result", async () => {
|
|
66
|
+
const { cli } = recordingCli();
|
|
67
|
+
const service = new AwsAthenaService(clientAnswering("CANCELLED"), cli);
|
|
68
|
+
const result = await service
|
|
69
|
+
.getResult("select 1", [], 0)
|
|
70
|
+
.then(() => "resolved")
|
|
71
|
+
.catch(() => "rejected");
|
|
72
|
+
expect(result).to.equal("rejected");
|
|
73
|
+
});
|
|
74
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FILE_TYPE, } from "../../integration-api/index.js";
|
|
2
|
+
import { copyToClipboard } from "../../../helpers/clipboard.js";
|
|
2
3
|
import { safe } from "../../../helpers/safe.js";
|
|
3
|
-
import clipboardy from "clipboardy";
|
|
4
4
|
export default class InstallResourceHandler {
|
|
5
5
|
customer;
|
|
6
6
|
configApi;
|
|
@@ -67,11 +67,10 @@ export default class InstallResourceHandler {
|
|
|
67
67
|
}
|
|
68
68
|
this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST");
|
|
69
69
|
const debugParameter = `?debugSpeedKit=${install.id.split("/").pop()}`;
|
|
70
|
-
|
|
71
|
-
clipboardy.writeSync(debugParameter);
|
|
70
|
+
if (await copyToClipboard(debugParameter)) {
|
|
72
71
|
this.cli.writeSuccess("Query parameter to activate the testing installation has been copied to the clipboard 📋");
|
|
73
72
|
}
|
|
74
|
-
|
|
73
|
+
else {
|
|
75
74
|
this.cli.writeSuccess(`Use the following parameter to active the testing installation in your browser: ${debugParameter}`);
|
|
76
75
|
}
|
|
77
76
|
}
|
|
@@ -41,7 +41,7 @@ import { ExtensionDownloader } from "./browser/extension/extension-downloader.js
|
|
|
41
41
|
import { ExecutableValidator } from "./browser/executable/executable-validator.js";
|
|
42
42
|
import { applyOriginProxy, withOriginProxyFlag, } from "../../helpers/origin-proxy.js";
|
|
43
43
|
import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
|
|
44
|
-
import {
|
|
44
|
+
import { QueryType } from "../query-builder/query-builder-model.js";
|
|
45
45
|
export class OnboardingServiceFactory {
|
|
46
46
|
context;
|
|
47
47
|
cliConfig;
|
|
@@ -155,7 +155,7 @@ export class OnboardingServiceFactory {
|
|
|
155
155
|
configName: this.context.configName,
|
|
156
156
|
customerPath: this.context.customerPath,
|
|
157
157
|
isWindows: this.cliConfig.runtimeConfig.windows,
|
|
158
|
-
}, this.cliConfig).buildService(
|
|
158
|
+
}, this.cliConfig).buildService(QueryType.parameter);
|
|
159
159
|
const DiffService = new DiffServiceFactory(new DiffContext(this.cliConfig.diffExec, this.cliConfig.diffExecTemplate)).getService();
|
|
160
160
|
const requestDiffService = new RequestDiffService(DiffService, documentHandler);
|
|
161
161
|
const handlers = [
|
|
@@ -3,5 +3,12 @@ import { AthenaService } from "../athena/index.js";
|
|
|
3
3
|
export declare class PopConfigService {
|
|
4
4
|
private athenaService;
|
|
5
5
|
constructor(athenaService: AthenaService);
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* The PoPs serving the app's traffic, per origin and region.
|
|
8
|
+
*
|
|
9
|
+
* @param useRum - count page-impression beacons instead of Speed Kit's own
|
|
10
|
+
* asset requests, which is what an app without production traffic needs.
|
|
11
|
+
* @param days - size of the window to look at, in days.
|
|
12
|
+
*/
|
|
13
|
+
getUsedPopsPerOrigin(app: string, useRum: boolean, days?: number): Promise<UsedPopsPerOrigin[]>;
|
|
7
14
|
}
|
|
@@ -1,36 +1,24 @@
|
|
|
1
1
|
import { AthenaBucket } from "../athena/index.js";
|
|
2
|
+
import { buildPopsQuery } from "../query-builder/queries/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* How long a result may be reused. The PoP distribution of an app barely
|
|
5
|
+
* moves within a day, and the query scans a week of request logs.
|
|
6
|
+
*/
|
|
7
|
+
const MAX_AGE_IN_MINUTES = 12 * 60;
|
|
2
8
|
export class PopConfigService {
|
|
3
9
|
athenaService;
|
|
4
10
|
constructor(athenaService) {
|
|
5
11
|
this.athenaService = athenaService;
|
|
6
12
|
}
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
and client.bot is null
|
|
18
|
-
and origin not in ('Unknown', ''))
|
|
19
|
-
|
|
20
|
-
select region,
|
|
21
|
-
pop,
|
|
22
|
-
coalesce(originWithPrefix, origin) as origin,
|
|
23
|
-
count() as hits
|
|
24
|
-
from data
|
|
25
|
-
group by grouping sets (
|
|
26
|
-
(region, pop, origin),
|
|
27
|
-
(region, pop, origin, originWithPrefix)
|
|
28
|
-
)
|
|
29
|
-
having originWithPrefix is null or originWithPrefix <> 'Unknown'`;
|
|
30
|
-
const endDate = new Date();
|
|
31
|
-
const startDate = new Date(endDate.getFullYear(), endDate.getMonth(), endDate.getDate() - 7);
|
|
32
|
-
const start = startDate.toISOString().slice(0, 10);
|
|
33
|
-
const end = endDate.toISOString().slice(0, 10);
|
|
34
|
-
return await this.athenaService.getResult(query, [start, end, app], 12 * 60, AthenaBucket.FastlyLogs);
|
|
13
|
+
/**
|
|
14
|
+
* The PoPs serving the app's traffic, per origin and region.
|
|
15
|
+
*
|
|
16
|
+
* @param useRum - count page-impression beacons instead of Speed Kit's own
|
|
17
|
+
* asset requests, which is what an app without production traffic needs.
|
|
18
|
+
* @param days - size of the window to look at, in days.
|
|
19
|
+
*/
|
|
20
|
+
async getUsedPopsPerOrigin(app, useRum, days) {
|
|
21
|
+
const { parameters, sql } = buildPopsQuery({ app, days, useRum });
|
|
22
|
+
return await this.athenaService.getResult(sql, parameters, MAX_AGE_IN_MINUTES, AthenaBucket.FastlyLogs);
|
|
35
23
|
}
|
|
36
24
|
}
|
|
@@ -26,7 +26,7 @@ export declare class AssetApiClient {
|
|
|
26
26
|
* handle parameter "bqvariation"
|
|
27
27
|
* reflect behaviour of SpeedKit.
|
|
28
28
|
* - do not send default variation
|
|
29
|
-
* - send mobile/tablet
|
|
29
|
+
* - send mobile/tablet as lowercase
|
|
30
30
|
* - send customVariations as they are
|
|
31
31
|
*
|
|
32
32
|
* As orestes will always make variations uppercase and does not care for caseSensitivity,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { URL } from "node:url";
|
|
2
2
|
import http2 from "node:http2";
|
|
3
|
-
import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR,
|
|
3
|
+
import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR, TIMING_HEADER_KEY, } from "../index.js";
|
|
4
|
+
import { DEVICE_VARIATIONS, } from "../../query-builder/queries/device.js";
|
|
4
5
|
import { normalize } from "../../../helpers/normalize.js";
|
|
5
6
|
import { InvalidOriginError } from "../error/invalid-origin-error.js";
|
|
6
7
|
export class AssetApiClient {
|
|
@@ -114,7 +115,7 @@ export class AssetApiClient {
|
|
|
114
115
|
* handle parameter "bqvariation"
|
|
115
116
|
* reflect behaviour of SpeedKit.
|
|
116
117
|
* - do not send default variation
|
|
117
|
-
* - send mobile/tablet
|
|
118
|
+
* - send mobile/tablet as lowercase
|
|
118
119
|
* - send customVariations as they are
|
|
119
120
|
*
|
|
120
121
|
* As orestes will always make variations uppercase and does not care for caseSensitivity,
|
|
@@ -130,8 +131,8 @@ export class AssetApiClient {
|
|
|
130
131
|
if (variation === "default" || variation === "desktop") {
|
|
131
132
|
return;
|
|
132
133
|
}
|
|
133
|
-
// on
|
|
134
|
-
if (
|
|
134
|
+
// on tablet|mobile set variation as lowerCase
|
|
135
|
+
if (DEVICE_VARIATIONS.includes(variation.toLowerCase())) {
|
|
135
136
|
assetUrl.searchParams.set("bqvariation", variation.toLowerCase());
|
|
136
137
|
return;
|
|
137
138
|
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
2
|
+
import { QueryFilterOptions } from "../query-builder/query-builder-model.js";
|
|
2
3
|
import { PreWarmService } from "./pre-warm-service.js";
|
|
3
|
-
export interface AutoPreWarmContext {
|
|
4
|
+
export interface AutoPreWarmContext extends QueryFilterOptions {
|
|
4
5
|
readonly customerPath: string;
|
|
5
6
|
readonly configName: string;
|
|
6
|
-
|
|
7
|
+
/** Device classes to warm each url in the variations it was requested in. */
|
|
8
|
+
readonly device?: readonly string[];
|
|
9
|
+
/** Regular expressions the pre-warmed urls have to match. */
|
|
10
|
+
readonly match?: readonly string[];
|
|
11
|
+
/** Variations to pair every url with, when no traffic can reveal them. */
|
|
12
|
+
readonly variation?: readonly string[];
|
|
7
13
|
readonly variationPath?: string;
|
|
8
14
|
readonly sortParameters?: boolean;
|
|
9
15
|
readonly quiet?: boolean;
|
|
@@ -16,6 +22,10 @@ export interface AutoPreWarmContext {
|
|
|
16
22
|
* config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
|
|
17
23
|
* against Athena (Baqend native path, AWS fallback), and feeds the resulting
|
|
18
24
|
* URLs straight into the prewarm job.
|
|
25
|
+
*
|
|
26
|
+
* `--device` and `--variation` are handed to the query, which returns the
|
|
27
|
+
* variation alongside every url; without them the query returns urls and the
|
|
28
|
+
* list is crossed with `--variationPath` or the default variation, as before.
|
|
19
29
|
*/
|
|
20
30
|
export declare class AutoPreWarmFactory {
|
|
21
31
|
private context;
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import { CliContext, CliServiceFactory } from "../cli/index.js";
|
|
2
2
|
import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
|
|
3
|
-
import {
|
|
3
|
+
import { QueryType, } from "../query-builder/query-builder-model.js";
|
|
4
|
+
import { normalizeDevices } from "../query-builder/queries/device.js";
|
|
4
5
|
import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
|
|
5
6
|
import { AssetApiClient } from "./assets/asset-api-client.js";
|
|
6
7
|
import { CsvReader } from "./csv-reader.js";
|
|
7
8
|
import { PreWarmService } from "./pre-warm-service.js";
|
|
8
9
|
import { resolveVariations } from "./pre-warm-factory.js";
|
|
10
|
+
import { toPrewarmTargets } from "./prewarm-targets.js";
|
|
9
11
|
/**
|
|
10
12
|
* Wires a {@link PreWarmService} that gets its URLs from a freshly built and
|
|
11
13
|
* executed prewarm query instead of a CSV file. Reads the app from the customer
|
|
12
14
|
* config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
|
|
13
15
|
* against Athena (Baqend native path, AWS fallback), and feeds the resulting
|
|
14
16
|
* URLs straight into the prewarm job.
|
|
17
|
+
*
|
|
18
|
+
* `--device` and `--variation` are handed to the query, which returns the
|
|
19
|
+
* variation alongside every url; without them the query returns urls and the
|
|
20
|
+
* list is crossed with `--variationPath` or the default variation, as before.
|
|
15
21
|
*/
|
|
16
22
|
export class AutoPreWarmFactory {
|
|
17
23
|
context;
|
|
@@ -25,18 +31,26 @@ export class AutoPreWarmFactory {
|
|
|
25
31
|
const queryService = await new QueryBuilderFactory({
|
|
26
32
|
configName: this.context.configName,
|
|
27
33
|
customerPath: this.context.customerPath,
|
|
34
|
+
days: this.context.days,
|
|
35
|
+
devices: normalizeDevices(this.context.device),
|
|
36
|
+
excludeNoindex: this.context.excludeNoindex,
|
|
37
|
+
excludeSuspicious: this.context.excludeSuspicious,
|
|
38
|
+
execute: true,
|
|
39
|
+
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
28
40
|
isWindows: this.context.isWindows,
|
|
41
|
+
match: this.context.match,
|
|
29
42
|
quiet: this.context.quiet,
|
|
30
|
-
|
|
31
|
-
}, this.userConfig).buildService(
|
|
43
|
+
variations: this.context.variation,
|
|
44
|
+
}, this.userConfig).buildService(QueryType.prewarm);
|
|
32
45
|
const app = queryService.app;
|
|
33
46
|
const rows = await queryService.execute();
|
|
34
|
-
const urls = rows.map((row) => row.url).filter(Boolean);
|
|
35
47
|
const configApi = new ConfigApiServiceFactory(new ConfigApiContext(app)).getService();
|
|
36
48
|
const entityManager = await configApi.client.getEntityManager();
|
|
37
49
|
const assetApiClient = new AssetApiClient(entityManager.token, app, this.context.sortParameters);
|
|
38
50
|
const csvReader = new CsvReader();
|
|
39
|
-
|
|
40
|
-
|
|
51
|
+
return new PreWarmService(assetApiClient, cli, toPrewarmTargets(rows.filter((row) => row.url),
|
|
52
|
+
// `--variation` was answered by the query already; a variations file
|
|
53
|
+
// is still applied here.
|
|
54
|
+
resolveVariations(csvReader, undefined, this.context.variationPath)), this.context.verbose);
|
|
41
55
|
}
|
|
42
56
|
}
|
|
@@ -1,5 +1,24 @@
|
|
|
1
|
+
import { PrewarmRow } from "./pre-warm-model.js";
|
|
1
2
|
export declare class CsvReader {
|
|
2
|
-
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* The pre-warm list of a csv: one url per line, together with the variation
|
|
5
|
+
* to warm it in where the file names one.
|
|
6
|
+
*
|
|
7
|
+
* A second column only carries a variation when the header says so. The file
|
|
8
|
+
* `sk query prewarm --execute` writes for an unsplit list has the hit count
|
|
9
|
+
* there, and a list without a header at all is what it always was — bare
|
|
10
|
+
* urls, all of them to be warmed in whatever variations the caller asks for.
|
|
11
|
+
*/
|
|
12
|
+
readList(filePath: string): PrewarmRow[];
|
|
13
|
+
/**
|
|
14
|
+
* The first column of a csv, for a file that is a plain list of values —
|
|
15
|
+
* the variations to warm every url in.
|
|
16
|
+
*/
|
|
17
|
+
readValues(filePath: string): string[];
|
|
18
|
+
/**
|
|
19
|
+
* The fields of a csv line. A quoted field is taken as it stands, so a comma
|
|
20
|
+
* inside a url stays part of it, and its doubled quotes are one quote.
|
|
21
|
+
*/
|
|
22
|
+
private fields;
|
|
4
23
|
private getEntries;
|
|
5
24
|
}
|
|
@@ -3,22 +3,84 @@ import path from "node:path";
|
|
|
3
3
|
import { ReadCsvError } from "./error/read-csv-error.js";
|
|
4
4
|
const ENCODING = "utf8";
|
|
5
5
|
const LINE_BREAK = /\r?\n/;
|
|
6
|
+
/** The header the query command writes above a split pre-warm list. */
|
|
7
|
+
const VARIATION_COLUMN = "variation";
|
|
6
8
|
export class CsvReader {
|
|
7
|
-
|
|
9
|
+
/**
|
|
10
|
+
* The pre-warm list of a csv: one url per line, together with the variation
|
|
11
|
+
* to warm it in where the file names one.
|
|
12
|
+
*
|
|
13
|
+
* A second column only carries a variation when the header says so. The file
|
|
14
|
+
* `sk query prewarm --execute` writes for an unsplit list has the hit count
|
|
15
|
+
* there, and a list without a header at all is what it always was — bare
|
|
16
|
+
* urls, all of them to be warmed in whatever variations the caller asks for.
|
|
17
|
+
*/
|
|
18
|
+
readList(filePath) {
|
|
8
19
|
try {
|
|
9
|
-
const
|
|
10
|
-
|
|
20
|
+
const lines = this.getEntries(filePath).map((line) => this.fields(line));
|
|
21
|
+
const [header] = lines;
|
|
22
|
+
const hasHeader = header?.[0]?.toLowerCase() === "url";
|
|
23
|
+
const carriesVariation = hasHeader && header[1]?.toLowerCase() === VARIATION_COLUMN;
|
|
24
|
+
return (hasHeader ? lines.slice(1) : lines)
|
|
25
|
+
.filter(([url]) => url)
|
|
26
|
+
.map(([url, second]) => carriesVariation && second ? { url, variation: second } : { url });
|
|
11
27
|
}
|
|
12
28
|
catch (error) {
|
|
13
29
|
throw new ReadCsvError(error.message);
|
|
14
30
|
}
|
|
15
31
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
32
|
+
/**
|
|
33
|
+
* The first column of a csv, for a file that is a plain list of values —
|
|
34
|
+
* the variations to warm every url in.
|
|
35
|
+
*/
|
|
36
|
+
readValues(filePath) {
|
|
37
|
+
try {
|
|
38
|
+
return this.getEntries(filePath)
|
|
39
|
+
.map((line) => this.fields(line)[0])
|
|
40
|
+
.filter(Boolean);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw new ReadCsvError(error.message);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The fields of a csv line. A quoted field is taken as it stands, so a comma
|
|
48
|
+
* inside a url stays part of it, and its doubled quotes are one quote.
|
|
49
|
+
*/
|
|
50
|
+
fields(line) {
|
|
51
|
+
const entry = line.trim();
|
|
52
|
+
const fields = [];
|
|
53
|
+
let field = "";
|
|
54
|
+
let quoted = false;
|
|
55
|
+
for (let index = 0; index < entry.length; index++) {
|
|
56
|
+
const character = entry[index];
|
|
57
|
+
if (quoted) {
|
|
58
|
+
if (character !== '"') {
|
|
59
|
+
field += character;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
// A doubled quote is one quote; a single one ends the field.
|
|
63
|
+
if (entry[index + 1] === '"') {
|
|
64
|
+
field += '"';
|
|
65
|
+
index++;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
quoted = false;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
if (character === '"' && field === "") {
|
|
72
|
+
quoted = true;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (character === ",") {
|
|
76
|
+
fields.push(field.trim());
|
|
77
|
+
field = "";
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
field += character;
|
|
81
|
+
}
|
|
82
|
+
fields.push(field.trim());
|
|
83
|
+
return fields;
|
|
22
84
|
}
|
|
23
85
|
getEntries(filePath) {
|
|
24
86
|
const fileContent = fs.readFileSync(path.resolve(filePath), ENCODING);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { mkdtempSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { expect } from "chai";
|
|
5
|
+
import { describe, it } from "mocha";
|
|
6
|
+
import { CsvReader } from "./csv-reader.js";
|
|
7
|
+
function write(content) {
|
|
8
|
+
const file = join(mkdtempSync(join(tmpdir(), "sk-csv-")), "urls.csv");
|
|
9
|
+
writeFileSync(file, content, "utf8");
|
|
10
|
+
return file;
|
|
11
|
+
}
|
|
12
|
+
/** The urls of a list, for the cases that say nothing about variations. */
|
|
13
|
+
function read(content) {
|
|
14
|
+
return readList(content).map(({ url }) => url);
|
|
15
|
+
}
|
|
16
|
+
function readList(content) {
|
|
17
|
+
return new CsvReader().readList(write(content));
|
|
18
|
+
}
|
|
19
|
+
describe("CsvReader", () => {
|
|
20
|
+
it("should read a bare list of urls", () => {
|
|
21
|
+
expect(read("https://a.de/\nhttps://b.de/\n")).to.deep.equal([
|
|
22
|
+
"https://a.de/",
|
|
23
|
+
"https://b.de/",
|
|
24
|
+
]);
|
|
25
|
+
});
|
|
26
|
+
it("should read quoted urls", () => {
|
|
27
|
+
expect(read('"https://a.de/"\n"https://b.de/"')).to.deep.equal([
|
|
28
|
+
"https://a.de/",
|
|
29
|
+
"https://b.de/",
|
|
30
|
+
]);
|
|
31
|
+
});
|
|
32
|
+
it("should take only the first column", () => {
|
|
33
|
+
// What `sk query prewarm --execute` writes: url and hit count.
|
|
34
|
+
expect(read('"url","hits"\n"https://a.de/",12\n"https://b.de/",3')).to.deep.eq(["https://a.de/", "https://b.de/"]);
|
|
35
|
+
});
|
|
36
|
+
it("should skip a header row", () => {
|
|
37
|
+
expect(read("url\nhttps://a.de/")).to.deep.equal(["https://a.de/"]);
|
|
38
|
+
});
|
|
39
|
+
it("should keep a comma inside a quoted url", () => {
|
|
40
|
+
expect(read('"https://a.de/?list=1,2",7')).to.deep.equal([
|
|
41
|
+
"https://a.de/?list=1,2",
|
|
42
|
+
]);
|
|
43
|
+
});
|
|
44
|
+
it("should read a doubled quote as one quote", () => {
|
|
45
|
+
expect(read('"https://a.de/?q=""x"""')).to.deep.equal([
|
|
46
|
+
'https://a.de/?q="x"',
|
|
47
|
+
]);
|
|
48
|
+
});
|
|
49
|
+
it("should drop empty lines", () => {
|
|
50
|
+
expect(read("https://a.de/\n\n\nhttps://b.de/\n")).to.have.lengthOf(2);
|
|
51
|
+
});
|
|
52
|
+
it("should read the variation of a split list", () => {
|
|
53
|
+
// What `sk query prewarm --device desktop,mobile --execute` writes.
|
|
54
|
+
expect(readList('"url","variation","hits"\n"https://a.de/","mobile",12\n"https://a.de/","desktop",3')).to.deep.equal([
|
|
55
|
+
{ url: "https://a.de/", variation: "mobile" },
|
|
56
|
+
{ url: "https://a.de/", variation: "desktop" },
|
|
57
|
+
]);
|
|
58
|
+
});
|
|
59
|
+
it("should not read a hit count as a variation", () => {
|
|
60
|
+
expect(readList('"url","hits"\n"https://a.de/",12')).to.deep.equal([
|
|
61
|
+
{ url: "https://a.de/" },
|
|
62
|
+
]);
|
|
63
|
+
});
|
|
64
|
+
it("should not read a second column of a headerless list", () => {
|
|
65
|
+
expect(readList('"https://a.de/","mobile"')).to.deep.equal([
|
|
66
|
+
{ url: "https://a.de/" },
|
|
67
|
+
]);
|
|
68
|
+
});
|
|
69
|
+
it("should keep a comma inside a quoted url of a split list", () => {
|
|
70
|
+
expect(readList('url,variation\n"https://a.de/?list=1,2",mobile')).to.deep.equal([{ url: "https://a.de/?list=1,2", variation: "mobile" }]);
|
|
71
|
+
});
|
|
72
|
+
it("should read a plain list of values", () => {
|
|
73
|
+
expect(new CsvReader().readValues(write("desktop\nmobile\n"))).to.deep.equal(["desktop", "mobile"]);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
@@ -4,8 +4,6 @@ export declare class PreWarmFactory {
|
|
|
4
4
|
private context;
|
|
5
5
|
constructor(context: PreWarmContext);
|
|
6
6
|
getService(): Promise<PreWarmService>;
|
|
7
|
-
private getUrls;
|
|
8
|
-
private getVariations;
|
|
9
7
|
}
|
|
10
8
|
/**
|
|
11
9
|
* Resolve the list of variations to prewarm: a `variationPath` CSV takes
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CliContext, CliServiceFactory } from "../cli/index.js";
|
|
2
|
-
import { DEFAULT_VARIATION, PreWarmService, VARIATION_SEPARATOR, } from "./index.js";
|
|
2
|
+
import { DEFAULT_VARIATION, PreWarmService, toPrewarmTargets, VARIATION_SEPARATOR, } from "./index.js";
|
|
3
3
|
import { AssetApiClient } from "./assets/asset-api-client.js";
|
|
4
4
|
import { CsvReader } from "./csv-reader.js";
|
|
5
5
|
import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
|
|
@@ -14,15 +14,12 @@ export class PreWarmFactory {
|
|
|
14
14
|
const entityManager = await configApi.client.getEntityManager();
|
|
15
15
|
const csvReader = new CsvReader();
|
|
16
16
|
const assetApiClient = new AssetApiClient(entityManager.token, this.context.app, this.context.keepParameterSorting);
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
return csvReader.
|
|
23
|
-
}
|
|
24
|
-
getVariations(csvReader) {
|
|
25
|
-
return resolveVariations(csvReader, this.context.variation, this.context.variationPath);
|
|
17
|
+
const rows = csvReader.readList(this.context.path);
|
|
18
|
+
const named = this.context.variation ?? this.context.variationPath;
|
|
19
|
+
if (named && rows.some((row) => row.variation)) {
|
|
20
|
+
cli.writeWarning(`The list names a variation for every url, so ${named} is not used`);
|
|
21
|
+
}
|
|
22
|
+
return new PreWarmService(assetApiClient, cli, toPrewarmTargets(rows, resolveVariations(csvReader, this.context.variation, this.context.variationPath)), this.context.verbose);
|
|
26
23
|
}
|
|
27
24
|
}
|
|
28
25
|
/**
|
|
@@ -37,7 +34,7 @@ export function resolveVariations(csvReader, variation, variationPath) {
|
|
|
37
34
|
variations = variation.split(VARIATION_SEPARATOR);
|
|
38
35
|
}
|
|
39
36
|
if (variationPath) {
|
|
40
|
-
variations = csvReader.
|
|
37
|
+
variations = csvReader.readValues(variationPath);
|
|
41
38
|
}
|
|
42
39
|
return variations;
|
|
43
40
|
}
|
|
@@ -8,6 +8,20 @@ export declare class PreWarmContext {
|
|
|
8
8
|
readonly verbose: boolean;
|
|
9
9
|
constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean, verbose?: boolean);
|
|
10
10
|
}
|
|
11
|
+
/**
|
|
12
|
+
* One url to warm, in one variation. A pre-warm list is a list of these: the
|
|
13
|
+
* variation is a fact about the url, not a dimension crossed over the whole
|
|
14
|
+
* list, so a url seen only on phones is warmed only as `mobile`.
|
|
15
|
+
*/
|
|
16
|
+
export interface PrewarmTarget {
|
|
17
|
+
readonly url: string;
|
|
18
|
+
readonly variation: string;
|
|
19
|
+
}
|
|
20
|
+
/** A row of a pre-warm list, which may or may not name a variation yet. */
|
|
21
|
+
export interface PrewarmRow {
|
|
22
|
+
readonly url: string;
|
|
23
|
+
readonly variation?: string;
|
|
24
|
+
}
|
|
11
25
|
export interface AssetItemInterface {
|
|
12
26
|
done(): void;
|
|
13
27
|
error?: Error;
|
|
@@ -36,4 +50,3 @@ export declare const TIMING_HEADER_KEY = "server-timing";
|
|
|
36
50
|
export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
|
|
37
51
|
export declare const VARIATION_SEPARATOR = ",";
|
|
38
52
|
export declare const DEFAULT_VARIATION: string[];
|
|
39
|
-
export declare const DEFAULT_VARIATIONS: string[];
|
|
@@ -36,4 +36,3 @@ export const TIMING_HEADER_KEY = "server-timing";
|
|
|
36
36
|
export const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
|
|
37
37
|
export const VARIATION_SEPARATOR = ",";
|
|
38
38
|
export const DEFAULT_VARIATION = ["default"];
|
|
39
|
-
export const DEFAULT_VARIATIONS = ["mobile", "tablet", "tv"];
|