@speedkit/cli 4.26.0 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +203 -15
- package/dist/commands/auto-prewarm.d.ts +12 -1
- package/dist/commands/auto-prewarm.js +26 -7
- package/dist/commands/generate-pop-config.d.ts +1 -0
- package/dist/commands/generate-pop-config.js +9 -3
- package/dist/commands/prewarm.js +3 -1
- package/dist/commands/query/parameter.d.ts +27 -0
- package/dist/commands/query/parameter.js +28 -0
- package/dist/commands/query/prewarm.d.ts +32 -0
- package/dist/commands/query/prewarm.js +32 -0
- package/dist/helpers/clipboard.d.ts +19 -0
- package/dist/helpers/clipboard.js +70 -0
- package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
- package/dist/helpers/evaluate-speed-kit-config.js +8 -6
- package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
- package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
- package/dist/models/cli-parameters.d.ts +106 -0
- package/dist/models/cli-parameters.js +183 -0
- package/dist/models/cli-parameters.spec.d.ts +1 -0
- package/dist/models/cli-parameters.spec.js +47 -0
- package/dist/services/athena/athena-service-factory.d.ts +4 -1
- package/dist/services/athena/athena-service-factory.js +9 -3
- package/dist/services/athena/athena-service.d.ts +13 -1
- package/dist/services/athena/athena-service.js +27 -5
- package/dist/services/athena/athena-service.spec.d.ts +1 -0
- package/dist/services/athena/athena-service.spec.js +93 -0
- package/dist/services/deploy/handler/install-resource-handler.js +3 -4
- package/dist/services/onboarding/onboarding-service-factory.js +2 -2
- package/dist/services/pop-config/pop-config-service.d.ts +8 -1
- package/dist/services/pop-config/pop-config-service.js +16 -28
- package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
- package/dist/services/prewarm/assets/asset-api-client.js +5 -4
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +20 -2
- package/dist/services/prewarm/auto-pre-warm-factory.js +24 -6
- package/dist/services/prewarm/csv-reader.d.ts +21 -2
- package/dist/services/prewarm/csv-reader.js +71 -9
- package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
- package/dist/services/prewarm/csv-reader.spec.js +75 -0
- package/dist/services/prewarm/index.d.ts +1 -0
- package/dist/services/prewarm/index.js +1 -0
- package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
- package/dist/services/prewarm/pre-warm-factory.js +8 -11
- package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
- package/dist/services/prewarm/pre-warm-model.js +0 -1
- package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
- package/dist/services/prewarm/pre-warm-service.js +18 -14
- package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
- package/dist/services/prewarm/prewarm-targets.js +18 -0
- package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
- package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
- package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
- package/dist/services/query-builder/queries/device.d.ts +21 -0
- package/dist/services/query-builder/queries/device.js +26 -0
- package/dist/services/query-builder/queries/index.d.ts +6 -0
- package/dist/services/query-builder/queries/index.js +6 -0
- package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
- package/dist/services/query-builder/queries/page-filter.js +135 -0
- package/dist/services/query-builder/queries/page-source.d.ts +30 -0
- package/dist/services/query-builder/queries/page-source.js +79 -0
- package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
- package/dist/services/query-builder/queries/parameter-query.js +142 -0
- package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
- package/dist/services/query-builder/queries/pops-query.js +65 -0
- package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
- package/dist/services/query-builder/queries/prewarm-query.js +227 -0
- package/dist/services/query-builder/queries/query-model.d.ts +97 -0
- package/dist/services/query-builder/queries/query-model.js +19 -0
- package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
- package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
- package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
- package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
- package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
- package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
- package/dist/services/query-builder/query-builder-factory.js +36 -8
- package/dist/services/query-builder/query-builder-model.d.ts +33 -19
- package/dist/services/query-builder/query-builder-model.js +5 -5
- package/dist/services/query-builder/query-builder-service.d.ts +26 -5
- package/dist/services/query-builder/query-builder-service.js +157 -44
- package/dist/services/query-builder/query-command.d.ts +27 -0
- package/dist/services/query-builder/query-command.js +36 -0
- package/dist/services/query-builder/rules/index.d.ts +8 -0
- package/dist/services/query-builder/rules/index.js +8 -0
- package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
- package/dist/services/query-builder/rules/match-pattern.js +141 -0
- package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
- package/dist/services/query-builder/rules/pattern-language.js +159 -0
- package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
- package/dist/services/query-builder/rules/rule-model.js +54 -0
- package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
- package/dist/services/query-builder/rules/rule-parser.js +189 -0
- package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
- package/dist/services/query-builder/rules/rule-reducer.js +383 -0
- package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
- package/dist/services/query-builder/rules/rule-relevance.js +69 -0
- package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
- package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
- package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
- package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
- package/dist/services/query-builder/rules/strip-parameters.js +197 -0
- package/dist/services/query-builder/sql/expression.d.ts +73 -0
- package/dist/services/query-builder/sql/expression.js +133 -0
- package/dist/services/query-builder/sql/format.d.ts +14 -0
- package/dist/services/query-builder/sql/format.js +48 -0
- package/dist/services/query-builder/sql/index.d.ts +4 -0
- package/dist/services/query-builder/sql/index.js +4 -0
- package/dist/services/query-builder/sql/predicate.d.ts +66 -0
- package/dist/services/query-builder/sql/predicate.js +124 -0
- package/dist/services/query-builder/sql/regex.d.ts +28 -0
- package/dist/services/query-builder/sql/regex.js +81 -0
- package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
- package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
- package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
- package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
- package/oclif.manifest.json +401 -100
- package/package.json +11 -2
- package/dist/commands/build-parameter-query.d.ts +0 -14
- package/dist/commands/build-parameter-query.js +0 -25
- package/dist/commands/build-prewarm-query.d.ts +0 -16
- package/dist/commands/build-prewarm-query.js +0 -37
- package/dist/helpers/build-query-helper.d.ts +0 -32
- package/dist/helpers/build-query-helper.js +0 -223
- package/dist/helpers/get-parsed-config.d.ts +0 -15
- package/dist/helpers/get-parsed-config.js +0 -62
- package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
- package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
|
@@ -8,14 +8,18 @@ import { BaqendAthenaService } from "./baqend-athena-service.js";
|
|
|
8
8
|
import { EntityManagerFactory } from "../config-api/entity-manager-factory.js";
|
|
9
9
|
export class AthenaServiceFactory {
|
|
10
10
|
app;
|
|
11
|
+
workgroup;
|
|
11
12
|
service;
|
|
12
13
|
/**
|
|
13
14
|
* @param app - the app to query. Enables the Baqend native query path (app
|
|
14
15
|
* token only) as a fallback when no AWS login is present. Without it, only
|
|
15
16
|
* the AWS path is available.
|
|
17
|
+
* @param workgroup - Athena workgroup for the AWS path. The Baqend path has
|
|
18
|
+
* no such notion, so it is ignored there.
|
|
16
19
|
*/
|
|
17
|
-
constructor(app) {
|
|
20
|
+
constructor(app, workgroup) {
|
|
18
21
|
this.app = app;
|
|
22
|
+
this.workgroup = workgroup;
|
|
19
23
|
}
|
|
20
24
|
async getService() {
|
|
21
25
|
if (!this.service) {
|
|
@@ -33,8 +37,10 @@ export class AthenaServiceFactory {
|
|
|
33
37
|
const cli = new CliServiceFactory().getService();
|
|
34
38
|
const client = await this.tryGetAthenaClient();
|
|
35
39
|
if (client) {
|
|
36
|
-
cli.comment(
|
|
37
|
-
|
|
40
|
+
cli.comment(this.workgroup
|
|
41
|
+
? `Using AWS credentials for Athena queries, workgroup ${this.workgroup}.`
|
|
42
|
+
: "Using AWS credentials for Athena queries.");
|
|
43
|
+
return new AwsAthenaService(client, cli, this.workgroup);
|
|
38
44
|
}
|
|
39
45
|
if (this.app) {
|
|
40
46
|
return new BaqendAthenaService(this.app, new EntityManagerFactory(), cli);
|
|
@@ -10,8 +10,20 @@ import { CliServiceInterface } from "../cli/index.js";
|
|
|
10
10
|
export declare class AwsAthenaService implements AthenaService {
|
|
11
11
|
private client;
|
|
12
12
|
private cli;
|
|
13
|
-
|
|
13
|
+
/**
|
|
14
|
+
* Athena falls back to the account's default workgroup when this is unset,
|
|
15
|
+
* which is only right where the credentials are allowed to use it.
|
|
16
|
+
*/
|
|
17
|
+
private workgroup?;
|
|
18
|
+
constructor(client: AthenaClient, cli: CliServiceInterface,
|
|
19
|
+
/**
|
|
20
|
+
* Athena falls back to the account's default workgroup when this is unset,
|
|
21
|
+
* which is only right where the credentials are allowed to use it.
|
|
22
|
+
*/
|
|
23
|
+
workgroup?: string);
|
|
14
24
|
getResult<T>(query: string, parameters?: string[], maxAgeInMinutes?: number): Promise<T[]>;
|
|
25
|
+
/** The query, from sending it off to the rows it produced. */
|
|
26
|
+
private runAndFetch;
|
|
15
27
|
private castValueToType;
|
|
16
28
|
/**
|
|
17
29
|
* Fetch the query result of the given query id, if the result is still available
|
|
@@ -11,21 +11,39 @@ import { safe } from "../../helpers/safe.js";
|
|
|
11
11
|
export class AwsAthenaService {
|
|
12
12
|
client;
|
|
13
13
|
cli;
|
|
14
|
-
|
|
14
|
+
workgroup;
|
|
15
|
+
constructor(client, cli,
|
|
16
|
+
/**
|
|
17
|
+
* Athena falls back to the account's default workgroup when this is unset,
|
|
18
|
+
* which is only right where the credentials are allowed to use it.
|
|
19
|
+
*/
|
|
20
|
+
workgroup) {
|
|
15
21
|
this.client = client;
|
|
16
22
|
this.cli = cli;
|
|
23
|
+
this.workgroup = workgroup;
|
|
17
24
|
}
|
|
18
25
|
async getResult(query, parameters = [], maxAgeInMinutes = DEFAULT_MAX_AGE_IN_MINUTES) {
|
|
19
26
|
this.cli.startAction("ATHENA:QUERY", "execute athena query");
|
|
20
|
-
|
|
21
|
-
|
|
27
|
+
// Everything the action covers is inside the guard, running the query
|
|
28
|
+
// included: Athena reports a rejected query by failing the execution, and
|
|
29
|
+
// a throw that escapes here leaves the spinner running — which is a timer,
|
|
30
|
+
// so the process stays alive with nothing left to do and has to be killed
|
|
31
|
+
// by hand.
|
|
32
|
+
const result = await safe(this.runAndFetch(query, parameters, maxAgeInMinutes));
|
|
22
33
|
if (result.success === true) {
|
|
23
34
|
this.cli.successAction("ATHENA:QUERY");
|
|
24
35
|
return result.data;
|
|
25
36
|
}
|
|
37
|
+
// Rethrown rather than answered with an empty result: no rows and no rows
|
|
38
|
+
// found look the same to a caller, and a pre-warm list that silently came
|
|
39
|
+
// back empty is worse than one that failed.
|
|
26
40
|
this.cli.failAction("ATHENA:QUERY");
|
|
27
|
-
|
|
28
|
-
|
|
41
|
+
throw result.errorObj ?? new Error(result.error);
|
|
42
|
+
}
|
|
43
|
+
/** The query, from sending it off to the rows it produced. */
|
|
44
|
+
async runAndFetch(query, parameters, maxAgeInMinutes) {
|
|
45
|
+
const queryId = await this.runQuery(query, parameters, maxAgeInMinutes);
|
|
46
|
+
return this.fetchResult(queryId);
|
|
29
47
|
}
|
|
30
48
|
castValueToType(value, type) {
|
|
31
49
|
if (value === null)
|
|
@@ -97,6 +115,10 @@ export class AwsAthenaService {
|
|
|
97
115
|
MaxAgeInMinutes: maxAgeInMinutes,
|
|
98
116
|
},
|
|
99
117
|
},
|
|
118
|
+
// Omitted rather than sent empty: Athena reads an absent WorkGroup as
|
|
119
|
+
// "the default one", and an empty string as a workgroup that does not
|
|
120
|
+
// exist.
|
|
121
|
+
...(this.workgroup ? { WorkGroup: this.workgroup } : {}),
|
|
100
122
|
});
|
|
101
123
|
const response = await this.client.send(command);
|
|
102
124
|
await this.waitOnResponse(response.QueryExecutionId);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { GetQueryResultsCommand, StartQueryExecutionCommand, } from "@aws-sdk/client-athena";
|
|
2
|
+
import { expect } from "chai";
|
|
3
|
+
import { describe, it } from "mocha";
|
|
4
|
+
import { AwsAthenaService } from "./athena-service.js";
|
|
5
|
+
/** The cli calls the service makes, in the order it made them. */
|
|
6
|
+
function recordingCli() {
|
|
7
|
+
const calls = [];
|
|
8
|
+
const cli = {
|
|
9
|
+
failAction: (name) => calls.push(`fail:${name}`),
|
|
10
|
+
startAction: (name) => calls.push(`start:${name}`),
|
|
11
|
+
successAction: (name) => calls.push(`success:${name}`),
|
|
12
|
+
writeError: (message) => calls.push(`error:${message}`),
|
|
13
|
+
};
|
|
14
|
+
return { calls, cli };
|
|
15
|
+
}
|
|
16
|
+
/** An Athena that answers every execution with the given state. */
|
|
17
|
+
function clientAnswering(state, reason, sent = []) {
|
|
18
|
+
return {
|
|
19
|
+
send: async (command) => {
|
|
20
|
+
sent.push(command);
|
|
21
|
+
if (command instanceof StartQueryExecutionCommand) {
|
|
22
|
+
return { QueryExecutionId: "query-1" };
|
|
23
|
+
}
|
|
24
|
+
if (command instanceof GetQueryResultsCommand) {
|
|
25
|
+
return {
|
|
26
|
+
ResultSet: {
|
|
27
|
+
ResultSetMetadata: {
|
|
28
|
+
ColumnInfo: [{ Name: "urls", Type: "bigint" }],
|
|
29
|
+
},
|
|
30
|
+
Rows: [
|
|
31
|
+
{ Data: [{ VarCharValue: "urls" }] },
|
|
32
|
+
{ Data: [{ VarCharValue: "42" }] },
|
|
33
|
+
],
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
QueryExecution: { Status: { State: state, StateChangeReason: reason } },
|
|
39
|
+
};
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
describe("AwsAthenaService", () => {
|
|
44
|
+
it("should return the rows of a query that succeeded", async () => {
|
|
45
|
+
const { calls, cli } = recordingCli();
|
|
46
|
+
const service = new AwsAthenaService(clientAnswering("SUCCEEDED"), cli);
|
|
47
|
+
expect(await service.getResult("select 1", [], 0)).to.deep.equal([
|
|
48
|
+
{ urls: 42 },
|
|
49
|
+
]);
|
|
50
|
+
expect(calls).to.deep.equal(["start:ATHENA:QUERY", "success:ATHENA:QUERY"]);
|
|
51
|
+
});
|
|
52
|
+
it("should end the action when the query itself failed", async () => {
|
|
53
|
+
// A failed execution throws from inside the action. Left unhandled it
|
|
54
|
+
// leaves the spinner running, and the spinner is a timer: the process then
|
|
55
|
+
// never exits and has to be killed by hand.
|
|
56
|
+
const { calls, cli } = recordingCli();
|
|
57
|
+
const service = new AwsAthenaService(clientAnswering("FAILED", "INVALID_ARGUMENTS: nope"), cli);
|
|
58
|
+
const result = await service
|
|
59
|
+
.getResult("select 1", [], 0)
|
|
60
|
+
.then(() => undefined)
|
|
61
|
+
.catch((error) => error);
|
|
62
|
+
expect(result).to.be.an("error");
|
|
63
|
+
expect(result.message).to.contain("INVALID_ARGUMENTS: nope");
|
|
64
|
+
expect(calls).to.deep.equal(["start:ATHENA:QUERY", "fail:ATHENA:QUERY"]);
|
|
65
|
+
});
|
|
66
|
+
it("should run in the workgroup it was given", async () => {
|
|
67
|
+
const { cli } = recordingCli();
|
|
68
|
+
const sent = [];
|
|
69
|
+
const service = new AwsAthenaService(clientAnswering("SUCCEEDED", undefined, sent), cli, "prewarming");
|
|
70
|
+
await service.getResult("select 1", [], 0);
|
|
71
|
+
const start = sent.find((command) => command instanceof StartQueryExecutionCommand);
|
|
72
|
+
expect(start.input.WorkGroup).to.equal("prewarming");
|
|
73
|
+
});
|
|
74
|
+
it("should leave the workgroup out when none was given", async () => {
|
|
75
|
+
// Athena reads an absent WorkGroup as the account default; an empty string
|
|
76
|
+
// is a workgroup that does not exist, so the key has to be gone entirely.
|
|
77
|
+
const { cli } = recordingCli();
|
|
78
|
+
const sent = [];
|
|
79
|
+
const service = new AwsAthenaService(clientAnswering("SUCCEEDED", undefined, sent), cli);
|
|
80
|
+
await service.getResult("select 1", [], 0);
|
|
81
|
+
const start = sent.find((command) => command instanceof StartQueryExecutionCommand);
|
|
82
|
+
expect(start.input).to.not.have.property("WorkGroup");
|
|
83
|
+
});
|
|
84
|
+
it("should not pass a failed query off as an empty result", async () => {
|
|
85
|
+
const { cli } = recordingCli();
|
|
86
|
+
const service = new AwsAthenaService(clientAnswering("CANCELLED"), cli);
|
|
87
|
+
const result = await service
|
|
88
|
+
.getResult("select 1", [], 0)
|
|
89
|
+
.then(() => "resolved")
|
|
90
|
+
.catch(() => "rejected");
|
|
91
|
+
expect(result).to.equal("rejected");
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FILE_TYPE, } from "../../integration-api/index.js";
|
|
2
|
+
import { copyToClipboard } from "../../../helpers/clipboard.js";
|
|
2
3
|
import { safe } from "../../../helpers/safe.js";
|
|
3
|
-
import clipboardy from "clipboardy";
|
|
4
4
|
export default class InstallResourceHandler {
|
|
5
5
|
customer;
|
|
6
6
|
configApi;
|
|
@@ -67,11 +67,10 @@ export default class InstallResourceHandler {
|
|
|
67
67
|
}
|
|
68
68
|
this.cli.successAction("DEPLOY:ACTIVATE:INSTALL_RESOURCE_TEST");
|
|
69
69
|
const debugParameter = `?debugSpeedKit=${install.id.split("/").pop()}`;
|
|
70
|
-
|
|
71
|
-
clipboardy.writeSync(debugParameter);
|
|
70
|
+
if (await copyToClipboard(debugParameter)) {
|
|
72
71
|
this.cli.writeSuccess("Query parameter to activate the testing installation has been copied to the clipboard 📋");
|
|
73
72
|
}
|
|
74
|
-
|
|
73
|
+
else {
|
|
75
74
|
this.cli.writeSuccess(`Use the following parameter to active the testing installation in your browser: ${debugParameter}`);
|
|
76
75
|
}
|
|
77
76
|
}
|
|
@@ -41,7 +41,7 @@ import { ExtensionDownloader } from "./browser/extension/extension-downloader.js
|
|
|
41
41
|
import { ExecutableValidator } from "./browser/executable/executable-validator.js";
|
|
42
42
|
import { applyOriginProxy, withOriginProxyFlag, } from "../../helpers/origin-proxy.js";
|
|
43
43
|
import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
|
|
44
|
-
import {
|
|
44
|
+
import { QueryType } from "../query-builder/query-builder-model.js";
|
|
45
45
|
export class OnboardingServiceFactory {
|
|
46
46
|
context;
|
|
47
47
|
cliConfig;
|
|
@@ -155,7 +155,7 @@ export class OnboardingServiceFactory {
|
|
|
155
155
|
configName: this.context.configName,
|
|
156
156
|
customerPath: this.context.customerPath,
|
|
157
157
|
isWindows: this.cliConfig.runtimeConfig.windows,
|
|
158
|
-
}, this.cliConfig).buildService(
|
|
158
|
+
}, this.cliConfig).buildService(QueryType.parameter);
|
|
159
159
|
const DiffService = new DiffServiceFactory(new DiffContext(this.cliConfig.diffExec, this.cliConfig.diffExecTemplate)).getService();
|
|
160
160
|
const requestDiffService = new RequestDiffService(DiffService, documentHandler);
|
|
161
161
|
const handlers = [
|
|
@@ -3,5 +3,12 @@ import { AthenaService } from "../athena/index.js";
|
|
|
3
3
|
export declare class PopConfigService {
|
|
4
4
|
private athenaService;
|
|
5
5
|
constructor(athenaService: AthenaService);
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* The PoPs serving the app's traffic, per origin and region.
|
|
8
|
+
*
|
|
9
|
+
* @param useRum - count page-impression beacons instead of Speed Kit's own
|
|
10
|
+
* asset requests, which is what an app without production traffic needs.
|
|
11
|
+
* @param days - size of the window to look at, in days.
|
|
12
|
+
*/
|
|
13
|
+
getUsedPopsPerOrigin(app: string, useRum: boolean, days?: number): Promise<UsedPopsPerOrigin[]>;
|
|
7
14
|
}
|
|
@@ -1,36 +1,24 @@
|
|
|
1
1
|
import { AthenaBucket } from "../athena/index.js";
|
|
2
|
+
import { buildPopsQuery } from "../query-builder/queries/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* How long a result may be reused. The PoP distribution of an app barely
|
|
5
|
+
* moves within a day, and the query scans a week of request logs.
|
|
6
|
+
*/
|
|
7
|
+
const MAX_AGE_IN_MINUTES = 12 * 60;
|
|
2
8
|
export class PopConfigService {
|
|
3
9
|
athenaService;
|
|
4
10
|
constructor(athenaService) {
|
|
5
11
|
this.athenaService = athenaService;
|
|
6
12
|
}
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
and client.bot is null
|
|
18
|
-
and origin not in ('Unknown', ''))
|
|
19
|
-
|
|
20
|
-
select region,
|
|
21
|
-
pop,
|
|
22
|
-
coalesce(originWithPrefix, origin) as origin,
|
|
23
|
-
count() as hits
|
|
24
|
-
from data
|
|
25
|
-
group by grouping sets (
|
|
26
|
-
(region, pop, origin),
|
|
27
|
-
(region, pop, origin, originWithPrefix)
|
|
28
|
-
)
|
|
29
|
-
having originWithPrefix is null or originWithPrefix <> 'Unknown'`;
|
|
30
|
-
const endDate = new Date();
|
|
31
|
-
const startDate = new Date(endDate.getFullYear(), endDate.getMonth(), endDate.getDate() - 7);
|
|
32
|
-
const start = startDate.toISOString().slice(0, 10);
|
|
33
|
-
const end = endDate.toISOString().slice(0, 10);
|
|
34
|
-
return await this.athenaService.getResult(query, [start, end, app], 12 * 60, AthenaBucket.FastlyLogs);
|
|
13
|
+
/**
|
|
14
|
+
* The PoPs serving the app's traffic, per origin and region.
|
|
15
|
+
*
|
|
16
|
+
* @param useRum - count page-impression beacons instead of Speed Kit's own
|
|
17
|
+
* asset requests, which is what an app without production traffic needs.
|
|
18
|
+
* @param days - size of the window to look at, in days.
|
|
19
|
+
*/
|
|
20
|
+
async getUsedPopsPerOrigin(app, useRum, days) {
|
|
21
|
+
const { parameters, sql } = buildPopsQuery({ app, days, useRum });
|
|
22
|
+
return await this.athenaService.getResult(sql, parameters, MAX_AGE_IN_MINUTES, AthenaBucket.FastlyLogs);
|
|
35
23
|
}
|
|
36
24
|
}
|
|
@@ -26,7 +26,7 @@ export declare class AssetApiClient {
|
|
|
26
26
|
* handle parameter "bqvariation"
|
|
27
27
|
* reflect behaviour of SpeedKit.
|
|
28
28
|
* - do not send default variation
|
|
29
|
-
* - send mobile/tablet
|
|
29
|
+
* - send mobile/tablet as lowercase
|
|
30
30
|
* - send customVariations as they are
|
|
31
31
|
*
|
|
32
32
|
* As orestes will always make variations uppercase and does not care for caseSensitivity,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { URL } from "node:url";
|
|
2
2
|
import http2 from "node:http2";
|
|
3
|
-
import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR,
|
|
3
|
+
import { MAX_REQUESTS_PER_SECOND, DEFAULT_REQUESTS_PER_SECOND, RATE_LIMIT_MULTIPLICATOR, TIMING_HEADER_KEY, } from "../index.js";
|
|
4
|
+
import { DEVICE_VARIATIONS, } from "../../query-builder/queries/device.js";
|
|
4
5
|
import { normalize } from "../../../helpers/normalize.js";
|
|
5
6
|
import { InvalidOriginError } from "../error/invalid-origin-error.js";
|
|
6
7
|
export class AssetApiClient {
|
|
@@ -114,7 +115,7 @@ export class AssetApiClient {
|
|
|
114
115
|
* handle parameter "bqvariation"
|
|
115
116
|
* reflect behaviour of SpeedKit.
|
|
116
117
|
* - do not send default variation
|
|
117
|
-
* - send mobile/tablet
|
|
118
|
+
* - send mobile/tablet as lowercase
|
|
118
119
|
* - send customVariations as they are
|
|
119
120
|
*
|
|
120
121
|
* As orestes will always make variations uppercase and does not care for caseSensitivity,
|
|
@@ -130,8 +131,8 @@ export class AssetApiClient {
|
|
|
130
131
|
if (variation === "default" || variation === "desktop") {
|
|
131
132
|
return;
|
|
132
133
|
}
|
|
133
|
-
// on
|
|
134
|
-
if (
|
|
134
|
+
// on tablet|mobile set variation as lowerCase
|
|
135
|
+
if (DEVICE_VARIATIONS.includes(variation.toLowerCase())) {
|
|
135
136
|
assetUrl.searchParams.set("bqvariation", variation.toLowerCase());
|
|
136
137
|
return;
|
|
137
138
|
}
|
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
import { UserCliConfig } from "../../helpers/cli-config.js";
|
|
2
|
+
import { QueryFilterOptions } from "../query-builder/query-builder-model.js";
|
|
2
3
|
import { PreWarmService } from "./pre-warm-service.js";
|
|
3
|
-
export interface AutoPreWarmContext {
|
|
4
|
+
export interface AutoPreWarmContext extends QueryFilterOptions {
|
|
4
5
|
readonly customerPath: string;
|
|
5
6
|
readonly configName: string;
|
|
6
|
-
|
|
7
|
+
/** Device classes to warm each url in the variations it was requested in. */
|
|
8
|
+
readonly device?: readonly string[];
|
|
9
|
+
/** Drop urls the origin mostly answers with a 404. */
|
|
10
|
+
readonly excludeNotFound?: boolean;
|
|
11
|
+
/** Keep only the most requested urls. */
|
|
12
|
+
readonly limit?: number;
|
|
13
|
+
/** Regular expressions the pre-warmed urls have to match. */
|
|
14
|
+
readonly match?: readonly string[];
|
|
15
|
+
/** Keep only urls requested at least this often in the window. */
|
|
16
|
+
readonly minHits?: number;
|
|
17
|
+
/** Athena workgroup the query runs in. */
|
|
18
|
+
readonly workgroup?: string;
|
|
19
|
+
/** Variations to pair every url with, when no traffic can reveal them. */
|
|
20
|
+
readonly variation?: readonly string[];
|
|
7
21
|
readonly variationPath?: string;
|
|
8
22
|
readonly sortParameters?: boolean;
|
|
9
23
|
readonly quiet?: boolean;
|
|
@@ -16,6 +30,10 @@ export interface AutoPreWarmContext {
|
|
|
16
30
|
* config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
|
|
17
31
|
* against Athena (Baqend native path, AWS fallback), and feeds the resulting
|
|
18
32
|
* URLs straight into the prewarm job.
|
|
33
|
+
*
|
|
34
|
+
* `--device` and `--variation` are handed to the query, which returns the
|
|
35
|
+
* variation alongside every url; without them the query returns urls and the
|
|
36
|
+
* list is crossed with `--variationPath` or the default variation, as before.
|
|
19
37
|
*/
|
|
20
38
|
export declare class AutoPreWarmFactory {
|
|
21
39
|
private context;
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import { CliContext, CliServiceFactory } from "../cli/index.js";
|
|
2
2
|
import { QueryBuilderFactory } from "../query-builder/query-builder-factory.js";
|
|
3
|
-
import {
|
|
3
|
+
import { QueryType, } from "../query-builder/query-builder-model.js";
|
|
4
|
+
import { normalizeDevices } from "../query-builder/queries/device.js";
|
|
4
5
|
import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
|
|
5
6
|
import { AssetApiClient } from "./assets/asset-api-client.js";
|
|
6
7
|
import { CsvReader } from "./csv-reader.js";
|
|
7
8
|
import { PreWarmService } from "./pre-warm-service.js";
|
|
8
9
|
import { resolveVariations } from "./pre-warm-factory.js";
|
|
10
|
+
import { toPrewarmTargets } from "./prewarm-targets.js";
|
|
9
11
|
/**
|
|
10
12
|
* Wires a {@link PreWarmService} that gets its URLs from a freshly built and
|
|
11
13
|
* executed prewarm query instead of a CSV file. Reads the app from the customer
|
|
12
14
|
* config, builds the prewarm query via {@link QueryBuilderFactory}, runs it
|
|
13
15
|
* against Athena (Baqend native path, AWS fallback), and feeds the resulting
|
|
14
16
|
* URLs straight into the prewarm job.
|
|
17
|
+
*
|
|
18
|
+
* `--device` and `--variation` are handed to the query, which returns the
|
|
19
|
+
* variation alongside every url; without them the query returns urls and the
|
|
20
|
+
* list is crossed with `--variationPath` or the default variation, as before.
|
|
15
21
|
*/
|
|
16
22
|
export class AutoPreWarmFactory {
|
|
17
23
|
context;
|
|
@@ -25,18 +31,30 @@ export class AutoPreWarmFactory {
|
|
|
25
31
|
const queryService = await new QueryBuilderFactory({
|
|
26
32
|
configName: this.context.configName,
|
|
27
33
|
customerPath: this.context.customerPath,
|
|
34
|
+
days: this.context.days,
|
|
35
|
+
devices: normalizeDevices(this.context.device),
|
|
36
|
+
excludeNoindex: this.context.excludeNoindex,
|
|
37
|
+
excludeNotFound: this.context.excludeNotFound,
|
|
38
|
+
excludeSuspicious: this.context.excludeSuspicious,
|
|
39
|
+
execute: true,
|
|
40
|
+
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
28
41
|
isWindows: this.context.isWindows,
|
|
42
|
+
limit: this.context.limit,
|
|
43
|
+
match: this.context.match,
|
|
44
|
+
minHits: this.context.minHits,
|
|
29
45
|
quiet: this.context.quiet,
|
|
30
|
-
|
|
31
|
-
|
|
46
|
+
variations: this.context.variation,
|
|
47
|
+
workgroup: this.context.workgroup,
|
|
48
|
+
}, this.userConfig).buildService(QueryType.prewarm);
|
|
32
49
|
const app = queryService.app;
|
|
33
50
|
const rows = await queryService.execute();
|
|
34
|
-
const urls = rows.map((row) => row.url).filter(Boolean);
|
|
35
51
|
const configApi = new ConfigApiServiceFactory(new ConfigApiContext(app)).getService();
|
|
36
52
|
const entityManager = await configApi.client.getEntityManager();
|
|
37
53
|
const assetApiClient = new AssetApiClient(entityManager.token, app, this.context.sortParameters);
|
|
38
54
|
const csvReader = new CsvReader();
|
|
39
|
-
|
|
40
|
-
|
|
55
|
+
return new PreWarmService(assetApiClient, cli, toPrewarmTargets(rows.filter((row) => row.url),
|
|
56
|
+
// `--variation` was answered by the query already; a variations file
|
|
57
|
+
// is still applied here.
|
|
58
|
+
resolveVariations(csvReader, undefined, this.context.variationPath)), this.context.verbose);
|
|
41
59
|
}
|
|
42
60
|
}
|
|
@@ -1,5 +1,24 @@
|
|
|
1
|
+
import { PrewarmRow } from "./pre-warm-model.js";
|
|
1
2
|
export declare class CsvReader {
|
|
2
|
-
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* The pre-warm list of a csv: one url per line, together with the variation
|
|
5
|
+
* to warm it in where the file names one.
|
|
6
|
+
*
|
|
7
|
+
* A second column only carries a variation when the header says so. The file
|
|
8
|
+
* `sk query prewarm --execute` writes for an unsplit list has the hit count
|
|
9
|
+
* there, and a list without a header at all is what it always was — bare
|
|
10
|
+
* urls, all of them to be warmed in whatever variations the caller asks for.
|
|
11
|
+
*/
|
|
12
|
+
readList(filePath: string): PrewarmRow[];
|
|
13
|
+
/**
|
|
14
|
+
* The first column of a csv, for a file that is a plain list of values —
|
|
15
|
+
* the variations to warm every url in.
|
|
16
|
+
*/
|
|
17
|
+
readValues(filePath: string): string[];
|
|
18
|
+
/**
|
|
19
|
+
* The fields of a csv line. A quoted field is taken as it stands, so a comma
|
|
20
|
+
* inside a url stays part of it, and its doubled quotes are one quote.
|
|
21
|
+
*/
|
|
22
|
+
private fields;
|
|
4
23
|
private getEntries;
|
|
5
24
|
}
|
|
@@ -3,22 +3,84 @@ import path from "node:path";
|
|
|
3
3
|
import { ReadCsvError } from "./error/read-csv-error.js";
|
|
4
4
|
const ENCODING = "utf8";
|
|
5
5
|
const LINE_BREAK = /\r?\n/;
|
|
6
|
+
/** The header the query command writes above a split pre-warm list. */
|
|
7
|
+
const VARIATION_COLUMN = "variation";
|
|
6
8
|
export class CsvReader {
|
|
7
|
-
|
|
9
|
+
/**
|
|
10
|
+
* The pre-warm list of a csv: one url per line, together with the variation
|
|
11
|
+
* to warm it in where the file names one.
|
|
12
|
+
*
|
|
13
|
+
* A second column only carries a variation when the header says so. The file
|
|
14
|
+
* `sk query prewarm --execute` writes for an unsplit list has the hit count
|
|
15
|
+
* there, and a list without a header at all is what it always was — bare
|
|
16
|
+
* urls, all of them to be warmed in whatever variations the caller asks for.
|
|
17
|
+
*/
|
|
18
|
+
readList(filePath) {
|
|
8
19
|
try {
|
|
9
|
-
const
|
|
10
|
-
|
|
20
|
+
const lines = this.getEntries(filePath).map((line) => this.fields(line));
|
|
21
|
+
const [header] = lines;
|
|
22
|
+
const hasHeader = header?.[0]?.toLowerCase() === "url";
|
|
23
|
+
const carriesVariation = hasHeader && header[1]?.toLowerCase() === VARIATION_COLUMN;
|
|
24
|
+
return (hasHeader ? lines.slice(1) : lines)
|
|
25
|
+
.filter(([url]) => url)
|
|
26
|
+
.map(([url, second]) => carriesVariation && second ? { url, variation: second } : { url });
|
|
11
27
|
}
|
|
12
28
|
catch (error) {
|
|
13
29
|
throw new ReadCsvError(error.message);
|
|
14
30
|
}
|
|
15
31
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
32
|
+
/**
|
|
33
|
+
* The first column of a csv, for a file that is a plain list of values —
|
|
34
|
+
* the variations to warm every url in.
|
|
35
|
+
*/
|
|
36
|
+
readValues(filePath) {
|
|
37
|
+
try {
|
|
38
|
+
return this.getEntries(filePath)
|
|
39
|
+
.map((line) => this.fields(line)[0])
|
|
40
|
+
.filter(Boolean);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw new ReadCsvError(error.message);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The fields of a csv line. A quoted field is taken as it stands, so a comma
|
|
48
|
+
* inside a url stays part of it, and its doubled quotes are one quote.
|
|
49
|
+
*/
|
|
50
|
+
fields(line) {
|
|
51
|
+
const entry = line.trim();
|
|
52
|
+
const fields = [];
|
|
53
|
+
let field = "";
|
|
54
|
+
let quoted = false;
|
|
55
|
+
for (let index = 0; index < entry.length; index++) {
|
|
56
|
+
const character = entry[index];
|
|
57
|
+
if (quoted) {
|
|
58
|
+
if (character !== '"') {
|
|
59
|
+
field += character;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
// A doubled quote is one quote; a single one ends the field.
|
|
63
|
+
if (entry[index + 1] === '"') {
|
|
64
|
+
field += '"';
|
|
65
|
+
index++;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
quoted = false;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
if (character === '"' && field === "") {
|
|
72
|
+
quoted = true;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (character === ",") {
|
|
76
|
+
fields.push(field.trim());
|
|
77
|
+
field = "";
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
field += character;
|
|
81
|
+
}
|
|
82
|
+
fields.push(field.trim());
|
|
83
|
+
return fields;
|
|
22
84
|
}
|
|
23
85
|
getEntries(filePath) {
|
|
24
86
|
const fileContent = fs.readFileSync(path.resolve(filePath), ENCODING);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|