@speedkit/cli 4.27.1 → 4.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +29 -14
- package/dist/commands/auto-prewarm.d.ts +1 -0
- package/dist/commands/auto-prewarm.js +2 -1
- package/dist/commands/query/parameter.d.ts +1 -0
- package/dist/commands/query/prewarm.d.ts +1 -0
- package/dist/helpers/clipboard.d.ts +6 -0
- package/dist/helpers/clipboard.js +7 -1
- package/dist/models/cli-parameters.d.ts +6 -0
- package/dist/models/cli-parameters.js +12 -0
- package/dist/services/athena/athena-service-factory.d.ts +5 -1
- package/dist/services/athena/athena-service-factory.js +8 -3
- package/dist/services/athena/athena-service-model.d.ts +17 -0
- package/dist/services/athena/athena-service-model.js +19 -0
- package/dist/services/athena/athena-service.d.ts +11 -1
- package/dist/services/athena/athena-service.js +10 -3
- package/dist/services/athena/athena-service.spec.js +16 -0
- package/dist/services/athena/baqend-athena-service.d.ts +13 -4
- package/dist/services/athena/baqend-athena-service.js +25 -10
- package/dist/services/athena/baqend-athena-service.spec.d.ts +1 -0
- package/dist/services/athena/baqend-athena-service.spec.js +51 -0
- package/dist/services/cli/cli-service.d.ts +9 -0
- package/dist/services/cli/cli-service.js +57 -3
- package/dist/services/cli/cli-service.spec.d.ts +1 -0
- package/dist/services/cli/cli-service.spec.js +67 -0
- package/dist/services/onboarding/dashboard/index.js +40 -1
- package/dist/services/prewarm/auto-pre-warm-factory.d.ts +2 -0
- package/dist/services/prewarm/auto-pre-warm-factory.js +1 -0
- package/dist/services/query-builder/queries/page-source.d.ts +11 -2
- package/dist/services/query-builder/queries/page-source.js +15 -3
- package/dist/services/query-builder/queries/query-model.d.ts +2 -0
- package/dist/services/query-builder/queries/spec/queries.spec.js +17 -0
- package/dist/services/query-builder/query-builder-factory.js +2 -1
- package/dist/services/query-builder/query-builder-model.d.ts +4 -0
- package/dist/services/query-builder/query-builder-service.d.ts +18 -2
- package/dist/services/query-builder/query-builder-service.js +38 -17
- package/dist/services/query-builder/query-builder-service.spec.d.ts +1 -0
- package/dist/services/query-builder/query-builder-service.spec.js +65 -0
- package/dist/services/query-builder/query-command.d.ts +1 -0
- package/dist/services/query-builder/query-command.js +1 -0
- package/oclif.manifest.json +31 -1
- package/package.json +1 -1
|
@@ -8,6 +8,9 @@ export declare class CliService implements CliServiceInterface {
|
|
|
8
8
|
private buffer;
|
|
9
9
|
private progressBar?;
|
|
10
10
|
private spinners;
|
|
11
|
+
private readonly animates;
|
|
12
|
+
/** The text of each running action, for the log line that replaces it. */
|
|
13
|
+
private readonly actions;
|
|
11
14
|
constructor(quiet?: boolean);
|
|
12
15
|
code(code: string, buffered?: boolean): void;
|
|
13
16
|
codeStyle(code: string): string;
|
|
@@ -30,6 +33,12 @@ export declare class CliService implements CliServiceInterface {
|
|
|
30
33
|
endAction(name: string, message?: string): void;
|
|
31
34
|
failAction(name: string, message?: string): void;
|
|
32
35
|
successAction(name: string, message?: string): void;
|
|
36
|
+
/**
|
|
37
|
+
* One line for an action that ended, where a spinner cannot be drawn. The
|
|
38
|
+
* text comes from the action that started, so a caller that ends one without
|
|
39
|
+
* a message still says which one it ended.
|
|
40
|
+
*/
|
|
41
|
+
private logAction;
|
|
33
42
|
startProgress(total: number, start?: number, payload?: object, config?: Options): void;
|
|
34
43
|
updateProgress(current: number, payload?: object): void;
|
|
35
44
|
stopProgress(): void;
|
|
@@ -6,12 +6,27 @@ import Spinnies from "spinnies";
|
|
|
6
6
|
import { safe } from "../../helpers/safe.js";
|
|
7
7
|
import { execSync } from "node:child_process";
|
|
8
8
|
import ApplicationError from "../error-handling/error/application-error.js";
|
|
9
|
+
/**
|
|
10
|
+
* Whether a spinner can actually spin here.
|
|
11
|
+
*
|
|
12
|
+
* The same condition spinnies itself uses to decide, so the two never
|
|
13
|
+
* disagree: without it spinnies falls back to re-rendering *every* spinner it
|
|
14
|
+
* knows on every update, which in a CI log means the first action's line is
|
|
15
|
+
* printed again under each later one — a page of "build prewarm-query" for a
|
|
16
|
+
* job that only ever built one.
|
|
17
|
+
*/
|
|
18
|
+
function canAnimate() {
|
|
19
|
+
return Boolean(!process.env.CI && process.stderr && process.stderr.isTTY);
|
|
20
|
+
}
|
|
9
21
|
export class CliService {
|
|
10
22
|
quiet;
|
|
11
23
|
style;
|
|
12
24
|
buffer = [];
|
|
13
25
|
progressBar;
|
|
14
26
|
spinners;
|
|
27
|
+
animates = canAnimate();
|
|
28
|
+
/** The text of each running action, for the log line that replaces it. */
|
|
29
|
+
actions = new Map();
|
|
15
30
|
constructor(quiet = false) {
|
|
16
31
|
this.quiet = quiet;
|
|
17
32
|
this.spinners = new Spinnies();
|
|
@@ -90,26 +105,65 @@ export class CliService {
|
|
|
90
105
|
if (this.quiet) {
|
|
91
106
|
return;
|
|
92
107
|
}
|
|
108
|
+
// Nothing is logged while the action runs: a log that cannot animate has
|
|
109
|
+
// no use for "this is happening", only for what happened.
|
|
110
|
+
if (!this.animates) {
|
|
111
|
+
this.actions.set(name, message);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
93
114
|
this.spinners.add(name, { text: message });
|
|
94
115
|
}
|
|
95
116
|
endAction(name, message) {
|
|
96
|
-
if (this.quiet
|
|
117
|
+
if (this.quiet) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (!this.animates) {
|
|
121
|
+
this.logAction("-", name, message);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if (!this.spinners.pick(name)) {
|
|
97
125
|
return;
|
|
98
126
|
}
|
|
99
127
|
this.spinners.update(name, { status: "stopped", text: message });
|
|
100
128
|
}
|
|
101
129
|
failAction(name, message) {
|
|
102
|
-
if (this.quiet
|
|
130
|
+
if (this.quiet) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (!this.animates) {
|
|
134
|
+
this.logAction(this.style.red("✖"), name, message);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (!this.spinners.pick(name)) {
|
|
103
138
|
return;
|
|
104
139
|
}
|
|
105
140
|
this.spinners.fail(name, { text: message });
|
|
106
141
|
}
|
|
107
142
|
successAction(name, message) {
|
|
108
|
-
if (this.quiet
|
|
143
|
+
if (this.quiet) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (!this.animates) {
|
|
147
|
+
this.logAction(this.style.green("✔"), name, message);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (!this.spinners.pick(name)) {
|
|
109
151
|
return;
|
|
110
152
|
}
|
|
111
153
|
this.spinners.succeed(name, { text: message, succeedColor: "white" });
|
|
112
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* One line for an action that ended, where a spinner cannot be drawn. The
|
|
157
|
+
* text comes from the action that started, so a caller that ends one without
|
|
158
|
+
* a message still says which one it ended.
|
|
159
|
+
*/
|
|
160
|
+
logAction(symbol, name, message) {
|
|
161
|
+
const text = message ?? this.actions.get(name);
|
|
162
|
+
if (!this.actions.delete(name) && !message) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
console.log(`${symbol} ${text}`);
|
|
166
|
+
}
|
|
113
167
|
startProgress(total, start = 0, payload, config) {
|
|
114
168
|
if (this.quiet) {
|
|
115
169
|
return;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { afterEach, beforeEach, describe, it } from "mocha";
|
|
3
|
+
import { CliService } from "./cli-service.js";
|
|
4
|
+
/** The lines a block of work put on stdout. */
|
|
5
|
+
function captured(work) {
|
|
6
|
+
const lines = [];
|
|
7
|
+
const log = console.log;
|
|
8
|
+
console.log = (line) => lines.push(line);
|
|
9
|
+
try {
|
|
10
|
+
work();
|
|
11
|
+
}
|
|
12
|
+
finally {
|
|
13
|
+
console.log = log;
|
|
14
|
+
}
|
|
15
|
+
return lines;
|
|
16
|
+
}
|
|
17
|
+
describe("CliService actions without a terminal", () => {
|
|
18
|
+
let ci;
|
|
19
|
+
beforeEach(() => {
|
|
20
|
+
ci = process.env.CI;
|
|
21
|
+
// What a CI runner sets, and what spinnies reads to decide it cannot spin.
|
|
22
|
+
process.env.CI = "true";
|
|
23
|
+
});
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
if (ci === undefined) {
|
|
26
|
+
delete process.env.CI;
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
process.env.CI = ci;
|
|
30
|
+
});
|
|
31
|
+
it("should say nothing while an action runs", () => {
|
|
32
|
+
const cli = new CliService();
|
|
33
|
+
expect(captured(() => cli.startAction("BUILD", "build prewarm-query"))).to.have.lengthOf(0);
|
|
34
|
+
});
|
|
35
|
+
it("should log an action that ended exactly once", () => {
|
|
36
|
+
// Spinnies re-renders every spinner it knows on each update when it cannot
|
|
37
|
+
// animate, so a running action reappears under each later one — a job with
|
|
38
|
+
// four actions logged "build prewarm-query" four times.
|
|
39
|
+
const cli = new CliService();
|
|
40
|
+
const lines = captured(() => {
|
|
41
|
+
cli.startAction("BUILD", "build prewarm-query");
|
|
42
|
+
cli.successAction("BUILD");
|
|
43
|
+
cli.startAction("QUERY", "execute athena query");
|
|
44
|
+
cli.failAction("QUERY");
|
|
45
|
+
});
|
|
46
|
+
expect(lines).to.have.lengthOf(2);
|
|
47
|
+
expect(lines[0]).to.contain("build prewarm-query");
|
|
48
|
+
expect(lines[1]).to.contain("execute athena query");
|
|
49
|
+
expect(lines.join("\n")).to.not.contain("build prewarm-query\n✖ execute athena query\n✔ build prewarm-query");
|
|
50
|
+
});
|
|
51
|
+
it("should keep the text of the action it ends", () => {
|
|
52
|
+
const cli = new CliService();
|
|
53
|
+
const lines = captured(() => {
|
|
54
|
+
cli.startAction("WARM", "warm the list");
|
|
55
|
+
cli.endAction("WARM");
|
|
56
|
+
});
|
|
57
|
+
expect(lines).to.deep.equal(["- warm the list"]);
|
|
58
|
+
});
|
|
59
|
+
it("should stay silent when it was told to be quiet", () => {
|
|
60
|
+
const cli = new CliService(true);
|
|
61
|
+
const lines = captured(() => {
|
|
62
|
+
cli.startAction("BUILD", "build prewarm-query");
|
|
63
|
+
cli.successAction("BUILD");
|
|
64
|
+
});
|
|
65
|
+
expect(lines).to.have.lengthOf(0);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
@@ -2,6 +2,38 @@ import { BaqendResponse } from "../browser/baqend-response.js";
|
|
|
2
2
|
import { safe } from "../../../helpers/safe.js";
|
|
3
3
|
import json2Csv from "json2csv";
|
|
4
4
|
import { AthenaBucket } from "../../athena/index.js";
|
|
5
|
+
/** A query option the caller left out, or the number it set. */
|
|
6
|
+
function readNumber(params, name) {
|
|
7
|
+
const raw = params.get(name);
|
|
8
|
+
if (raw === null || raw.trim() === "") {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
const value = Number(raw);
|
|
12
|
+
return Number.isFinite(value) ? value : undefined;
|
|
13
|
+
}
|
|
14
|
+
/** A query option the caller left out, or the flag it set. */
|
|
15
|
+
function readFlag(params, name) {
|
|
16
|
+
const raw = params.get(name);
|
|
17
|
+
if (raw === null) {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
return raw !== "false" && raw !== "0";
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* The query options carried on the url. An option that is not there stays
|
|
24
|
+
* undefined so the value the command line was started with keeps applying —
|
|
25
|
+
* the editor sends what it shows, and it shows nothing until it has asked.
|
|
26
|
+
*/
|
|
27
|
+
function readQueryOverrides(params) {
|
|
28
|
+
return {
|
|
29
|
+
bySet: readFlag(params, "bySet"),
|
|
30
|
+
days: readNumber(params, "days"),
|
|
31
|
+
excludeNoindex: readFlag(params, "excludeNoindex"),
|
|
32
|
+
excludeSuspicious: readFlag(params, "excludeSuspicious"),
|
|
33
|
+
hardNavigationsOnly: readFlag(params, "hardNavigationsOnly"),
|
|
34
|
+
minImpressions: readNumber(params, "minImpressions"),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
5
37
|
export class Dashboard {
|
|
6
38
|
customerConfig;
|
|
7
39
|
parameterQueryBuilder;
|
|
@@ -28,7 +60,14 @@ export class Dashboard {
|
|
|
28
60
|
return this.createBaqendResponse(requestOrigin, "unsupported method");
|
|
29
61
|
}
|
|
30
62
|
if (url.pathname.startsWith("/query/create")) {
|
|
31
|
-
|
|
63
|
+
const overrides = readQueryOverrides(url.searchParams);
|
|
64
|
+
return this.createBaqendResponse(requestOrigin,
|
|
65
|
+
// The options travel back with the query: the editor offering them
|
|
66
|
+
// has no other way to learn what the command line already set.
|
|
67
|
+
JSON.stringify({
|
|
68
|
+
query: this.parameterQueryBuilder.query(overrides),
|
|
69
|
+
options: this.parameterQueryBuilder.resolvedOptions(overrides),
|
|
70
|
+
}), "application/json; charset=UTF-8");
|
|
32
71
|
}
|
|
33
72
|
if (url.pathname.startsWith("/query/execute")) {
|
|
34
73
|
const postData = request.postData;
|
|
@@ -16,6 +16,8 @@ export interface AutoPreWarmContext extends QueryFilterOptions {
|
|
|
16
16
|
readonly minHits?: number;
|
|
17
17
|
/** Athena workgroup the query runs in. */
|
|
18
18
|
readonly workgroup?: string;
|
|
19
|
+
/** Athena data catalog the rum tables live in. */
|
|
20
|
+
readonly catalog?: string;
|
|
19
21
|
/** Variations to pair every url with, when no traffic can reveal them. */
|
|
20
22
|
readonly variation?: readonly string[];
|
|
21
23
|
readonly variationPath?: string;
|
|
@@ -45,6 +45,7 @@ export class AutoPreWarmFactory {
|
|
|
45
45
|
quiet: this.context.quiet,
|
|
46
46
|
variations: this.context.variation,
|
|
47
47
|
workgroup: this.context.workgroup,
|
|
48
|
+
catalog: this.context.catalog,
|
|
48
49
|
}, this.userConfig).buildService(QueryType.prewarm);
|
|
49
50
|
const app = queryService.app;
|
|
50
51
|
const rows = await queryService.execute();
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { RuleSubjectExpressions } from "../rules/index.js";
|
|
2
2
|
import { QueryInput } from "./query-model.js";
|
|
3
|
-
/** The
|
|
4
|
-
export declare const PAGE_IMPRESSIONS = "
|
|
3
|
+
/** The database and table every query reads from, within its catalog. */
|
|
4
|
+
export declare const PAGE_IMPRESSIONS = "rum.pi";
|
|
5
|
+
/**
|
|
6
|
+
* The relation every query reads from, qualified by the catalog it lives in.
|
|
7
|
+
*
|
|
8
|
+
* The catalog is named in the query and not only in the context it runs in, so
|
|
9
|
+
* that a query pasted into the Athena console reads the same tables as the one
|
|
10
|
+
* the CLI ran — the console resolves an unqualified name against whichever
|
|
11
|
+
* catalog its dropdown happens to hold.
|
|
12
|
+
*/
|
|
13
|
+
export declare function pageImpressions(catalog?: string): string;
|
|
5
14
|
/** The name of the common table expression the queries select from. */
|
|
6
15
|
export declare const PAGE_SOURCE = "pages";
|
|
7
16
|
export interface PageSource {
|
|
@@ -2,8 +2,20 @@ import { column, formatPredicate } from "../sql/index.js";
|
|
|
2
2
|
import { resolvedSubject, urlSubjects, } from "../rules/index.js";
|
|
3
3
|
import { normalizeDevices } from "./device.js";
|
|
4
4
|
import { buildRowFilter } from "./page-filter.js";
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
import { athenaCatalog } from "../../athena/athena-service-model.js";
|
|
6
|
+
/** The database and table every query reads from, within its catalog. */
|
|
7
|
+
export const PAGE_IMPRESSIONS = "rum.pi";
|
|
8
|
+
/**
|
|
9
|
+
* The relation every query reads from, qualified by the catalog it lives in.
|
|
10
|
+
*
|
|
11
|
+
* The catalog is named in the query and not only in the context it runs in, so
|
|
12
|
+
* that a query pasted into the Athena console reads the same tables as the one
|
|
13
|
+
* the CLI ran — the console resolves an unqualified name against whichever
|
|
14
|
+
* catalog its dropdown happens to hold.
|
|
15
|
+
*/
|
|
16
|
+
export function pageImpressions(catalog) {
|
|
17
|
+
return `${athenaCatalog(catalog)}.${PAGE_IMPRESSIONS}`;
|
|
18
|
+
}
|
|
7
19
|
/** The name of the common table expression the queries select from. */
|
|
8
20
|
export const PAGE_SOURCE = "pages";
|
|
9
21
|
const COLUMN_NAMES = {
|
|
@@ -37,7 +49,7 @@ export function buildPageSource(input, options = {}) {
|
|
|
37
49
|
const cte = `${PAGE_SOURCE} as (
|
|
38
50
|
select
|
|
39
51
|
${projections.join(",\n ")}
|
|
40
|
-
from ${
|
|
52
|
+
from ${pageImpressions(input.catalog)}
|
|
41
53
|
where ${formatPredicate(buildRowFilter(input), { level: 2 })}
|
|
42
54
|
)`;
|
|
43
55
|
// An unused subject keeps its inline expression: nothing refers to it, and
|
|
@@ -61,6 +61,8 @@ export interface ParameterOptions {
|
|
|
61
61
|
/** Everything a query is built from. */
|
|
62
62
|
export interface QueryInput extends ParameterOptions, PrewarmOptions {
|
|
63
63
|
readonly app: string;
|
|
64
|
+
/** The data catalog the rum tables live in. Defaults to `live`. */
|
|
65
|
+
readonly catalog?: string;
|
|
64
66
|
/** Size of the window the query looks at, in days. */
|
|
65
67
|
readonly days: number;
|
|
66
68
|
/** Drop page impressions of pages marked `noindex`. */
|
|
@@ -188,6 +188,23 @@ describe("the --variation pairing", () => {
|
|
|
188
188
|
expect(buildPrewarmQuery(inputFor(CONFIG, { variations: ["it's"] })).sql).to.contain("array['it''s']");
|
|
189
189
|
});
|
|
190
190
|
});
|
|
191
|
+
describe("the --catalog the query reads from", () => {
|
|
192
|
+
it("should read the rum tables from the live catalog by default", () => {
|
|
193
|
+
expect(buildPrewarmQuery(inputFor(CONFIG)).sql).to.contain("from live.rum.pi");
|
|
194
|
+
expect(buildParameterQuery(inputFor(CONFIG)).sql).to.contain("from live.rum.pi");
|
|
195
|
+
});
|
|
196
|
+
it("should read from the live catalog when given a blank one", () => {
|
|
197
|
+
expect(buildPrewarmQuery(inputFor(CONFIG, { catalog: "" })).sql).to.contain("from live.rum.pi");
|
|
198
|
+
});
|
|
199
|
+
it("should name the catalog it was given instead", () => {
|
|
200
|
+
// Which catalog holds the tables depends on the credentials: our own
|
|
201
|
+
// logins see them under `live`, a role scoped to one account under
|
|
202
|
+
// `AwsDataCatalog`.
|
|
203
|
+
const options = { catalog: "AwsDataCatalog" };
|
|
204
|
+
expect(buildPrewarmQuery(inputFor(CONFIG, options)).sql).to.contain("from AwsDataCatalog.rum.pi");
|
|
205
|
+
expect(buildParameterQuery(inputFor(CONFIG, options)).sql).to.not.contain("live.rum.pi");
|
|
206
|
+
});
|
|
207
|
+
});
|
|
191
208
|
describe("the --count report", () => {
|
|
192
209
|
it("should count the urls and the traffic they stand for", () => {
|
|
193
210
|
const { sql } = buildPrewarmQuery(inputFor(CONFIG, { count: true }));
|
|
@@ -25,6 +25,7 @@ export class QueryBuilderFactory {
|
|
|
25
25
|
const serviceContext = {
|
|
26
26
|
app,
|
|
27
27
|
bySet: this.context.bySet,
|
|
28
|
+
catalog: this.context.catalog,
|
|
28
29
|
count: this.context.count,
|
|
29
30
|
days: this.context.days,
|
|
30
31
|
devices: this.context.devices,
|
|
@@ -47,7 +48,7 @@ export class QueryBuilderFactory {
|
|
|
47
48
|
// The Athena service is only needed when the query is executed directly —
|
|
48
49
|
// which `--count` is, a count query being of no use as text.
|
|
49
50
|
const athenaService = this.context.execute || this.context.count
|
|
50
|
-
? await new AthenaServiceFactory(app, this.context.workgroup).getService()
|
|
51
|
+
? await new AthenaServiceFactory(app, this.context.workgroup, this.context.catalog).getService()
|
|
51
52
|
: undefined;
|
|
52
53
|
return new QueryBuilderService(serviceContext, parseRuleSets(speedKitConfig), cli, athenaService);
|
|
53
54
|
}
|
|
@@ -16,6 +16,8 @@ export interface QueryFilterOptions {
|
|
|
16
16
|
}
|
|
17
17
|
export interface QueryBuilderContext extends ParameterOptions, PrewarmOptions, QueryFilterOptions {
|
|
18
18
|
readonly app: string;
|
|
19
|
+
/** Athena data catalog the rum tables live in. */
|
|
20
|
+
readonly catalog?: string;
|
|
19
21
|
readonly execute?: boolean;
|
|
20
22
|
/** Report every rule the query leaves out, reduces or approximates. */
|
|
21
23
|
readonly explain?: boolean;
|
|
@@ -26,6 +28,8 @@ export interface QueryBuilderContext extends ParameterOptions, PrewarmOptions, Q
|
|
|
26
28
|
readonly quiet?: boolean;
|
|
27
29
|
}
|
|
28
30
|
export interface QueryBuilderFactoryContext extends ParameterOptions, PrewarmOptions, QueryFilterOptions {
|
|
31
|
+
/** Athena data catalog the rum tables live in. */
|
|
32
|
+
readonly catalog?: string;
|
|
29
33
|
readonly configName: string;
|
|
30
34
|
readonly customerPath: string;
|
|
31
35
|
readonly execute?: boolean;
|
|
@@ -2,6 +2,14 @@ import { QueryBuilderContext } from "./query-builder-model.js";
|
|
|
2
2
|
import { ParsedRuleSets } from "./rules/index.js";
|
|
3
3
|
import { CliServiceInterface } from "../cli/index.js";
|
|
4
4
|
import { AthenaService } from "../athena/index.js";
|
|
5
|
+
/**
|
|
6
|
+
* The options a caller may set per query instead of per command — what the
|
|
7
|
+
* onboarding dashboard's query editor offers. Everything else a query is built
|
|
8
|
+
* from (the app, its rule sets, the catalog) is fixed for the process.
|
|
9
|
+
*/
|
|
10
|
+
export type QueryOverrides = Pick<QueryBuilderContext, "bySet" | "days" | "excludeNoindex" | "excludeSuspicious" | "hardNavigationsOnly" | "minImpressions">;
|
|
11
|
+
/** {@link QueryOverrides} with every default filled in. */
|
|
12
|
+
export type ResolvedQueryOptions = Required<QueryOverrides>;
|
|
5
13
|
export declare class QueryBuilderService {
|
|
6
14
|
private context;
|
|
7
15
|
private ruleSets;
|
|
@@ -17,8 +25,16 @@ export declare class QueryBuilderService {
|
|
|
17
25
|
* directly instead of exporting it (e.g. `auto-prewarm`).
|
|
18
26
|
*/
|
|
19
27
|
execute<T>(): Promise<T[]>;
|
|
20
|
-
/** The query as SQL. */
|
|
21
|
-
query(): string;
|
|
28
|
+
/** The query as SQL, optionally shaped by per-call overrides. */
|
|
29
|
+
query(overrides?: QueryOverrides): string;
|
|
30
|
+
/**
|
|
31
|
+
* The options a query would be built with right now, the defaults filled in.
|
|
32
|
+
*
|
|
33
|
+
* A caller that offers the options for editing — the onboarding dashboard —
|
|
34
|
+
* has to show what they currently are, and only the service knows which of
|
|
35
|
+
* them the command line left to a default.
|
|
36
|
+
*/
|
|
37
|
+
resolvedOptions(overrides?: QueryOverrides): ResolvedQueryOptions;
|
|
22
38
|
private build;
|
|
23
39
|
/**
|
|
24
40
|
* Points out where the query is not exactly what the config says — a rule
|
|
@@ -44,10 +44,10 @@ export class QueryBuilderService {
|
|
|
44
44
|
this.cli.startAction(action, `build ${this.context.queryType}-query`);
|
|
45
45
|
const result = safe(() => this.build());
|
|
46
46
|
if (result.success !== true) {
|
|
47
|
+
// Fail the action before throwing: the spinner is a timer, and one left
|
|
48
|
+
// running keeps the process alive with nothing to do.
|
|
47
49
|
this.cli.failAction(action, result.error);
|
|
48
|
-
|
|
49
|
-
console.error(result.errorObj);
|
|
50
|
-
return;
|
|
50
|
+
throw result.errorObj ?? new Error(result.error);
|
|
51
51
|
}
|
|
52
52
|
this.cli.successAction(action);
|
|
53
53
|
const query = result.data;
|
|
@@ -92,28 +92,45 @@ export class QueryBuilderService {
|
|
|
92
92
|
async execute() {
|
|
93
93
|
return this.runQuery(this.query());
|
|
94
94
|
}
|
|
95
|
-
/** The query as SQL. */
|
|
96
|
-
query() {
|
|
97
|
-
return this.build().sql;
|
|
95
|
+
/** The query as SQL, optionally shaped by per-call overrides. */
|
|
96
|
+
query(overrides = {}) {
|
|
97
|
+
return this.build(overrides).sql;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The options a query would be built with right now, the defaults filled in.
|
|
101
|
+
*
|
|
102
|
+
* A caller that offers the options for editing — the onboarding dashboard —
|
|
103
|
+
* has to show what they currently are, and only the service knows which of
|
|
104
|
+
* them the command line left to a default.
|
|
105
|
+
*/
|
|
106
|
+
resolvedOptions(overrides = {}) {
|
|
107
|
+
return {
|
|
108
|
+
bySet: overrides.bySet ?? this.context.bySet ?? false,
|
|
109
|
+
days: overrides.days ??
|
|
110
|
+
this.context.days ??
|
|
111
|
+
DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
112
|
+
excludeNoindex: overrides.excludeNoindex ?? this.context.excludeNoindex ?? false,
|
|
113
|
+
excludeSuspicious: overrides.excludeSuspicious ?? this.context.excludeSuspicious ?? false,
|
|
114
|
+
hardNavigationsOnly: overrides.hardNavigationsOnly ??
|
|
115
|
+
this.context.hardNavigationsOnly ??
|
|
116
|
+
false,
|
|
117
|
+
minImpressions: overrides.minImpressions ?? this.context.minImpressions ?? 1,
|
|
118
|
+
};
|
|
98
119
|
}
|
|
99
|
-
build() {
|
|
120
|
+
build(overrides = {}) {
|
|
100
121
|
const input = {
|
|
101
122
|
app: this.context.app,
|
|
102
|
-
|
|
123
|
+
catalog: this.context.catalog,
|
|
103
124
|
count: this.context.count,
|
|
104
|
-
days: this.context.days ?? DEFAULT_QUERY_DAYS[this.context.queryType],
|
|
105
125
|
devices: this.context.devices,
|
|
106
126
|
excludeNotFound: this.context.excludeNotFound,
|
|
107
|
-
excludeNoindex: this.context.excludeNoindex,
|
|
108
|
-
excludeSuspicious: this.context.excludeSuspicious,
|
|
109
|
-
hardNavigationsOnly: this.context.hardNavigationsOnly,
|
|
110
127
|
hosts: this.context.hosts,
|
|
111
128
|
limit: this.context.limit,
|
|
112
129
|
match: this.context.match,
|
|
113
130
|
minHits: this.context.minHits,
|
|
114
|
-
minImpressions: this.context.minImpressions,
|
|
115
131
|
ruleSets: this.ruleSets,
|
|
116
132
|
variations: this.context.variations,
|
|
133
|
+
...this.resolvedOptions(overrides),
|
|
117
134
|
};
|
|
118
135
|
return this.context.queryType === QueryType.prewarm
|
|
119
136
|
? buildPrewarmQuery(input)
|
|
@@ -178,9 +195,12 @@ export class QueryBuilderService {
|
|
|
178
195
|
this.cli.startAction("QUERY:COUNT", "count the pre-warm list");
|
|
179
196
|
const result = await safe(this.runQuery(query));
|
|
180
197
|
if (result.success !== true) {
|
|
198
|
+
// Thrown, not reported and swallowed. A count that did not run is a
|
|
199
|
+
// failed command, and something driving this from a script — a CI job
|
|
200
|
+
// reading the exit code — has no other way to find that out. Printing
|
|
201
|
+
// the message and exiting 0 made a broken query look like a clean run.
|
|
181
202
|
this.cli.failAction("QUERY:COUNT");
|
|
182
|
-
|
|
183
|
-
return;
|
|
203
|
+
throw result.errorObj ?? new Error(result.error);
|
|
184
204
|
}
|
|
185
205
|
this.cli.successAction("QUERY:COUNT");
|
|
186
206
|
const rows = result.data;
|
|
@@ -210,9 +230,10 @@ export class QueryBuilderService {
|
|
|
210
230
|
this.cli.startAction("QUERY:EXECUTE", "execute query and export csv");
|
|
211
231
|
const result = await safe(this.runQuery(query));
|
|
212
232
|
if (result.success !== true) {
|
|
233
|
+
// Thrown for the same reason as the count path: an export that produced
|
|
234
|
+
// no file must not exit 0.
|
|
213
235
|
this.cli.failAction("QUERY:EXECUTE");
|
|
214
|
-
|
|
215
|
-
return;
|
|
236
|
+
throw result.errorObj ?? new Error(result.error);
|
|
216
237
|
}
|
|
217
238
|
const rows = result.data;
|
|
218
239
|
const date = new Date().toISOString().split("T")[0];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { expect } from "chai";
|
|
2
|
+
import { describe, it } from "mocha";
|
|
3
|
+
import { QueryBuilderService } from "./query-builder-service.js";
|
|
4
|
+
import { QueryType } from "./query-builder-model.js";
|
|
5
|
+
import { parseRuleSets } from "./rules/index.js";
|
|
6
|
+
/** Records the actions so a left-running spinner shows up as a missing fail. */
|
|
7
|
+
function recordingCli() {
|
|
8
|
+
const calls = [];
|
|
9
|
+
const cli = {
|
|
10
|
+
comment: () => calls.push("comment"),
|
|
11
|
+
failAction: (name) => calls.push(`fail:${name}`),
|
|
12
|
+
startAction: (name) => calls.push(`start:${name}`),
|
|
13
|
+
successAction: (name) => calls.push(`success:${name}`),
|
|
14
|
+
table: () => calls.push("table"),
|
|
15
|
+
writeError: (message) => calls.push(`error:${message}`),
|
|
16
|
+
writeSuccess: () => calls.push("success"),
|
|
17
|
+
writeWarning: () => calls.push("warning"),
|
|
18
|
+
};
|
|
19
|
+
return { calls, cli };
|
|
20
|
+
}
|
|
21
|
+
/** An Athena that always refuses, the way a bad catalog or workgroup does. */
|
|
22
|
+
const refusingAthena = {
|
|
23
|
+
getResult: () => Promise.reject(new Error("The Amazon Athena query failed to run with error message: CATALOG_NOT_FOUND")),
|
|
24
|
+
};
|
|
25
|
+
function countService(cli) {
|
|
26
|
+
return new QueryBuilderService({
|
|
27
|
+
app: "obi-de",
|
|
28
|
+
count: true,
|
|
29
|
+
hosts: ["www.obi.de"],
|
|
30
|
+
queryType: QueryType.prewarm,
|
|
31
|
+
quiet: true,
|
|
32
|
+
},
|
|
33
|
+
// The real parser on an empty config: a hand-rolled stand-in would fail the
|
|
34
|
+
// build step and the test would pass for the wrong reason.
|
|
35
|
+
parseRuleSets({}), cli, refusingAthena);
|
|
36
|
+
}
|
|
37
|
+
describe("QueryBuilderService — a query that could not run", () => {
|
|
38
|
+
// Regression: this used to print the message and return, so the process
|
|
39
|
+
// exited 0. A CI job counting urls then reported success with no count, and
|
|
40
|
+
// the pipeline that read it had no way to tell a broken query from an empty
|
|
41
|
+
// one.
|
|
42
|
+
it("fails the command instead of exiting cleanly", async () => {
|
|
43
|
+
const { cli } = recordingCli();
|
|
44
|
+
const outcome = await countService(cli)
|
|
45
|
+
.run()
|
|
46
|
+
.then(() => "resolved")
|
|
47
|
+
.catch(() => "rejected");
|
|
48
|
+
expect(outcome).to.equal("rejected");
|
|
49
|
+
});
|
|
50
|
+
it("carries the reason, so the log says what went wrong", async () => {
|
|
51
|
+
const { cli } = recordingCli();
|
|
52
|
+
const error = await countService(cli)
|
|
53
|
+
.run()
|
|
54
|
+
.then(() => undefined)
|
|
55
|
+
.catch((thrown) => thrown);
|
|
56
|
+
expect(error?.message).to.contain("CATALOG_NOT_FOUND");
|
|
57
|
+
});
|
|
58
|
+
it("ends the spinner before throwing — it is a timer that holds the process open", async () => {
|
|
59
|
+
const { calls, cli } = recordingCli();
|
|
60
|
+
await countService(cli)
|
|
61
|
+
.run()
|
|
62
|
+
.catch(() => undefined);
|
|
63
|
+
expect(calls).to.contain("fail:QUERY:COUNT");
|
|
64
|
+
});
|
|
65
|
+
});
|
|
@@ -8,6 +8,7 @@ import { QueryFilterOptions, QueryType } from "./query-builder-model.js";
|
|
|
8
8
|
* the two the caller gives rather than the list the builder works on.
|
|
9
9
|
*/
|
|
10
10
|
export interface QueryCommandFlags extends ParameterOptions, Omit<PrewarmOptions, "devices" | "variations">, QueryFilterOptions {
|
|
11
|
+
readonly catalog?: string;
|
|
11
12
|
readonly configName: string;
|
|
12
13
|
readonly device?: readonly string[];
|
|
13
14
|
readonly execute?: boolean;
|
|
@@ -12,6 +12,7 @@ import { normalizeDevices } from "./queries/device.js";
|
|
|
12
12
|
export async function runQuery(queryType, customerPath, flags, config) {
|
|
13
13
|
const service = await new QueryBuilderFactory({
|
|
14
14
|
bySet: flags.bySet,
|
|
15
|
+
catalog: flags.catalog,
|
|
15
16
|
configName: flags.configName,
|
|
16
17
|
count: flags.count,
|
|
17
18
|
customerPath,
|