@speedkit/cli 3.19.0 → 3.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +4 -3
- package/dist/commands/prewarm.d.ts +1 -0
- package/dist/commands/prewarm.js +7 -3
- package/dist/models/cli-parameters.d.ts +1 -0
- package/dist/models/cli-parameters.js +1 -0
- package/dist/services/prewarm/assets/asset-api-client.d.ts +11 -2
- package/dist/services/prewarm/assets/asset-api-client.js +70 -14
- package/dist/services/prewarm/assets/asset-item.d.ts +2 -3
- package/dist/services/prewarm/assets/asset-item.js +4 -3
- package/dist/services/prewarm/assets/asset-list.d.ts +3 -4
- package/dist/services/prewarm/assets/asset-list.js +2 -13
- package/dist/services/prewarm/pre-warm-factory.js +1 -1
- package/dist/services/prewarm/pre-warm-model.d.ts +3 -4
- package/dist/services/prewarm/pre-warm-model.js +6 -3
- package/dist/services/prewarm/pre-warm-service.d.ts +5 -4
- package/dist/services/prewarm/pre-warm-service.js +51 -36
- package/oclif.manifest.json +8 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
# [3.20.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.19.0...v3.20.0) (2026-01-06)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **prewarm:** improve concurrency handling of prewarming ([509d19f](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/509d19f8d0a7131e81906c18ce83a338aa3c4c82))
|
|
7
|
+
|
|
1
8
|
# [3.19.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.18.0...v3.19.0) (2025-12-19)
|
|
2
9
|
|
|
3
10
|
|
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
|
|
|
21
21
|
$ sk COMMAND
|
|
22
22
|
running command...
|
|
23
23
|
$ sk (--version)
|
|
24
|
-
@speedkit/cli/3.
|
|
24
|
+
@speedkit/cli/3.20.0 linux-x64 node-v20.19.6
|
|
25
25
|
$ sk --help [COMMAND]
|
|
26
26
|
USAGE
|
|
27
27
|
$ sk COMMAND
|
|
@@ -437,7 +437,7 @@ Pre-warm Fastly
|
|
|
437
437
|
|
|
438
438
|
```
|
|
439
439
|
USAGE
|
|
440
|
-
$ sk prewarm APP PATH [-p <value> | -v <value>] [-k] [-q] [-d <value>]
|
|
440
|
+
$ sk prewarm APP PATH [-p <value> | -v <value>] [-k] [-q] [--verbose] [-d <value>]
|
|
441
441
|
|
|
442
442
|
ARGUMENTS
|
|
443
443
|
APP The customer app name
|
|
@@ -448,7 +448,8 @@ FLAGS
|
|
|
448
448
|
-k, --keepParameterSorting Prevents alphabetical sorting of the URL parameters
|
|
449
449
|
-p, --variationPath=<value> path to variations.csv
|
|
450
450
|
-q, --quiet less output non interactive
|
|
451
|
-
-v, --variation=<value> [default:
|
|
451
|
+
-v, --variation=<value> [default: default]
|
|
452
|
+
--verbose Log information about failed requests
|
|
452
453
|
|
|
453
454
|
DESCRIPTION
|
|
454
455
|
Pre-warm Fastly
|
|
@@ -12,6 +12,7 @@ export default class Prewarm extends Command {
|
|
|
12
12
|
variationPath: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
|
|
13
13
|
keepParameterSorting: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
|
|
14
14
|
quiet: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
|
|
15
|
+
verbose: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
|
|
15
16
|
delay: import("@oclif/core/lib/interfaces").OptionFlag<number, import("@oclif/core/lib/interfaces").CustomOptions>;
|
|
16
17
|
};
|
|
17
18
|
static examples: string[];
|
package/dist/commands/prewarm.js
CHANGED
|
@@ -20,7 +20,7 @@ class Prewarm extends core_1.Command {
|
|
|
20
20
|
static flags = {
|
|
21
21
|
[cli_parameters_1.CLIParameters.Variation]: core_1.Flags.string({
|
|
22
22
|
char: cli_parameters_1.CLIParametersChar.Variation,
|
|
23
|
-
default: "
|
|
23
|
+
default: "default",
|
|
24
24
|
}),
|
|
25
25
|
[cli_parameters_1.CLIParameters.VariationPath]: core_1.Flags.file({
|
|
26
26
|
char: cli_parameters_1.CLIParametersChar.VariationPath,
|
|
@@ -37,6 +37,10 @@ class Prewarm extends core_1.Command {
|
|
|
37
37
|
default: false,
|
|
38
38
|
description: "less output non interactive",
|
|
39
39
|
}),
|
|
40
|
+
[cli_parameters_1.CLIParameters.Verbose]: core_1.Flags.boolean({
|
|
41
|
+
default: false,
|
|
42
|
+
description: "Log information about failed requests",
|
|
43
|
+
}),
|
|
40
44
|
["delay"]: core_1.Flags.integer({
|
|
41
45
|
char: "d",
|
|
42
46
|
description: "Request delay in ms (deprecated)",
|
|
@@ -54,9 +58,9 @@ class Prewarm extends core_1.Command {
|
|
|
54
58
|
`$ sk prewarm ${cli_parameters_1.CLIParametersExample.AppName} ${cli_parameters_1.CLIParametersExample.Path} -${cli_parameters_1.CLIParametersChar.Path} ${cli_parameters_1.CLIParametersExample.VariationPath} -${cli_parameters_1.CLIParametersChar.KeepParameterSorting}`,
|
|
55
59
|
];
|
|
56
60
|
async run() {
|
|
57
|
-
const { args: { app, path }, flags: { variation, variationPath, keepParameterSorting, quiet }, } = await this.parse(Prewarm);
|
|
61
|
+
const { args: { app, path }, flags: { variation, variationPath, keepParameterSorting, quiet, verbose }, } = await this.parse(Prewarm);
|
|
58
62
|
const sortParameters = !keepParameterSorting;
|
|
59
|
-
const service = await new prewarm_1.PreWarmFactory(new prewarm_1.PreWarmContext(app, path, variation, variationPath, sortParameters, quiet)).getService();
|
|
63
|
+
const service = await new prewarm_1.PreWarmFactory(new prewarm_1.PreWarmContext(app, path, variation, variationPath, sortParameters, quiet, verbose)).getService();
|
|
60
64
|
await service.run();
|
|
61
65
|
}
|
|
62
66
|
}
|
|
@@ -28,6 +28,7 @@ var CLIParameters;
|
|
|
28
28
|
CLIParameters["Urls"] = "urls";
|
|
29
29
|
CLIParameters["Variation"] = "variation";
|
|
30
30
|
CLIParameters["VariationPath"] = "variationPath";
|
|
31
|
+
CLIParameters["Verbose"] = "verbose";
|
|
31
32
|
CLIParameters["useTestConfig"] = "useTestConfig";
|
|
32
33
|
})(CLIParameters || (exports.CLIParameters = CLIParameters = {}));
|
|
33
34
|
/* eslint-disable @typescript-eslint/no-duplicate-enum-values */
|
|
@@ -1,13 +1,21 @@
|
|
|
1
|
+
export type SimpleResponse = {
|
|
2
|
+
headers: Headers;
|
|
3
|
+
status: number;
|
|
4
|
+
ok: boolean;
|
|
5
|
+
};
|
|
1
6
|
export declare class AssetApiClient {
|
|
2
7
|
private authenticationToken;
|
|
3
8
|
private app;
|
|
4
9
|
private sortParameters;
|
|
5
|
-
private
|
|
10
|
+
private sessions;
|
|
6
11
|
private rateLimit;
|
|
7
12
|
private lowestRateLimit;
|
|
8
13
|
private isRetry;
|
|
14
|
+
private lastRequestTime;
|
|
9
15
|
constructor(authenticationToken: string, app: string, sortParameters: boolean);
|
|
10
|
-
fetchURL(url: string, variation: string): Promise<
|
|
16
|
+
fetchURL(url: string, variation: string, verbose: boolean): Promise<SimpleResponse>;
|
|
17
|
+
private makeHttp2Request;
|
|
18
|
+
destroy(): void;
|
|
11
19
|
setIsRetry(): void;
|
|
12
20
|
getRateLimit(): number;
|
|
13
21
|
reduceMaxRequestsPerSecond(): void;
|
|
@@ -30,4 +38,5 @@ export declare class AssetApiClient {
|
|
|
30
38
|
* @private
|
|
31
39
|
*/
|
|
32
40
|
private setVariationParameter;
|
|
41
|
+
private sleepByRate;
|
|
33
42
|
}
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AssetApiClient = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const node_https_1 = require("node:https");
|
|
6
5
|
const node_url_1 = require("node:url");
|
|
7
|
-
const
|
|
6
|
+
const node_http2_1 = tslib_1.__importDefault(require("node:http2"));
|
|
8
7
|
const __1 = require("../");
|
|
9
8
|
const normalize_1 = require("../../../helpers/normalize");
|
|
10
9
|
const invalid_origin_error_1 = require("../error/invalid-origin-error");
|
|
@@ -12,34 +11,83 @@ class AssetApiClient {
|
|
|
12
11
|
authenticationToken;
|
|
13
12
|
app;
|
|
14
13
|
sortParameters;
|
|
15
|
-
|
|
16
|
-
keepAlive: true,
|
|
17
|
-
rejectUnauthorized: false,
|
|
18
|
-
});
|
|
14
|
+
sessions = new Map();
|
|
19
15
|
rateLimit = __1.DEFAULT_REQUESTS_PER_SECOND;
|
|
20
16
|
lowestRateLimit = __1.DEFAULT_REQUESTS_PER_SECOND;
|
|
21
17
|
isRetry = false;
|
|
18
|
+
lastRequestTime = Date.now();
|
|
22
19
|
constructor(authenticationToken, app, sortParameters) {
|
|
23
20
|
this.authenticationToken = authenticationToken;
|
|
24
21
|
this.app = app;
|
|
25
22
|
this.sortParameters = sortParameters;
|
|
26
23
|
}
|
|
27
|
-
async fetchURL(url, variation) {
|
|
24
|
+
async fetchURL(url, variation, verbose) {
|
|
25
|
+
await this.sleepByRate();
|
|
28
26
|
this.isValidUrl(url);
|
|
29
27
|
const assetUrl = new node_url_1.URL((0, normalize_1.normalize)(url, this.sortParameters));
|
|
30
28
|
this.setVariationParameter(variation, assetUrl);
|
|
31
29
|
assetUrl.searchParams.set("bqpass", "1");
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
const origin = `https://${this.app}.app.baqend.com`;
|
|
31
|
+
const start = Date.now();
|
|
32
|
+
const result = await this.makeHttp2Request(origin, `/v1/asset/${assetUrl.toString()}`, assetUrl.origin);
|
|
33
|
+
if (verbose) {
|
|
34
|
+
console.log(`fetch ${url} - ${variation} took ${Date.now() - start}ms, cdnTime: ${result.headers.get("x-timer") || "-"}, status: ${result.status}, cache: ${result.headers.get("x-cache")}, timings: ${result.headers.get(__1.TIMING_HEADER_KEY)}`);
|
|
35
|
+
}
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
makeHttp2Request(origin, path, assetOrigin) {
|
|
39
|
+
return new Promise((resolve, reject) => {
|
|
40
|
+
// we are using a session pool for requests since node seems to stack the requests up when using the same
|
|
41
|
+
// http2 session for several requests concurrently. This causes then that a single slow request can completely
|
|
42
|
+
// delay all requests wich are issued afterward
|
|
43
|
+
let sessions = this.sessions.get(origin);
|
|
44
|
+
if (!sessions) {
|
|
45
|
+
sessions = [];
|
|
46
|
+
this.sessions.set(origin, sessions);
|
|
47
|
+
}
|
|
48
|
+
let session = sessions?.shift();
|
|
49
|
+
if (!session || session.destroyed || session.closed) {
|
|
50
|
+
session = node_http2_1.default.connect(origin, {
|
|
51
|
+
rejectUnauthorized: false,
|
|
52
|
+
});
|
|
53
|
+
session.on("error", (error) => {
|
|
54
|
+
reject(error);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
const request = session.request({
|
|
58
|
+
[node_http2_1.default.constants.HTTP2_HEADER_METHOD]: "HEAD",
|
|
59
|
+
[node_http2_1.default.constants.HTTP2_HEADER_PATH]: path,
|
|
37
60
|
authorization: `BAT ${this.authenticationToken}`,
|
|
61
|
+
origin: assetOrigin,
|
|
62
|
+
"accept-encoding": "gzip, deflate, br, dcb",
|
|
38
63
|
"baqend-choose-dictionary": "1",
|
|
39
|
-
},
|
|
40
|
-
|
|
64
|
+
}, {
|
|
65
|
+
endStream: true,
|
|
66
|
+
});
|
|
67
|
+
request.on("response", (headers) => {
|
|
68
|
+
resolve({
|
|
69
|
+
status: headers[":status"],
|
|
70
|
+
headers: new Headers(Object.entries(headers).filter(([header, value]) => !header.startsWith(":") && value !== undefined)),
|
|
71
|
+
ok: headers[":status"] >= 200 && headers[":status"] < 300,
|
|
72
|
+
});
|
|
73
|
+
// terminate the request
|
|
74
|
+
request.destroy();
|
|
75
|
+
// release session back to pool
|
|
76
|
+
sessions.push(session);
|
|
77
|
+
});
|
|
78
|
+
request.on("error", reject);
|
|
79
|
+
request.end();
|
|
41
80
|
});
|
|
42
81
|
}
|
|
82
|
+
// Add a method to close sessions when done
|
|
83
|
+
destroy() {
|
|
84
|
+
for (const sessions of this.sessions.values()) {
|
|
85
|
+
for (const session of sessions) {
|
|
86
|
+
session.destroy();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
this.sessions.clear();
|
|
90
|
+
}
|
|
43
91
|
setIsRetry() {
|
|
44
92
|
this.isRetry = true;
|
|
45
93
|
this.rateLimit = this.lowestRateLimit;
|
|
@@ -94,5 +142,13 @@ class AssetApiClient {
|
|
|
94
142
|
// on any other customVariation set variation as it is
|
|
95
143
|
assetUrl.searchParams.set("bqvariation", variation);
|
|
96
144
|
}
|
|
145
|
+
async sleepByRate() {
|
|
146
|
+
const now = Date.now();
|
|
147
|
+
const timeSinceLastRequest = now - this.lastRequestTime;
|
|
148
|
+
const timeBetweenRequests = 1000 / this.rateLimit;
|
|
149
|
+
const sleepTime = Math.max(0, timeBetweenRequests - timeSinceLastRequest);
|
|
150
|
+
this.lastRequestTime = now + sleepTime;
|
|
151
|
+
await new Promise((resolve) => setTimeout(resolve, sleepTime));
|
|
152
|
+
}
|
|
97
153
|
}
|
|
98
154
|
exports.AssetApiClient = AssetApiClient;
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import { AssetItemInterface, ItemStatus } from "../";
|
|
2
|
-
import { AssetApiClient } from "./asset-api-client";
|
|
3
2
|
export declare class AssetItem implements AssetItemInterface {
|
|
4
3
|
readonly url: string;
|
|
5
4
|
readonly variation: string;
|
|
6
|
-
readonly callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>;
|
|
7
5
|
error?: Error;
|
|
8
6
|
status: ItemStatus;
|
|
9
|
-
constructor(url: string, variation: string
|
|
7
|
+
constructor(url: string, variation: string);
|
|
10
8
|
done(): void;
|
|
11
9
|
failed(): void;
|
|
12
10
|
retry(): void;
|
|
13
11
|
setError(error: Error): void;
|
|
12
|
+
toString(): string;
|
|
14
13
|
}
|
|
@@ -5,13 +5,11 @@ const __1 = require("../");
|
|
|
5
5
|
class AssetItem {
|
|
6
6
|
url;
|
|
7
7
|
variation;
|
|
8
|
-
callback;
|
|
9
8
|
error = null;
|
|
10
9
|
status = __1.ItemStatus.NEW;
|
|
11
|
-
constructor(url, variation
|
|
10
|
+
constructor(url, variation) {
|
|
12
11
|
this.url = url;
|
|
13
12
|
this.variation = variation;
|
|
14
|
-
this.callback = callback;
|
|
15
13
|
}
|
|
16
14
|
done() {
|
|
17
15
|
this.status = __1.ItemStatus.DONE;
|
|
@@ -25,5 +23,8 @@ class AssetItem {
|
|
|
25
23
|
setError(error) {
|
|
26
24
|
this.error = error;
|
|
27
25
|
}
|
|
26
|
+
toString() {
|
|
27
|
+
return `${this.url} - ${this.variation} -> ${this.status}: ${this.error?.message ?? ""}`;
|
|
28
|
+
}
|
|
28
29
|
}
|
|
29
30
|
exports.AssetItem = AssetItem;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { AssetItemInterface, ItemStatus } from "../";
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import { AssetItem } from "./asset-item";
|
|
3
|
+
export declare class AssetList implements Iterable<AssetItem> {
|
|
4
4
|
private elements;
|
|
5
|
-
|
|
5
|
+
[Symbol.iterator](): Iterator<AssetItem>;
|
|
6
6
|
addItem(item: AssetItemInterface): void;
|
|
7
|
-
getBatch(offset: number, limit: number): AssetItemInterface[];
|
|
8
7
|
getByStatus(status: ItemStatus): AssetItemInterface[];
|
|
9
8
|
getErrorCount(): number;
|
|
10
9
|
getErrors(): Error[];
|
|
@@ -3,24 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.AssetList = void 0;
|
|
4
4
|
const __1 = require("../");
|
|
5
5
|
class AssetList {
|
|
6
|
-
variations;
|
|
7
6
|
elements = [];
|
|
8
|
-
|
|
9
|
-
this.
|
|
7
|
+
[Symbol.iterator]() {
|
|
8
|
+
return this.elements[Symbol.iterator]();
|
|
10
9
|
}
|
|
11
10
|
addItem(item) {
|
|
12
11
|
this.elements.push(item);
|
|
13
12
|
}
|
|
14
|
-
getBatch(offset, limit) {
|
|
15
|
-
let batchLimit = offset + limit;
|
|
16
|
-
if (offset >= this.elements.length) {
|
|
17
|
-
return [];
|
|
18
|
-
}
|
|
19
|
-
if (batchLimit >= this.elements.length) {
|
|
20
|
-
batchLimit = this.elements.length;
|
|
21
|
-
}
|
|
22
|
-
return this.elements.slice(offset, batchLimit);
|
|
23
|
-
}
|
|
24
13
|
getByStatus(status) {
|
|
25
14
|
return this.elements.filter((element) => element.status === status);
|
|
26
15
|
}
|
|
@@ -19,7 +19,7 @@ class PreWarmFactory {
|
|
|
19
19
|
const assetApiClient = new asset_api_client_1.AssetApiClient(entityManager.token, this.context.app, this.context.keepParameterSorting);
|
|
20
20
|
const urls = this.getUrls(csvReader);
|
|
21
21
|
const variations = this.getVariations(csvReader);
|
|
22
|
-
return new _1.PreWarmService(assetApiClient, cli, urls, variations);
|
|
22
|
+
return new _1.PreWarmService(assetApiClient, cli, urls, variations, this.context.verbose);
|
|
23
23
|
}
|
|
24
24
|
getUrls(csvReader) {
|
|
25
25
|
return csvReader.readPath(this.context.path);
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { AssetApiClient } from "./assets/asset-api-client";
|
|
2
1
|
export declare class PreWarmContext {
|
|
3
2
|
readonly app: string;
|
|
4
3
|
readonly path: string;
|
|
@@ -6,10 +5,10 @@ export declare class PreWarmContext {
|
|
|
6
5
|
readonly variationPath?: string;
|
|
7
6
|
readonly keepParameterSorting: boolean;
|
|
8
7
|
readonly quiet: boolean;
|
|
9
|
-
|
|
8
|
+
readonly verbose: boolean;
|
|
9
|
+
constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean, verbose?: boolean);
|
|
10
10
|
}
|
|
11
11
|
export interface AssetItemInterface {
|
|
12
|
-
callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>;
|
|
13
12
|
done(): void;
|
|
14
13
|
error?: Error;
|
|
15
14
|
failed(): void;
|
|
@@ -34,7 +33,7 @@ export declare const RATE_LIMIT_MULTIPLICATOR = 0.8;
|
|
|
34
33
|
export declare const RATE_LIMIT_CODES: string[];
|
|
35
34
|
export declare const TIMING_HEADER_ERROR_CODE_SELECTOR: RegExp;
|
|
36
35
|
export declare const TIMING_HEADER_KEY = "server-timing";
|
|
37
|
-
export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {
|
|
36
|
+
export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
|
|
38
37
|
export declare const VARIATION_SEPARATOR = ",";
|
|
39
38
|
export declare const DEFAULT_VARIATION: string[];
|
|
40
39
|
export declare const DEFAULT_VARIATIONS: string[];
|
|
@@ -8,13 +8,15 @@ class PreWarmContext {
|
|
|
8
8
|
variationPath;
|
|
9
9
|
keepParameterSorting;
|
|
10
10
|
quiet;
|
|
11
|
-
|
|
11
|
+
verbose;
|
|
12
|
+
constructor(app, path, variation, variationPath, keepParameterSorting = true, quiet = false, verbose = false) {
|
|
12
13
|
this.app = app;
|
|
13
14
|
this.path = path;
|
|
14
15
|
this.variation = variation;
|
|
15
16
|
this.variationPath = variationPath;
|
|
16
17
|
this.keepParameterSorting = keepParameterSorting;
|
|
17
18
|
this.quiet = quiet;
|
|
19
|
+
this.verbose = verbose;
|
|
18
20
|
}
|
|
19
21
|
}
|
|
20
22
|
exports.PreWarmContext = PreWarmContext;
|
|
@@ -31,10 +33,11 @@ exports.TIMOUT_AFTER_DETECTED_RATE_LIMIT = 5000;
|
|
|
31
33
|
exports.DEFAULT_REQUESTS_PER_SECOND = 10;
|
|
32
34
|
exports.MAX_REQUESTS_PER_SECOND = 20;
|
|
33
35
|
exports.RATE_LIMIT_MULTIPLICATOR = 0.8;
|
|
34
|
-
|
|
36
|
+
// we also use some codes that indicate some server overloads
|
|
37
|
+
exports.RATE_LIMIT_CODES = ["306", "308", "310", "311", "314", "316"];
|
|
35
38
|
exports.TIMING_HEADER_ERROR_CODE_SELECTOR = /errorcode;desc=([^,]*),/;
|
|
36
39
|
exports.TIMING_HEADER_KEY = "server-timing";
|
|
37
|
-
exports.PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {
|
|
40
|
+
exports.PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
|
|
38
41
|
exports.VARIATION_SEPARATOR = ",";
|
|
39
42
|
exports.DEFAULT_VARIATION = ["default"];
|
|
40
43
|
exports.DEFAULT_VARIATIONS = ["mobile", "tablet", "tv"];
|
|
@@ -6,15 +6,16 @@ export declare class PreWarmService {
|
|
|
6
6
|
private cli;
|
|
7
7
|
private urls;
|
|
8
8
|
private variations;
|
|
9
|
+
private verbose;
|
|
9
10
|
private requestsSend;
|
|
10
|
-
constructor(assetClient: AssetApiClient, cli: CliServiceInterface, urls: string[], variations: string[]);
|
|
11
|
-
|
|
11
|
+
constructor(assetClient: AssetApiClient, cli: CliServiceInterface, urls: string[], variations: string[], verbose: boolean);
|
|
12
|
+
handleAsset(assetClient: AssetApiClient, entry: AssetItemInterface): Promise<void>;
|
|
12
13
|
run(): Promise<void>;
|
|
13
|
-
private
|
|
14
|
+
private fetchAssetList;
|
|
15
|
+
private handleProgress;
|
|
14
16
|
private fetchUrls;
|
|
15
17
|
private generateAssetRequestList;
|
|
16
18
|
private printSummary;
|
|
17
|
-
private sleep;
|
|
18
19
|
private startProgressBar;
|
|
19
20
|
private stopProgressBar;
|
|
20
21
|
private updateProgressBar;
|
|
@@ -10,15 +10,17 @@ class PreWarmService {
|
|
|
10
10
|
cli;
|
|
11
11
|
urls;
|
|
12
12
|
variations;
|
|
13
|
+
verbose;
|
|
13
14
|
requestsSend = 0;
|
|
14
|
-
constructor(assetClient, cli, urls, variations) {
|
|
15
|
+
constructor(assetClient, cli, urls, variations, verbose) {
|
|
15
16
|
this.assetClient = assetClient;
|
|
16
17
|
this.cli = cli;
|
|
17
18
|
this.urls = urls;
|
|
18
19
|
this.variations = variations;
|
|
20
|
+
this.verbose = verbose;
|
|
19
21
|
}
|
|
20
|
-
async
|
|
21
|
-
const fetchUrlResponse = await (0, safe_1.safe)(assetClient.fetchURL(entry.url, entry.variation));
|
|
22
|
+
async handleAsset(assetClient, entry) {
|
|
23
|
+
const fetchUrlResponse = await (0, safe_1.safe)(assetClient.fetchURL(entry.url, entry.variation, this.verbose));
|
|
22
24
|
// a generic error occurred
|
|
23
25
|
if (fetchUrlResponse.success === false) {
|
|
24
26
|
entry.failed();
|
|
@@ -67,38 +69,55 @@ class PreWarmService {
|
|
|
67
69
|
await this.fetchUrls(retryList);
|
|
68
70
|
await this.printSummary(retryList);
|
|
69
71
|
}
|
|
70
|
-
async
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
72
|
+
async fetchAssetList(assetList) {
|
|
73
|
+
const iterator = assetList[Symbol.iterator]();
|
|
74
|
+
// run with 20 concurrent requests
|
|
75
|
+
let batch = [];
|
|
76
|
+
let completed = 0;
|
|
77
|
+
let lastTime = Date.now();
|
|
78
|
+
const workers = Array.from({ length: 10 }).map(async () => {
|
|
79
|
+
for (;;) {
|
|
80
|
+
const { done, value } = iterator.next();
|
|
81
|
+
if (done) {
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
await this.handleAsset(this.assetClient, value);
|
|
85
|
+
if (this.verbose && value.status !== _1.ItemStatus.DONE) {
|
|
86
|
+
this.cli.write(`${value.toString()}\n`);
|
|
87
|
+
}
|
|
88
|
+
completed++;
|
|
89
|
+
batch.push(value);
|
|
90
|
+
if (completed % 20 === 0) {
|
|
91
|
+
if (this.verbose)
|
|
92
|
+
this.cli.write("\n");
|
|
93
|
+
this.handleProgress(completed, assetList, lastTime, batch);
|
|
94
|
+
if (this.verbose)
|
|
95
|
+
this.cli.write("\n");
|
|
96
|
+
lastTime = Date.now();
|
|
97
|
+
batch = [];
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
await Promise.all(workers);
|
|
102
|
+
this.handleProgress(completed, assetList, lastTime, batch);
|
|
103
|
+
}
|
|
104
|
+
handleProgress(completed, assetList, lastTime, batch) {
|
|
105
|
+
this.updateProgressBar(completed, assetList, Date.now() - lastTime);
|
|
106
|
+
this.updateRateLimit(batch);
|
|
87
107
|
this.showMinimalStatusUpdate(batch, assetList);
|
|
88
|
-
// fetch next batch
|
|
89
|
-
await this.fetchBatch(assetList, offset);
|
|
90
108
|
}
|
|
91
109
|
async fetchUrls(assetList) {
|
|
92
110
|
this.cli.write(`start prewarm ${assetList.getLength()} assets`);
|
|
93
111
|
this.startProgressBar(assetList);
|
|
94
|
-
await this.
|
|
112
|
+
await this.fetchAssetList(assetList);
|
|
95
113
|
this.stopProgressBar();
|
|
96
114
|
}
|
|
97
115
|
generateAssetRequestList(urls) {
|
|
98
|
-
const assetList = new asset_list_1.AssetList(
|
|
116
|
+
const assetList = new asset_list_1.AssetList();
|
|
99
117
|
for (const variation of this.variations) {
|
|
100
|
-
for (const url of urls)
|
|
101
|
-
assetList.addItem(new asset_item_1.AssetItem(url, variation
|
|
118
|
+
for (const url of urls) {
|
|
119
|
+
assetList.addItem(new asset_item_1.AssetItem(url, variation));
|
|
120
|
+
}
|
|
102
121
|
}
|
|
103
122
|
return assetList;
|
|
104
123
|
}
|
|
@@ -125,33 +144,29 @@ class PreWarmService {
|
|
|
125
144
|
});
|
|
126
145
|
}
|
|
127
146
|
}
|
|
128
|
-
async sleep(ms) {
|
|
129
|
-
return new Promise((resolve) => {
|
|
130
|
-
setTimeout(resolve, ms);
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
147
|
startProgressBar(assetList) {
|
|
134
148
|
this.cli.startProgress(assetList.getLength(), 0, {
|
|
135
149
|
errors: 0,
|
|
136
150
|
rate: this.assetClient.getRateLimit(),
|
|
137
151
|
retry: 0,
|
|
138
|
-
|
|
152
|
+
batchDuration: "-",
|
|
153
|
+
}, { format: _1.PROGRESS_BAR_FORMAT, linewrap: !!this.verbose });
|
|
139
154
|
}
|
|
140
155
|
stopProgressBar() {
|
|
141
156
|
this.cli.stopProgress();
|
|
142
157
|
}
|
|
143
|
-
updateProgressBar(offset, assetList) {
|
|
158
|
+
updateProgressBar(offset, assetList, batchDurationInMs) {
|
|
144
159
|
this.cli.updateProgress(offset, {
|
|
145
160
|
errors: assetList.getErrorCount(),
|
|
146
|
-
rate: this.assetClient.getRateLimit()
|
|
161
|
+
rate: `${this.assetClient.getRateLimit()} req/s`,
|
|
147
162
|
retry: assetList.getRetryCount(),
|
|
163
|
+
batchDuration: `${Math.round(batchDurationInMs / 100) / 10}s`,
|
|
148
164
|
});
|
|
149
165
|
}
|
|
150
|
-
|
|
166
|
+
updateRateLimit(batch) {
|
|
151
167
|
const retryCount = batch.filter((entry) => entry.status === _1.ItemStatus.RETRY).length;
|
|
152
168
|
// found rateLimiterResponse wait and reduce rateLimit
|
|
153
169
|
if (retryCount > 0) {
|
|
154
|
-
await this.sleep(_1.TIMOUT_AFTER_DETECTED_RATE_LIMIT);
|
|
155
170
|
this.assetClient.reduceMaxRequestsPerSecond();
|
|
156
171
|
return;
|
|
157
172
|
}
|
package/oclif.manifest.json
CHANGED
|
@@ -552,7 +552,7 @@
|
|
|
552
552
|
"variation": {
|
|
553
553
|
"char": "v",
|
|
554
554
|
"name": "variation",
|
|
555
|
-
"default": "
|
|
555
|
+
"default": "default",
|
|
556
556
|
"hasDynamicHelp": false,
|
|
557
557
|
"multiple": false,
|
|
558
558
|
"type": "option"
|
|
@@ -582,6 +582,12 @@
|
|
|
582
582
|
"allowNo": false,
|
|
583
583
|
"type": "boolean"
|
|
584
584
|
},
|
|
585
|
+
"verbose": {
|
|
586
|
+
"description": "Log information about failed requests",
|
|
587
|
+
"name": "verbose",
|
|
588
|
+
"allowNo": false,
|
|
589
|
+
"type": "boolean"
|
|
590
|
+
},
|
|
585
591
|
"delay": {
|
|
586
592
|
"char": "d",
|
|
587
593
|
"deprecated": true,
|
|
@@ -732,5 +738,5 @@
|
|
|
732
738
|
]
|
|
733
739
|
}
|
|
734
740
|
},
|
|
735
|
-
"version": "3.
|
|
741
|
+
"version": "3.20.0"
|
|
736
742
|
}
|