@speedkit/cli 3.18.0 → 3.20.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [3.20.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.19.0...v3.20.0) (2026-01-06)
+
+
+### Features
+
+* **prewarm:** improve concurrency handling of prewarming ([509d19f](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/509d19f8d0a7131e81906c18ce83a338aa3c4c82))
+
+# [3.19.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.18.0...v3.19.0) (2025-12-19)
+
+
+### Features
+
+* replace cf rocketloader workaround with our plugin ([1ca3bb4](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/1ca3bb4d5563e22bf9df6d5c7fc263638b412b38))
+
 # [3.18.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.17.0...v3.18.0) (2025-12-12)
 
 

package/README.md

@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
 $ sk COMMAND
 running command...
 $ sk (--version)
-@speedkit/cli/3.18.0 linux-x64 node-v20.19.6
+@speedkit/cli/3.20.0 linux-x64 node-v20.19.6
 $ sk --help [COMMAND]
 USAGE
   $ sk COMMAND
@@ -437,7 +437,7 @@ Pre-warm Fastly
 
 ```
 USAGE
-  $ sk prewarm APP PATH [-p <value> | -v <value>] [-k] [-q] [-d <value>]
+  $ sk prewarm APP PATH [-p <value> | -v <value>] [-k] [-q] [--verbose] [-d <value>]
 
 ARGUMENTS
   APP   The customer app name
@@ -448,7 +448,8 @@ FLAGS
   -k, --keepParameterSorting   Prevents alphabetical sorting of the URL parameters
   -p, --variationPath=<value>  path to variations.csv
   -q, --quiet                  less output non interactive
-  -v, --variation=<value>      [default: DEFAULT]
+  -v, --variation=<value>      [default: default]
+      --verbose                Log information about failed requests
 
 DESCRIPTION
   Pre-warm Fastly

package/dist/commands/prewarm.d.ts

@@ -12,6 +12,7 @@ export default class Prewarm extends Command {
         variationPath: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
         keepParameterSorting: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
         quiet: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
+        verbose: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
         delay: import("@oclif/core/lib/interfaces").OptionFlag<number, import("@oclif/core/lib/interfaces").CustomOptions>;
     };
     static examples: string[];

package/dist/commands/prewarm.js

@@ -20,7 +20,7 @@ class Prewarm extends core_1.Command {
     static flags = {
         [cli_parameters_1.CLIParameters.Variation]: core_1.Flags.string({
             char: cli_parameters_1.CLIParametersChar.Variation,
-            default: "DEFAULT",
+            default: "default",
         }),
         [cli_parameters_1.CLIParameters.VariationPath]: core_1.Flags.file({
             char: cli_parameters_1.CLIParametersChar.VariationPath,
@@ -37,6 +37,10 @@ class Prewarm extends core_1.Command {
             default: false,
             description: "less output non interactive",
         }),
+        [cli_parameters_1.CLIParameters.Verbose]: core_1.Flags.boolean({
+            default: false,
+            description: "Log information about failed requests",
+        }),
         ["delay"]: core_1.Flags.integer({
             char: "d",
             description: "Request delay in ms (deprecated)",
@@ -54,9 +58,9 @@ class Prewarm extends core_1.Command {
         `$ sk prewarm ${cli_parameters_1.CLIParametersExample.AppName} ${cli_parameters_1.CLIParametersExample.Path} -${cli_parameters_1.CLIParametersChar.Path} ${cli_parameters_1.CLIParametersExample.VariationPath} -${cli_parameters_1.CLIParametersChar.KeepParameterSorting}`,
     ];
     async run() {
-        const { args: { app, path }, flags: { variation, variationPath, keepParameterSorting, quiet }, } = await this.parse(Prewarm);
+        const { args: { app, path }, flags: { variation, variationPath, keepParameterSorting, quiet, verbose }, } = await this.parse(Prewarm);
         const sortParameters = !keepParameterSorting;
-        const service = await new prewarm_1.PreWarmFactory(new prewarm_1.PreWarmContext(app, path, variation, variationPath, sortParameters, quiet)).getService();
+        const service = await new prewarm_1.PreWarmFactory(new prewarm_1.PreWarmContext(app, path, variation, variationPath, sortParameters, quiet, verbose)).getService();
         await service.run();
     }
 }

package/dist/models/cli-parameters.d.ts

@@ -23,6 +23,7 @@ export declare enum CLIParameters {
     Urls = "urls",
     Variation = "variation",
     VariationPath = "variationPath",
+    Verbose = "verbose",
     useTestConfig = "useTestConfig"
 }
 export declare enum CLIParametersChar {

package/dist/models/cli-parameters.js

@@ -28,6 +28,7 @@ var CLIParameters;
     CLIParameters["Urls"] = "urls";
     CLIParameters["Variation"] = "variation";
     CLIParameters["VariationPath"] = "variationPath";
+    CLIParameters["Verbose"] = "verbose";
     CLIParameters["useTestConfig"] = "useTestConfig";
 })(CLIParameters || (exports.CLIParameters = CLIParameters = {}));
 /* eslint-disable @typescript-eslint/no-duplicate-enum-values */

package/dist/services/customer-config/customer-config-service-model.d.ts

@@ -35,7 +35,7 @@ export interface CustomerConfigSettings {
     useGATracking?: boolean;
     useScrapingBee?: boolean;
     withGoogleOptimize?: boolean;
-    activateCfRocketLoaderWorkaround?: boolean;
+    activateCfRocketLoaderPlugin?: boolean;
     hasSoftNavigations?: boolean;
     isPOV?: boolean;
     isShopify?: boolean;

package/dist/services/customer-config/customer-config-service.js

@@ -136,7 +136,7 @@ class CustomerConfigService {
         return true;
     }
     async getConfigSettings() {
-        let { production, staging, activateScopedDeployments, subRouteScope, includeServiceWorker, activateImageOptimisation, removeLazyLoading, addSSR, addSSRInnerShadowDomSupport, addDeviceDetection, useGATracking, useScrapingBee, withGoogleOptimize, activateCfRocketLoaderWorkaround, hasSoftNavigations, isShopify, isShopware, isSalesforce, isOxid, isPlentymarkets, isPOV, } = {};
+        let { production, staging, activateScopedDeployments, subRouteScope, includeServiceWorker, activateImageOptimisation, removeLazyLoading, addSSR, addSSRInnerShadowDomSupport, addDeviceDetection, useGATracking, useScrapingBee, withGoogleOptimize, activateCfRocketLoaderPlugin, hasSoftNavigations, isShopify, isShopware, isSalesforce, isOxid, isPlentymarkets, isPOV, } = {};
         this.appName = await this.cli.prompt(`[App Name] Enter desired SK app name:`, {
             validator: (input) => /^[\da-z]+(?:-[\da-z]+)*$/.test(input) ? "" : "No valid kebab-case",
             defaultAnswer: this.appName,
@@ -200,9 +200,7 @@ class CustomerConfigService {
         hasSoftNavigations = await this.cli.confirm("[Tracking] Add tracking for Soft Navigations?", false);
         isPOV = await this.cli.confirm("[Tracking] Is a POV (proof of value) anticipated, so that additional tracking is needed?", false);
         withGoogleOptimize = await this.cli.confirm("[Services] Is Google Optimize used?", false);
-        activateCfRocketLoaderWorkaround = isShopify
-            ? true
-            : await this.cli.confirm("[Services] Is Cloudflare's Rocket Loader used, for which we would need to activate a workaround?", false);
+        activateCfRocketLoaderPlugin = await this.cli.confirm("[Services] Is Cloudflare's Rocket Loader used, for which we need to activate our rocket loader plugin?", false);
         return {
             appName: this.appName,
             production,
@@ -218,7 +216,7 @@ class CustomerConfigService {
             useGATracking,
             useScrapingBee,
             withGoogleOptimize,
-            activateCfRocketLoaderWorkaround,
+            activateCfRocketLoaderPlugin,
             hasSoftNavigations,
             isShopify,
             isShopware,

package/dist/services/customer-config/templates/config_documentHandler.js.hbs

@@ -119,6 +119,9 @@ const config = {
   stripComments: false, // Plentymarkets specific
 {{/if}}
   ignoreAssets: [],
+{{#if activateCfRocketLoaderPlugin}}
+  handleRocketLoader: true,
+{{/if}}
 {{#if isShopify}}
   delayScriptPath: "/apps/speed-kit/speed-kit-dom-ready.js", // Shopify specific
 {{else}}

package/dist/services/customer-config/templates/config_dynamicBlocks.es6.hbs

@@ -89,23 +89,6 @@
         detectChanges: [{compare: ['text']}],
       },
 {{/if}}
-{{#if activateCfRocketLoaderWorkaround}}
-
-      // Workaround for Cloudflare's Rocket Loader (e.g. needed for Shopify)
-      {
-        selector: "script[data-cf-settings]",
-        mergeFunction: function (localBlock, remoteBlock) {
-          let cFHash = remoteBlock.getAttribute("data-cf-settings");
-          localBlock.setAttribute("data-cf-settings", cFHash);
-          let deferScripts = document.querySelectorAll("script[src][defer]");
-                deferScripts.forEach((deferScript) => {
-            let type = deferScript.type;
-            type = type.replace(/[a-f0-9]{24}/, cFHash.split('-')[0]);
-            deferScript.type = type;
-          })
-        },
-      },
-{{/if}}
 
       // default entries
       { selector: 'link[rel="canonical"]'},

package/dist/services/prewarm/assets/asset-api-client.d.ts

@@ -1,13 +1,21 @@
+export type SimpleResponse = {
+    headers: Headers;
+    status: number;
+    ok: boolean;
+};
 export declare class AssetApiClient {
     private authenticationToken;
     private app;
     private sortParameters;
-    private agent;
+    private sessions;
     private rateLimit;
     private lowestRateLimit;
     private isRetry;
+    private lastRequestTime;
     constructor(authenticationToken: string, app: string, sortParameters: boolean);
-    fetchURL(url: string, variation: string): Promise<Response>;
+    fetchURL(url: string, variation: string, verbose: boolean): Promise<SimpleResponse>;
+    private makeHttp2Request;
+    destroy(): void;
     setIsRetry(): void;
     getRateLimit(): number;
     reduceMaxRequestsPerSecond(): void;
@@ -30,4 +38,5 @@ export declare class AssetApiClient {
      * @private
      */
     private setVariationParameter;
+    private sleepByRate;
 }

package/dist/services/prewarm/assets/asset-api-client.js

@@ -2,9 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AssetApiClient = void 0;
 const tslib_1 = require("tslib");
-const node_https_1 = require("node:https");
 const node_url_1 = require("node:url");
-const node_fetch_1 = tslib_1.__importDefault(require("node-fetch"));
+const node_http2_1 = tslib_1.__importDefault(require("node:http2"));
 const __1 = require("../");
 const normalize_1 = require("../../../helpers/normalize");
 const invalid_origin_error_1 = require("../error/invalid-origin-error");
@@ -12,34 +11,83 @@ class AssetApiClient {
     authenticationToken;
     app;
     sortParameters;
-    agent = new node_https_1.Agent({
-        keepAlive: true,
-        rejectUnauthorized: false,
-    });
+    sessions = new Map();
     rateLimit = __1.DEFAULT_REQUESTS_PER_SECOND;
     lowestRateLimit = __1.DEFAULT_REQUESTS_PER_SECOND;
     isRetry = false;
+    lastRequestTime = Date.now();
     constructor(authenticationToken, app, sortParameters) {
         this.authenticationToken = authenticationToken;
         this.app = app;
         this.sortParameters = sortParameters;
     }
-    async fetchURL(url, variation) {
+    async fetchURL(url, variation, verbose) {
+        await this.sleepByRate();
         this.isValidUrl(url);
         const assetUrl = new node_url_1.URL((0, normalize_1.normalize)(url, this.sortParameters));
         this.setVariationParameter(variation, assetUrl);
         assetUrl.searchParams.set("bqpass", "1");
-        return (0, node_fetch_1.default)(`https://${this.app}.app.baqend.com/v1/asset/${assetUrl.toString()}`, {
-            agent: this.agent,
-            headers: {
-                "Accept-Encoding": "gzip, deflate, br, dcb",
-                Origin: assetUrl.origin,
+        const origin = `https://${this.app}.app.baqend.com`;
+        const start = Date.now();
+        const result = await this.makeHttp2Request(origin, `/v1/asset/${assetUrl.toString()}`, assetUrl.origin);
+        if (verbose) {
+            console.log(`fetch ${url} - ${variation} took ${Date.now() - start}ms, cdnTime: ${result.headers.get("x-timer") || "-"}, status: ${result.status}, cache: ${result.headers.get("x-cache")}, timings: ${result.headers.get(__1.TIMING_HEADER_KEY)}`);
+        }
+        return result;
+    }
+    makeHttp2Request(origin, path, assetOrigin) {
+        return new Promise((resolve, reject) => {
+            // we are using a session pool for requests since node seems to stack the requests up when using the same
+            // http2 session for several requests concurrently. This causes then that a single slow request can completely
+            // delay all requests wich are issued afterward
+            let sessions = this.sessions.get(origin);
+            if (!sessions) {
+                sessions = [];
+                this.sessions.set(origin, sessions);
+            }
+            let session = sessions?.shift();
+            if (!session || session.destroyed || session.closed) {
+                session = node_http2_1.default.connect(origin, {
+                    rejectUnauthorized: false,
+                });
+                session.on("error", (error) => {
+                    reject(error);
+                });
+            }
+            const request = session.request({
+                [node_http2_1.default.constants.HTTP2_HEADER_METHOD]: "HEAD",
+                [node_http2_1.default.constants.HTTP2_HEADER_PATH]: path,
                 authorization: `BAT ${this.authenticationToken}`,
+                origin: assetOrigin,
+                "accept-encoding": "gzip, deflate, br, dcb",
                 "baqend-choose-dictionary": "1",
-            },
-            method: "HEAD",
+            }, {
+                endStream: true,
+            });
+            request.on("response", (headers) => {
+                resolve({
+                    status: headers[":status"],
+                    headers: new Headers(Object.entries(headers).filter(([header, value]) => !header.startsWith(":") && value !== undefined)),
+                    ok: headers[":status"] >= 200 && headers[":status"] < 300,
+                });
+                // terminate the request
+                request.destroy();
+                // release session back to pool
+                sessions.push(session);
+            });
+            request.on("error", reject);
+            request.end();
         });
     }
+    // Add a method to close sessions when done
+    destroy() {
+        for (const sessions of this.sessions.values()) {
+            for (const session of sessions) {
+                session.destroy();
+            }
+        }
+        this.sessions.clear();
+    }
     setIsRetry() {
         this.isRetry = true;
         this.rateLimit = this.lowestRateLimit;
@@ -94,5 +142,13 @@ class AssetApiClient {
         // on any other customVariation set variation as it is
         assetUrl.searchParams.set("bqvariation", variation);
     }
+    async sleepByRate() {
+        const now = Date.now();
+        const timeSinceLastRequest = now - this.lastRequestTime;
+        const timeBetweenRequests = 1000 / this.rateLimit;
+        const sleepTime = Math.max(0, timeBetweenRequests - timeSinceLastRequest);
+        this.lastRequestTime = now + sleepTime;
+        await new Promise((resolve) => setTimeout(resolve, sleepTime));
+    }
 }
 exports.AssetApiClient = AssetApiClient;

package/dist/services/prewarm/assets/asset-item.d.ts

@@ -1,14 +1,13 @@
 import { AssetItemInterface, ItemStatus } from "../";
-import { AssetApiClient } from "./asset-api-client";
 export declare class AssetItem implements AssetItemInterface {
     readonly url: string;
     readonly variation: string;
-    readonly callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>;
     error?: Error;
     status: ItemStatus;
-    constructor(url: string, variation: string, callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>);
+    constructor(url: string, variation: string);
     done(): void;
     failed(): void;
     retry(): void;
     setError(error: Error): void;
+    toString(): string;
 }

package/dist/services/prewarm/assets/asset-item.js

@@ -5,13 +5,11 @@ const __1 = require("../");
 class AssetItem {
     url;
     variation;
-    callback;
     error = null;
     status = __1.ItemStatus.NEW;
-    constructor(url, variation, callback) {
+    constructor(url, variation) {
         this.url = url;
         this.variation = variation;
-        this.callback = callback;
     }
     done() {
         this.status = __1.ItemStatus.DONE;
@@ -25,5 +23,8 @@ class AssetItem {
     setError(error) {
         this.error = error;
     }
+    toString() {
+        return `${this.url} - ${this.variation} -> ${this.status}: ${this.error?.message ?? ""}`;
+    }
 }
 exports.AssetItem = AssetItem;

package/dist/services/prewarm/assets/asset-list.d.ts

@@ -1,10 +1,9 @@
 import { AssetItemInterface, ItemStatus } from "../";
-export declare class AssetList {
-    private variations;
+import { AssetItem } from "./asset-item";
+export declare class AssetList implements Iterable<AssetItem> {
     private elements;
-    constructor(variations: string[]);
+    [Symbol.iterator](): Iterator<AssetItem>;
     addItem(item: AssetItemInterface): void;
-    getBatch(offset: number, limit: number): AssetItemInterface[];
     getByStatus(status: ItemStatus): AssetItemInterface[];
     getErrorCount(): number;
     getErrors(): Error[];

package/dist/services/prewarm/assets/asset-list.js

@@ -3,24 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AssetList = void 0;
 const __1 = require("../");
 class AssetList {
-    variations;
     elements = [];
-    constructor(variations) {
-        this.variations = variations;
+    [Symbol.iterator]() {
+        return this.elements[Symbol.iterator]();
     }
     addItem(item) {
         this.elements.push(item);
     }
-    getBatch(offset, limit) {
-        let batchLimit = offset + limit;
-        if (offset >= this.elements.length) {
-            return [];
-        }
-        if (batchLimit >= this.elements.length) {
-            batchLimit = this.elements.length;
-        }
-        return this.elements.slice(offset, batchLimit);
-    }
     getByStatus(status) {
         return this.elements.filter((element) => element.status === status);
     }

package/dist/services/prewarm/pre-warm-factory.js

@@ -19,7 +19,7 @@ class PreWarmFactory {
         const assetApiClient = new asset_api_client_1.AssetApiClient(entityManager.token, this.context.app, this.context.keepParameterSorting);
         const urls = this.getUrls(csvReader);
         const variations = this.getVariations(csvReader);
-        return new _1.PreWarmService(assetApiClient, cli, urls, variations);
+        return new _1.PreWarmService(assetApiClient, cli, urls, variations, this.context.verbose);
     }
     getUrls(csvReader) {
         return csvReader.readPath(this.context.path);

package/dist/services/prewarm/pre-warm-model.d.ts

@@ -1,4 +1,3 @@
-import { AssetApiClient } from "./assets/asset-api-client";
 export declare class PreWarmContext {
     readonly app: string;
     readonly path: string;
@@ -6,10 +5,10 @@ export declare class PreWarmContext {
     readonly variationPath?: string;
     readonly keepParameterSorting: boolean;
     readonly quiet: boolean;
-    constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean);
+    readonly verbose: boolean;
+    constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean, verbose?: boolean);
 }
 export interface AssetItemInterface {
-    callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>;
     done(): void;
     error?: Error;
     failed(): void;
@@ -34,7 +33,7 @@ export declare const RATE_LIMIT_MULTIPLICATOR = 0.8;
 export declare const RATE_LIMIT_CODES: string[];
 export declare const TIMING_HEADER_ERROR_CODE_SELECTOR: RegExp;
 export declare const TIMING_HEADER_KEY = "server-timing";
-export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry}";
+export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
 export declare const VARIATION_SEPARATOR = ",";
 export declare const DEFAULT_VARIATION: string[];
 export declare const DEFAULT_VARIATIONS: string[];

package/dist/services/prewarm/pre-warm-model.js

@@ -8,13 +8,15 @@ class PreWarmContext {
     variationPath;
     keepParameterSorting;
     quiet;
-    constructor(app, path, variation, variationPath, keepParameterSorting = true, quiet = false) {
+    verbose;
+    constructor(app, path, variation, variationPath, keepParameterSorting = true, quiet = false, verbose = false) {
         this.app = app;
         this.path = path;
         this.variation = variation;
         this.variationPath = variationPath;
         this.keepParameterSorting = keepParameterSorting;
         this.quiet = quiet;
+        this.verbose = verbose;
     }
 }
 exports.PreWarmContext = PreWarmContext;
@@ -31,10 +33,11 @@ exports.TIMOUT_AFTER_DETECTED_RATE_LIMIT = 5000;
 exports.DEFAULT_REQUESTS_PER_SECOND = 10;
 exports.MAX_REQUESTS_PER_SECOND = 20;
 exports.RATE_LIMIT_MULTIPLICATOR = 0.8;
-exports.RATE_LIMIT_CODES = ["308", "310", "316"];
+// we also use some codes that indicate some server overloads
+exports.RATE_LIMIT_CODES = ["306", "308", "310", "311", "314", "316"];
 exports.TIMING_HEADER_ERROR_CODE_SELECTOR = /errorcode;desc=([^,]*),/;
 exports.TIMING_HEADER_KEY = "server-timing";
-exports.PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry}";
+exports.PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta_formatted}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry} | batchDur: {batchDuration}";
 exports.VARIATION_SEPARATOR = ",";
 exports.DEFAULT_VARIATION = ["default"];
 exports.DEFAULT_VARIATIONS = ["mobile", "tablet", "tv"];

package/dist/services/prewarm/pre-warm-service.d.ts

@@ -6,15 +6,16 @@ export declare class PreWarmService {
     private cli;
     private urls;
     private variations;
+    private verbose;
     private requestsSend;
-    constructor(assetClient: AssetApiClient, cli: CliServiceInterface, urls: string[], variations: string[]);
-    handleRequestCallback(assetClient: AssetApiClient, entry: AssetItemInterface): Promise<void>;
+    constructor(assetClient: AssetApiClient, cli: CliServiceInterface, urls: string[], variations: string[], verbose: boolean);
+    handleAsset(assetClient: AssetApiClient, entry: AssetItemInterface): Promise<void>;
     run(): Promise<void>;
-    private fetchBatch;
+    private fetchAssetList;
+    private handleProgress;
     private fetchUrls;
     private generateAssetRequestList;
     private printSummary;
-    private sleep;
     private startProgressBar;
     private stopProgressBar;
     private updateProgressBar;

package/dist/services/prewarm/pre-warm-service.js

@@ -10,15 +10,17 @@ class PreWarmService {
     cli;
     urls;
     variations;
+    verbose;
     requestsSend = 0;
-    constructor(assetClient, cli, urls, variations) {
+    constructor(assetClient, cli, urls, variations, verbose) {
         this.assetClient = assetClient;
         this.cli = cli;
         this.urls = urls;
         this.variations = variations;
+        this.verbose = verbose;
     }
-    async handleRequestCallback(assetClient, entry) {
-        const fetchUrlResponse = await (0, safe_1.safe)(assetClient.fetchURL(entry.url, entry.variation));
+    async handleAsset(assetClient, entry) {
+        const fetchUrlResponse = await (0, safe_1.safe)(assetClient.fetchURL(entry.url, entry.variation, this.verbose));
         // a generic error occurred
         if (fetchUrlResponse.success === false) {
             entry.failed();
@@ -67,38 +69,55 @@ class PreWarmService {
         await this.fetchUrls(retryList);
         await this.printSummary(retryList);
     }
-    async fetchBatch(assetList, offset) {
-        // get batch in size of current rateLimit
-        const batch = assetList.getBatch(offset, this.assetClient.getRateLimit());
-        if (batch.length <= 0) {
-            return;
-        }
-        await Promise.all([
-            ...batch.map((entry) => {
-                return entry.callback(this.assetClient, entry);
-            }),
-            // one batch must take at least on second to run
-            this.sleep(_1.MIN_TIME_TO_FETCH_BATCH),
-        ]);
-        // update offset for next batch
-        offset += batch.length;
-        this.updateProgressBar(offset, assetList);
-        await this.updateRateLimit(batch);
+    async fetchAssetList(assetList) {
+        const iterator = assetList[Symbol.iterator]();
+        // run with 20 concurrent requests
+        let batch = [];
+        let completed = 0;
+        let lastTime = Date.now();
+        const workers = Array.from({ length: 10 }).map(async () => {
+            for (;;) {
+                const { done, value } = iterator.next();
+                if (done) {
+                    break;
+                }
+                await this.handleAsset(this.assetClient, value);
+                if (this.verbose && value.status !== _1.ItemStatus.DONE) {
+                    this.cli.write(`${value.toString()}\n`);
+                }
+                completed++;
+                batch.push(value);
+                if (completed % 20 === 0) {
+                    if (this.verbose)
+                        this.cli.write("\n");
+                    this.handleProgress(completed, assetList, lastTime, batch);
+                    if (this.verbose)
+                        this.cli.write("\n");
+                    lastTime = Date.now();
+                    batch = [];
+                }
+            }
+        });
+        await Promise.all(workers);
+        this.handleProgress(completed, assetList, lastTime, batch);
+    }
+    handleProgress(completed, assetList, lastTime, batch) {
+        this.updateProgressBar(completed, assetList, Date.now() - lastTime);
+        this.updateRateLimit(batch);
         this.showMinimalStatusUpdate(batch, assetList);
-        // fetch next batch
-        await this.fetchBatch(assetList, offset);
     }
     async fetchUrls(assetList) {
         this.cli.write(`start prewarm ${assetList.getLength()} assets`);
         this.startProgressBar(assetList);
-        await this.fetchBatch(assetList, 0);
+        await this.fetchAssetList(assetList);
         this.stopProgressBar();
     }
     generateAssetRequestList(urls) {
-        const assetList = new asset_list_1.AssetList(this.variations);
+        const assetList = new asset_list_1.AssetList();
         for (const variation of this.variations) {
-            for (const url of urls)
-                assetList.addItem(new asset_item_1.AssetItem(url, variation, this.handleRequestCallback));
+            for (const url of urls) {
+                assetList.addItem(new asset_item_1.AssetItem(url, variation));
+            }
         }
         return assetList;
     }
@@ -125,33 +144,29 @@ class PreWarmService {
             });
         }
     }
-    async sleep(ms) {
-        return new Promise((resolve) => {
-            setTimeout(resolve, ms);
-        });
-    }
     startProgressBar(assetList) {
         this.cli.startProgress(assetList.getLength(), 0, {
             errors: 0,
             rate: this.assetClient.getRateLimit(),
             retry: 0,
-        }, { format: _1.PROGRESS_BAR_FORMAT });
+            batchDuration: "-",
+        }, { format: _1.PROGRESS_BAR_FORMAT, linewrap: !!this.verbose });
     }
     stopProgressBar() {
         this.cli.stopProgress();
     }
-    updateProgressBar(offset, assetList) {
+    updateProgressBar(offset, assetList, batchDurationInMs) {
         this.cli.updateProgress(offset, {
             errors: assetList.getErrorCount(),
-            rate: this.assetClient.getRateLimit(),
+            rate: `${this.assetClient.getRateLimit()} req/s`,
             retry: assetList.getRetryCount(),
+            batchDuration: `${Math.round(batchDurationInMs / 100) / 10}s`,
         });
     }
-    async updateRateLimit(batch) {
+    updateRateLimit(batch) {
         const retryCount = batch.filter((entry) => entry.status === _1.ItemStatus.RETRY).length;
         // found rateLimiterResponse wait and reduce rateLimit
         if (retryCount > 0) {
-            await this.sleep(_1.TIMOUT_AFTER_DETECTED_RATE_LIMIT);
             this.assetClient.reduceMaxRequestsPerSecond();
             return;
         }

package/oclif.manifest.json

@@ -552,7 +552,7 @@
         "variation": {
           "char": "v",
           "name": "variation",
-          "default": "DEFAULT",
+          "default": "default",
           "hasDynamicHelp": false,
           "multiple": false,
           "type": "option"
@@ -582,6 +582,12 @@
           "allowNo": false,
           "type": "boolean"
         },
+        "verbose": {
+          "description": "Log information about failed requests",
+          "name": "verbose",
+          "allowNo": false,
+          "type": "boolean"
+        },
         "delay": {
           "char": "d",
           "deprecated": true,
@@ -732,5 +738,5 @@
       ]
     }
   },
-  "version": "3.18.0"
+  "version": "3.20.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@speedkit/cli",
   "description": "Speed Kit CLI",
-  "version": "3.18.0",
+  "version": "3.20.0",
   "author": {
     "name": "Baqend.com",
     "email": "info@baqend.com"