@speedkit/cli 2.22.0 → 2.23.0

package/dist/services/prewarm/pre-warm-model.d.ts

@@ -0,0 +1,38 @@
+import { AssetApiClient } from "./assets/asset-api-client";
+export declare class PreWarmContext {
+    readonly app: string;
+    readonly path: string;
+    readonly variation?: string;
+    readonly variationPath?: string;
+    readonly keepParameterSorting: boolean;
+    readonly quiet: boolean;
+    constructor(app: string, path: string, variation?: string, variationPath?: string, keepParameterSorting?: boolean, quiet?: boolean);
+}
+export interface AssetItemInterface {
+    callback: (client: AssetApiClient, entry: AssetItemInterface) => Promise<void>;
+    done(): void;
+    error?: Error;
+    failed(): void;
+    retry(): void;
+    setError(error: Error): void;
+    status: ItemStatus;
+    url: string;
+    variation: string;
+}
+export declare enum ItemStatus {
+    DONE = "done",
+    ERROR = "error",
+    NEW = "new",
+    RETRY = "retry"
+}
+export declare const MIN_TIME_TO_FETCH_BATCH = 1000;
+export declare const FETCH_TIMEOUT = 10000;
+export declare const TIMOUT_AFTER_DETECTED_RATE_LIMIT = 5000;
+export declare const DEFAULT_MAX_REQUESTS_PER_SECOND = 15;
+export declare const RATE_LIMIT_MULTIPLICATOR = 0.8;
+export declare const RATE_LIMIT_CODES: string[];
+export declare const TIMING_HEADER_ERROR_CODE_SELECTOR: RegExp;
+export declare const TIMING_HEADER_KEY = "server-timing";
+export declare const PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry}";
+export declare const VARIATION_SEPARATOR = ",";
+export declare const DEFAULT_VARIATION: string[];

package/dist/services/prewarm/pre-warm-model.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_VARIATION = exports.VARIATION_SEPARATOR = exports.PROGRESS_BAR_FORMAT = exports.TIMING_HEADER_KEY = exports.TIMING_HEADER_ERROR_CODE_SELECTOR = exports.RATE_LIMIT_CODES = exports.RATE_LIMIT_MULTIPLICATOR = exports.DEFAULT_MAX_REQUESTS_PER_SECOND = exports.TIMOUT_AFTER_DETECTED_RATE_LIMIT = exports.FETCH_TIMEOUT = exports.MIN_TIME_TO_FETCH_BATCH = exports.ItemStatus = exports.PreWarmContext = void 0;
+class PreWarmContext {
+    app;
+    path;
+    variation;
+    variationPath;
+    keepParameterSorting;
+    quiet;
+    constructor(app, path, variation, variationPath, keepParameterSorting = true, quiet = false) {
+        this.app = app;
+        this.path = path;
+        this.variation = variation;
+        this.variationPath = variationPath;
+        this.keepParameterSorting = keepParameterSorting;
+        this.quiet = quiet;
+    }
+}
+exports.PreWarmContext = PreWarmContext;
+var ItemStatus;
+(function (ItemStatus) {
+    ItemStatus["DONE"] = "done";
+    ItemStatus["ERROR"] = "error";
+    ItemStatus["NEW"] = "new";
+    ItemStatus["RETRY"] = "retry";
+})(ItemStatus || (exports.ItemStatus = ItemStatus = {}));
+exports.MIN_TIME_TO_FETCH_BATCH = 1000;
+exports.FETCH_TIMEOUT = 10_000;
+exports.TIMOUT_AFTER_DETECTED_RATE_LIMIT = 5000;
+exports.DEFAULT_MAX_REQUESTS_PER_SECOND = 15;
+exports.RATE_LIMIT_MULTIPLICATOR = 0.8;
+exports.RATE_LIMIT_CODES = ["308", "310", "316"];
+exports.TIMING_HEADER_ERROR_CODE_SELECTOR = /errorcode;desc=([^,]*),/;
+exports.TIMING_HEADER_KEY = "server-timing";
+exports.PROGRESS_BAR_FORMAT = "[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | error:{errors} | rate: {rate} | retry: {retry}";
+exports.VARIATION_SEPARATOR = ",";
+exports.DEFAULT_VARIATION = ["DEFAULT"];

package/dist/services/prewarm/pre-warm-service.d.ts

@@ -0,0 +1,21 @@
+import { CliServiceInterface } from "../cli";
+import { AssetItemInterface } from "./";
+import { AssetApiClient } from "./assets/asset-api-client";
+export declare class PreWarmService {
+    private assetClient;
+    private cli;
+    private urls;
+    private variations;
+    constructor(assetClient: AssetApiClient, cli: CliServiceInterface, urls: string[], variations: string[]);
+    handleRequestCallback(assetClient: AssetApiClient, entry: AssetItemInterface): Promise<void>;
+    run(): Promise<void>;
+    private fetchBatch;
+    private fetchUrls;
+    private generateAssetRequestList;
+    private printSummary;
+    private sleep;
+    private startProgressBar;
+    private stopProgressBar;
+    private updateProgressBar;
+    private updateRateLimit;
+}

package/dist/services/prewarm/pre-warm-service.js

@@ -0,0 +1,154 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PreWarmService = void 0;
+const safe_1 = require("../../helpers/safe");
+const _1 = require("./");
+const asset_item_1 = require("./assets/asset-item");
+const asset_list_1 = require("./assets/asset-list");
+class PreWarmService {
+    assetClient;
+    cli;
+    urls;
+    variations;
+    constructor(assetClient, cli, urls, variations) {
+        this.assetClient = assetClient;
+        this.cli = cli;
+        this.urls = urls;
+        this.variations = variations;
+    }
+    async handleRequestCallback(assetClient, entry) {
+        const fetchUrlResponse = await (0, safe_1.safe)(assetClient.fetchURL(entry.url, entry.variation));
+        // a generic error occurred
+        if (fetchUrlResponse.success === false) {
+            entry.failed();
+            entry.setError(fetchUrlResponse.errorObj);
+            return;
+        }
+        const response = fetchUrlResponse.data;
+        // success
+        if (response.status === 200) {
+            entry.done();
+            return;
+        }
+        // got undefined errorCode
+        if (!response.headers.has(_1.TIMING_HEADER_KEY) ||
+            !response.headers.get(_1.TIMING_HEADER_KEY)?.includes("error")) {
+            entry.failed();
+            entry.setError(new Error(`status: ${response.status}`));
+        }
+        // get errorCode from server-timing-header
+        const serverTiming = response.headers.get(_1.TIMING_HEADER_KEY);
+        const [, serverTimingErrorCode] = serverTiming.match(_1.TIMING_HEADER_ERROR_CODE_SELECTOR) || ["", "-"];
+        // rateLimiter detected, retry
+        if (_1.RATE_LIMIT_CODES.some((code) => serverTimingErrorCode.includes(code))) {
+            entry.retry();
+            return;
+        }
+        // other errorCode detected
+        entry.failed();
+        entry.setError(new Error(`status: ${response.status}, sk-error: ${serverTimingErrorCode}`));
+    }
+    async run() {
+        const assetList = this.generateAssetRequestList(this.urls);
+        await this.fetchUrls(assetList);
+        await this.printSummary(assetList);
+        if (assetList.getRetryCount() === 0) {
+            return;
+        }
+        this.cli.write("Retry rateLimited requests.");
+        const urlsToRetry = assetList
+            .getByStatus(_1.ItemStatus.RETRY)
+            .map((entry) => entry.url);
+        const retryList = this.generateAssetRequestList(urlsToRetry);
+        await this.fetchUrls(retryList);
+        await this.printSummary(retryList);
+    }
+    async fetchBatch(assetList, offset) {
+        // get batch in size of current rateLimit
+        const batch = assetList.getBatch(offset, this.assetClient.getRateLimit());
+        if (batch.length <= 0) {
+            return;
+        }
+        await Promise.all([
+            ...batch.map((entry) => {
+                return entry.callback(this.assetClient, entry);
+            }),
+            // one batch must take at least on second to run
+            this.sleep(_1.MIN_TIME_TO_FETCH_BATCH),
+        ]);
+        // update offset for next batch
+        offset += batch.length;
+        this.updateProgressBar(offset, assetList);
+        await this.updateRateLimit(batch);
+        // fetch next batch
+        await this.fetchBatch(assetList, offset);
+    }
+    async fetchUrls(assetList) {
+        this.cli.write(`start prewarm ${assetList.getLength()} assets`);
+        this.startProgressBar(assetList);
+        await this.fetchBatch(assetList, 0);
+        this.stopProgressBar();
+    }
+    generateAssetRequestList(urls) {
+        const assetList = new asset_list_1.AssetList(this.variations);
+        for (const variation of this.variations) {
+            for (const url of urls)
+                assetList.addItem(new asset_item_1.AssetItem(url, variation, this.handleRequestCallback));
+        }
+        return assetList;
+    }
+    async printSummary(assetList) {
+        const successCount = assetList.getSuccessCount();
+        const errorCount = assetList.getErrorCount();
+        const rateLimitCount = assetList.getRetryCount();
+        this.cli.table([{ error: errorCount, retry: rateLimitCount, succeed: successCount }], {
+            error: {},
+            retry: {},
+            succeed: {},
+        });
+        if (await this.cli.confirm("show errors?", true)) {
+            const elements = assetList.getByStatus(_1.ItemStatus.ERROR);
+            this.cli.table(elements.map((element) => {
+                return { ...element };
+            }), {
+                error: {
+                    get(row) {
+                        return row.error.message;
+                    },
+                },
+                url: {},
+            });
+        }
+    }
+    async sleep(ms) {
+        return new Promise((resolve) => {
+            setTimeout(resolve, ms);
+        });
+    }
+    startProgressBar(assetList) {
+        this.cli.startProgress(assetList.getLength(), 0, {
+            errors: 0,
+            rate: this.assetClient.getRateLimit(),
+            retry: 0,
+        }, { format: _1.PROGRESS_BAR_FORMAT });
+    }
+    stopProgressBar() {
+        this.cli.stopProgress();
+    }
+    updateProgressBar(offset, assetList) {
+        this.cli.updateProgress(offset, {
+            errors: assetList.getErrorCount(),
+            rate: this.assetClient.getRateLimit(),
+            retry: assetList.getRetryCount(),
+        });
+    }
+    async updateRateLimit(batch) {
+        const retryCount = batch.filter((entry) => entry.status === _1.ItemStatus.RETRY).length;
+        // found rateLimiterResponse wait and reduce rateLimit
+        if (retryCount > 0) {
+            await this.sleep(_1.TIMOUT_AFTER_DETECTED_RATE_LIMIT);
+            this.assetClient.reduceMaxRequestsPerSecond();
+        }
+    }
+}
+exports.PreWarmService = PreWarmService;

package/oclif.manifest.json

@@ -476,54 +476,6 @@
         "onboarding.js"
       ]
     },
-    "pop-prewarm": {
-      "aliases": [],
-      "args": {
-        "app": {
-          "description": "The customers appName",
-          "hidden": false,
-          "name": "app",
-          "required": true
-        }
-      },
-      "description": "Pre-warm Fastly PoPs",
-      "examples": [
-        "$ sk pop-prewarm <app> -c <contentTypes> -i <ips>",
-        "$ sk pop-prewarm decathlon -c image,style -i 151.101.120.194"
-      ],
-      "flags": {
-        "ips": {
-          "char": "i",
-          "name": "ips",
-          "default": "151.101.120.194",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "contentTypes": {
-          "char": "c",
-          "name": "contentTypes",
-          "default": "image",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        }
-      },
-      "hasDynamicHelp": false,
-      "hiddenAliases": [],
-      "id": "pop-prewarm",
-      "pluginAlias": "@speedkit/cli",
-      "pluginName": "@speedkit/cli",
-      "pluginType": "core",
-      "strict": true,
-      "enableJsonFlag": false,
-      "isESM": false,
-      "relativePath": [
-        "dist",
-        "commands",
-        "pop-prewarm.js"
-      ]
-    },
     "prewarm": {
       "aliases": [],
       "args": {
@@ -533,7 +485,7 @@
           "required": true
         },
         "path": {
-          "description": "path to prewarmFile",
+          "description": "path to prewarmFile.csv",
           "name": "path",
           "required": true
         }
@@ -541,12 +493,13 @@
       "description": "Pre-warm Fastly",
       "examples": [
         "for a single variation:",
-        "$ sk prewarm <app> <path> -v <variation> -d <delay> -k",
-        "$ sk prewarm decathlon C:\\Users\\testUser\\file.csv -v MOBILE -d 1200 -k",
+        "$ sk prewarm <app> <path> -v <variation> -k",
+        "$ sk prewarm decathlon C:\\Users\\testUser\\file.csv -v MOBILE -k",
         "",
         "for a list of variations:",
-        "$ sk prewarm <app> <path> -p <variationPath> -d <delay> -k",
-        "$ sk prewarm decathlon C:\\Users\\testUser\\file.csv -p C:\\Users\\testUser\\variations.csv -d 1200 -k"
+        "$ sk prewarm <app> <path> -p <variationPath> -k",
+        "$ sk prewarm decathlon C:\\Users\\testUser\\file.csv -v MOBILE,DESKTOP -k",
+        "$ sk prewarm decathlon C:\\Users\\testUser\\file.csv -p C:\\Users\\testUser\\variations.csv -k"
       ],
       "flags": {
         "variation": {
@@ -559,6 +512,7 @@
         },
         "variationPath": {
           "char": "p",
+          "description": "path to variations.csv",
           "exclusive": [
             "variation"
           ],
@@ -567,21 +521,28 @@
           "multiple": false,
           "type": "option"
         },
-        "delay": {
-          "char": "d",
-          "description": "Request delay in ms",
-          "name": "delay",
-          "default": 0,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
         "keepParameterSorting": {
           "char": "k",
           "description": "Prevents alphabetical sorting of the URL parameters",
           "name": "keepParameterSorting",
           "allowNo": false,
           "type": "boolean"
+        },
+        "quiet": {
+          "char": "q",
+          "description": "less output non interactive",
+          "name": "quiet",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "delay": {
+          "char": "d",
+          "deprecated": true,
+          "description": "Request delay in ms (deprecated)",
+          "name": "delay",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
         }
       },
       "hasDynamicHelp": false,
@@ -699,5 +660,5 @@
       ]
     }
   },
-  "version": "2.22.0"
+  "version": "2.23.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@speedkit/cli",
   "description": "Speed Kit CLI",
-  "version": "2.22.0",
+  "version": "2.23.0",
   "author": {
     "name": "Baqend.com",
     "email": "info@baqend.com"

package/dist/commands/pop-prewarm.d.ts

@@ -1,17 +0,0 @@
-import { Command } from "@oclif/core";
-import { EntityManager } from "baqend";
-export default class PopPrewarm extends Command {
-    static description: string;
-    static args: {
-        app: import("@oclif/core/lib/interfaces").Arg<string, Record<string, unknown>>;
-    };
-    static flags: {
-        ips: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
-        contentTypes: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
-    };
-    static examples: string[];
-    run(): Promise<void>;
-    getCount(database: EntityManager, contentTypes: string[]): Promise<any>;
-    queryDb(database: EntityManager, contentTypes: string[], lastId?: string): Promise<any>;
-    fetchAssets(database: EntityManager, ips: string[], contentTypes: string[], app: string, progress: any, lastId?: string): Promise<any>;
-}

package/dist/commands/pop-prewarm.js

@@ -1,90 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-const core_1 = require("@oclif/core");
-const cli_1 = require("baqend/cli");
-const core_2 = require("@oclif/core");
-const prewarm_service_1 = tslib_1.__importDefault(require("../services/prewarm-service"));
-const cli_parameters_1 = require("../models/cli-parameters");
-const logging_helper_1 = require("../helpers/logging-helper");
-class PopPrewarm extends core_1.Command {
-    static description = "Pre-warm Fastly PoPs";
-    static args = {
-        [cli_parameters_1.CLIParameters.AppName]: core_1.Args.string({
-            name: cli_parameters_1.CLIParameters.AppName, // name of arg to show in help and reference with args[name]
-            required: true, // make the arg required with `required: true`
-            description: "The customers appName", // help description
-            hidden: false, // hide this arg from help
-        }),
-    };
-    static flags = {
-        [cli_parameters_1.CLIParameters.Ips]: core_1.Flags.string({
-            default: "151.101.120.194",
-            char: cli_parameters_1.CLIParametersChar.Ips,
-        }),
-        [cli_parameters_1.CLIParameters.ContentTypes]: core_1.Flags.string({
-            default: "image",
-            char: cli_parameters_1.CLIParametersChar.ContentTypes,
-        }),
-    };
-    static examples = [
-        `$ sk pop-prewarm <${cli_parameters_1.CLIParameters.AppName}> -${cli_parameters_1.CLIParametersChar.ContentTypes} <${cli_parameters_1.CLIParameters.ContentTypes}> -${cli_parameters_1.CLIParametersChar.Ips} <${cli_parameters_1.CLIParameters.Ips}>`,
-        `$ sk pop-prewarm decathlon -${cli_parameters_1.CLIParametersChar.ContentTypes} ${cli_parameters_1.CLIParametersExample.ContentTypes} -${cli_parameters_1.CLIParametersChar.Ips} ${cli_parameters_1.CLIParametersExample.Ips}`,
-    ];
-    async run() {
-        const { args: { app }, flags: { ips, contentTypes }, } = await this.parse(PopPrewarm);
-        const database = await cli_1.account.login({ app });
-        const contentTypeArray = contentTypes.split(",");
-        const ipsArray = ips.split(",");
-        const count = await this.getCount(database, contentTypeArray);
-        const progress = core_2.ux.progress();
-        progress.start(count * ips.length, 0);
-        await this.fetchAssets(database, ipsArray, contentTypeArray, app, progress);
-        progress.stop();
-    }
-    async getCount(database, contentTypes) {
-        return database["speedKit.Asset"]
-            .find()
-            .in("contentType", contentTypes)
-            .count();
-    }
-    async queryDb(database, contentTypes, lastId = "") {
-        let qb = database["speedKit.Asset"].find().in("contentType", contentTypes);
-        if (lastId) {
-            qb = qb.gt("id", lastId);
-        }
-        return qb.limit(100).sort({ id: 1 }).resultList();
-    }
-    /* eslint-disable @typescript-eslint/no-explicit-any */
-    async fetchAssets(database, ips, contentTypes, app, progress, lastId = "") {
-        const resultList = await this.queryDb(database, contentTypes, lastId);
-        if (resultList.length === 0) {
-            return;
-        }
-        const assetChunks = prewarm_service_1.default.chunk(resultList, 50);
-        let fetches = [];
-        for (const assets of assetChunks) {
-            for (const ip of ips) {
-                for (const asset of assets) {
-                    let { url } = asset;
-                    const { origin, variation, extendedURL } = asset;
-                    url += extendedURL || "";
-                    fetches.push(prewarm_service_1.default.fetchURLByIp(ip, url, origin, variation, app, database.token));
-                }
-                try {
-                    // eslint-disable-next-line no-await-in-loop
-                    await Promise.all(fetches);
-                }
-                catch (error) {
-                    (0, logging_helper_1.logWarning)(error);
-                }
-                progress.increment(fetches.length);
-                fetches = [];
-            }
-        }
-        database.clear();
-        this.log(resultList.at(-1).url);
-        return this.fetchAssets(database, ips, contentTypes, app, progress, resultList.at(-1).id);
-    }
-}
-exports.default = PopPrewarm;

package/dist/services/cli/hooks.d.ts

@@ -1,11 +0,0 @@
-import { CliService } from "./cli-service";
-export default abstract class Hooks {
-    readonly preHook?: string;
-    readonly postHook?: string;
-    protected cli: CliService;
-    protected constructor(cli: CliService, preHook?: string, postHook?: string);
-    run(): Promise<any>;
-    runPreHook(): Promise<void>;
-    runPostHook(): Promise<void>;
-    abstract _run(): Promise<any>;
-}

package/dist/services/cli/hooks.js

@@ -1,33 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const node_child_process_1 = require("node:child_process");
-class Hooks {
-    preHook;
-    postHook;
-    cli;
-    constructor(cli, preHook, postHook) {
-        this.cli = cli;
-        this.preHook = preHook;
-        this.postHook = postHook;
-    }
-    async run() {
-        await this.runPreHook();
-        await this._run();
-        await this.runPostHook();
-    }
-    async runPreHook() {
-        if (!this.preHook) {
-            return;
-        }
-        this.cli.writeSuccess(this.preHook);
-        this.cli.write((0, node_child_process_1.execSync)(this.preHook) + "");
-    }
-    async runPostHook() {
-        if (!this.postHook) {
-            return;
-        }
-        this.cli.writeSuccess(this.postHook);
-        this.cli.write((0, node_child_process_1.execSync)(this.postHook) + "");
-    }
-}
-exports.default = Hooks;