@speedkit/cli 3.28.0 → 3.29.0

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+# [3.29.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.28.0...v3.29.0) (2026-02-24)
+
+
+### Features
+
+* **content-encoding:** complete custom text encoding support in the CLI ([afeab5a](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/afeab5a7add0e967a7ddafa8079e769ad668fac8))
+* **prewarm:** improve prewarming speed by using more concurrent workers ([40b57ce](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/40b57ce5aaca260a6aff6549e764a3c240afc2e3))
+
 # [3.28.0](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.27.0...v3.28.0) (2026-02-20)
 
 

package/README.md

@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
 $ sk COMMAND
 running command...
 $ sk (--version)
-@speedkit/cli/3.28.0 linux-x64 node-v20.20.0
+@speedkit/cli/3.29.0 linux-x64 node-v20.20.0
 $ sk --help [COMMAND]
 USAGE
   $ sk COMMAND

package/dist/services/document-handler-runtime/document-handler-server.d.ts

@@ -15,7 +15,7 @@ export declare class DocumentHandlerServer {
     private documentHandlerConfigCode;
     private database;
     constructor(customerConfig: CustomerConfig, files: FileListInterface, bundler: BundleService, cli: CliService, nodeModulesPath: string, verboseLevel?: boolean, entityManager?: EntityManager);
-    transform(html: string, variation: string, url: string, headers: {
+    transform(contentType: string, html: string, variation: string, url: string, headers: {
         [name: string]: string;
     }): Promise<DocumentHandlerResponse>;
     private getTransformFunctionWrapper;

package/dist/services/document-handler-runtime/document-handler-server.js

@@ -36,11 +36,13 @@ class DocumentHandlerServer {
         this.verboseLevel = verboseLevel;
         this.entityManager = entityManager;
     }
-    async transform(html, variation, url, headers) {
+    async transform(contentType, html, variation, url, headers) {
         if (!this.documentHandlerCode) {
             await this.buildDocumentHandler();
         }
-        const skInternalRequest = new request_1.Request(url, html, variation, headers);
+        const skInternalRequest = new request_1.Request(url, html, variation, headers, {
+            "content-type": contentType,
+        });
         const skInternalResponse = new document_handler_response_1.DocumentHandlerResponse();
         const vmContext = {
             db: await this.createDataBaseMock(),

package/dist/services/document-handler-runtime/server/document-handler-response.js

@@ -6,13 +6,13 @@ class DocumentHandlerResponse {
     body = "";
     resolve;
     header(key, value) {
-        this.set(key, value);
+        this.set(key.toLowerCase(), value);
     }
     setHeader(key, value) {
-        this.set(key, value);
+        this.set(key.toLowerCase(), value);
     }
     set(key, value) {
-        this.headers[key] = value;
+        this.headers[key.toLowerCase()] = value;
     }
     send(html) {
         this.body = html;

package/dist/services/document-handler-runtime/server/request.d.ts

@@ -4,10 +4,14 @@ type query = {
     headers: string;
 };
 export declare class Request {
-    private headers;
     query: query;
     body: string;
-    constructor(url: string, body: string, variation: string, headers: {
+    headers: {
+        [name: string]: string;
+    };
+    constructor(url: string, body: string, variation: string, originalHeaders: {
+        [name: string]: string;
+    }, requestHeaders?: {
         [name: string]: string;
     });
     get(key: string): string | null;

package/dist/services/document-handler-runtime/server/request.js

@@ -2,16 +2,19 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Request = void 0;
 class Request {
-    headers;
     query;
     body;
-    constructor(url, body, variation, headers) {
-        this.headers = headers;
-        this.query = { variation, headers: JSON.stringify(headers), url };
+    headers;
+    constructor(url, body, variation, originalHeaders, requestHeaders = {}) {
+        this.query = { variation, headers: JSON.stringify(originalHeaders), url };
         this.body = body;
+        this.headers = {};
+        for (const key of Object.keys(requestHeaders)) {
+            this.headers[key.toLowerCase()] = requestHeaders[key];
+        }
     }
     get(key) {
-        return this.headers?.[key] || null;
+        return this.headers[key.toLowerCase()] || null;
     }
     header(key) {
         return this.get(key);

package/dist/services/onboarding/browser/baqend-response.js

@@ -52,8 +52,8 @@ class BaqendResponse {
                 value: String(this.contentLength),
             });
         }
-        if (!headers?.find((o) => o.name === "Content-Type")) {
-            headers.push({ name: "Content-Type", value: String(this.contentType) });
+        if (!headers?.find((o) => o.name.toLowerCase() === "content-type")) {
+            headers.push({ name: "content-type", value: String(this.contentType) });
         }
         headers.push({ name: "via", value: "sk-onboarding" }, { name: "Service-Worker-Allowed", value: "/" });
         const exposeHeaders = headers

package/dist/services/onboarding/dashboard/diff-against-current-page.js

@@ -26,13 +26,15 @@ class DiffAgainstCurrentPage {
         this.cli.startAction(`transform documents`);
         if (transform) {
             // todo check if the currentHTML is already transformed
-            const currentDocumentHandlerResponse = await this.documentHandlerRuntime.transform(currentHtml, variant, currentUrl, {});
+            // TODO: we may need to forward the correct content type from the origin response
+            const contentType = "text/html; charset=utf-8";
+            const currentDocumentHandlerResponse = await this.documentHandlerRuntime.transform(contentType, currentHtml, variant, currentUrl, {});
             const currentResponse = currentDocumentHandlerResponse.body;
             if (Buffer.isBuffer(currentResponse)) {
                 throw new TypeError("cant diff body as buffer");
             }
             currentHtml = currentResponse;
-            const originDocumentHandlerResponse = await this.documentHandlerRuntime.transform(originHtml, variant, currentUrl, {});
+            const originDocumentHandlerResponse = await this.documentHandlerRuntime.transform(contentType, originHtml, variant, currentUrl, {});
             const originResponse = originDocumentHandlerResponse.body;
             if (Buffer.isBuffer(originResponse)) {
                 throw new TypeError("cant diff body as buffer");

package/dist/services/onboarding/dashboard/request-diff-service.js

@@ -40,13 +40,17 @@ class RequestDiffService {
         const htmlWithoutParameter = await this.fetchHtml(urlWithoutParameter);
         const replaceRegEx = /<meta[^>]*name="baqend:asset_url"[^>]*>/;
         // <meta name="baqend:asset_url" content="https://www.kik.de/c/damen/damen-bekleidung-shirts-tops?page=7">
-        const transformedHtmlWithParameterResponse = await this.documentHandlerRuntime.transform(htmlWithParameter, "", urlWithParameter, {});
+        const transformedHtmlWithParameterResponse = await this.documentHandlerRuntime.transform(
+        // TODO: ma use original response content type
+        "text/html; charset=utf-8", htmlWithParameter, "", urlWithParameter, {});
         if (Buffer.isBuffer(transformedHtmlWithParameterResponse.body)) {
             throw new TypeError("Can't diff body if it's a buffer");
         }
         let transformedHtmlWithParameter = transformedHtmlWithParameterResponse.body;
         transformedHtmlWithParameter = transformedHtmlWithParameter.replace(replaceRegEx, "");
-        const transformedHtmlWithoutParameterResponse = await this.documentHandlerRuntime.transform(htmlWithoutParameter, "", urlWithParameter, {});
+        const transformedHtmlWithoutParameterResponse = await this.documentHandlerRuntime.transform(
+        // TODO: ma use original response content type
+        "text/html; charset=utf-8", htmlWithoutParameter, "", urlWithParameter, {});
         if (Buffer.isBuffer(transformedHtmlWithoutParameterResponse.body)) {
             throw new TypeError("Can't diff body if it's a buffer");
         }

package/dist/services/onboarding/virtual-orestes-app/index.js

@@ -6,9 +6,11 @@ const baqend_response_1 = require("../browser/baqend-response");
 const safe_1 = require("../../../helpers/safe");
 const onboarding_model_1 = require("../onboarding-model");
 const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
-const encoding_japanese_1 = tslib_1.__importDefault(require("encoding-japanese"));
-const iconv_lite_2 = tslib_1.__importDefault(require("iconv-lite"));
 const unsupported_encoding_error_1 = require("../error/unsupported-encoding-error");
+// for none encodable chars use a space, so the broken char is invisible.
+// We are using the same trick in our node.js server
+iconv_lite_1.default.defaultCharSingleByte = " ";
+iconv_lite_1.default.defaultCharUnicode = " ";
 class VirtualOrestesApp {
     customerConfig;
     crawler;
@@ -35,7 +37,7 @@ class VirtualOrestesApp {
         const response = await this.crawler.fetchRemote(originUrl, variation);
         // todo check if we can handle more then htmlResponses
         // only handle html responses
-        if (!response?.headers?.get("Content-Type")?.includes("html")) {
+        if (!response?.headers?.get("content-type")?.includes("html")) {
             return;
         }
         const hasNoStoreFlag = response?.headers
@@ -49,18 +51,14 @@ class VirtualOrestesApp {
         if (response.status > 300 && response.status < 400) {
             return this.returnRedirect(customHeaders, response, event);
         }
-        const { responseContent, responseEncoding } = await this.getResponseText(response);
+        const { responseContent, textEncoding } = await this.getResponseText(response);
         if (!this.isHtmlContentHeader(response?.headers) ||
             !this.isValidHtmlContent(responseContent)) {
-            const customResponse = new baqend_response_1.BaqendResponse(responseContent, `text/html; charset=${responseEncoding}`, customHeaders);
+            const customResponse = new baqend_response_1.BaqendResponse(responseContent, `text/html; charset=${textEncoding}`, customHeaders);
             this.cache.addEntry(event.request.url, customResponse);
             return customResponse;
         }
-        const convertedResponseHeaders = this.convertResponseHeadersToOrestesFormat(response);
-        const customResponse = await (0, safe_1.safe)(this.documentHandler.transform(responseContent, variation, originUrl, {
-            ...convertedResponseHeaders,
-            "content-type": `text/html;charset=${responseEncoding}`,
-        }));
+        const customResponse = await (0, safe_1.safe)(this.documentHandler.transform(`text/html;charset=${textEncoding}`, responseContent, variation, originUrl, this.convertResponseHeadersToOrestesFormat(response)));
         if (customResponse.success === false) {
             return this.returnErrorResponse(customResponse, customHeaders);
         }
@@ -81,9 +79,11 @@ class VirtualOrestesApp {
         });
         customHeaders.push({
             name: "x-sk-detected-encoding",
-            value: responseEncoding,
+            value: textEncoding,
         });
-        const baqendResponse = new baqend_response_1.BaqendResponse(skResponse.body, response?.headers.get("Content-Type"), [...customHeaders, ...originHeaders]);
+        const baqendResponse = new baqend_response_1.BaqendResponse(Buffer.isBuffer(skResponse.body)
+            ? skResponse.body
+            : iconv_lite_1.default.encode(skResponse.body, textEncoding), skResponse.headers["content-type"] ?? `text/html;charset=${textEncoding}`, [...customHeaders, ...originHeaders]);
         this.cache.addEntry(event.request.url, baqendResponse);
         return baqendResponse;
     }
@@ -91,27 +91,26 @@ class VirtualOrestesApp {
         const buffer = Buffer.from(await response.arrayBuffer());
         const contentType = response?.headers.get("content-type") || "";
         const charsetRegex = /charset=([^\s"(),./:;<=>?@[\]]*)/i;
-        const encodingFromHeader = contentType && charsetRegex.test(contentType)
+        const encodingFromHeader = (contentType && charsetRegex.test(contentType)
             ? charsetRegex.exec(contentType)[1]
-            : null;
-        const encodingFromBuffer = encoding_japanese_1.default.detect(buffer);
-        const encodedBody = iconv_lite_2.default
-            .decode(buffer, encodingFromHeader ?? "utf-8")
-            .toString();
+            : null) ?? "utf-8";
+        if (!iconv_lite_1.default.encodingExists(encodingFromHeader)) {
+            throw new unsupported_encoding_error_1.UnsupportedEncodingError(`encoding is not supported by cli: "${encodingFromHeader}"`);
+        }
+        let encodedBody = iconv_lite_1.default.decode(buffer, encodingFromHeader);
         const metaSelector = onboarding_model_1.META_CONTENT_TYPE_REGEX;
         const encodingFromBodyMeta = metaSelector.test(encodedBody)
             ? metaSelector.exec(encodedBody)[1]
-            : "";
-        const detectedEncoding = encodingFromHeader ||
-            encodingFromBodyMeta ||
-            encodingFromBuffer ||
-            "utf-8";
-        if (!iconv_lite_2.default.encodingExists(detectedEncoding)) {
-            throw new unsupported_encoding_error_1.UnsupportedEncodingError(`encoding is not supported by cli: "${detectedEncoding}"`);
+            : null;
+        if (encodingFromBodyMeta && encodingFromBodyMeta !== encodingFromHeader) {
+            if (!iconv_lite_1.default.encodingExists(encodingFromBodyMeta)) {
+                throw new unsupported_encoding_error_1.UnsupportedEncodingError(`encoding is not supported by cli: "${encodingFromBodyMeta}"`);
+            }
+            encodedBody = iconv_lite_1.default.decode(buffer, encodingFromBodyMeta);
         }
         return {
-            responseContent: iconv_lite_1.default.decode(Buffer.from(buffer), detectedEncoding),
-            responseEncoding: detectedEncoding,
+            responseContent: encodedBody,
+            textEncoding: encodingFromBodyMeta ?? encodingFromHeader,
         };
     }
     returnErrorResponse(customResponse, customHeaders) {

package/dist/services/prewarm/pre-warm-service.js

@@ -75,7 +75,7 @@ class PreWarmService {
         let batch = [];
         let completed = 0;
         let lastTime = Date.now();
-        const workers = Array.from({ length: 10 }).map(async () => {
+        const workers = Array.from({ length: 100 }).map(async () => {
             for (;;) {
                 const { done, value } = iterator.next();
                 if (done) {

package/oclif.manifest.json

@@ -731,5 +731,5 @@
       ]
     }
   },
-  "version": "3.28.0"
+  "version": "3.29.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@speedkit/cli",
   "description": "Speed Kit CLI",
-  "version": "3.28.0",
+  "version": "3.29.0",
   "author": {
     "name": "Baqend.com",
     "email": "info@baqend.com"
@@ -67,6 +67,7 @@
   "dependencies": {
     "@aws-sdk/client-athena": "^3.529",
     "@inquirer/prompts": "^3.3.2",
+    "@nats-io/transport-node": "^3.3.0",
     "@oclif/core": "^3.23",
     "@oclif/plugin-autocomplete": "^3.0",
     "@oclif/plugin-help": "^6.2.19",
@@ -90,7 +91,7 @@
     "fs-extra": "^11.2.0",
     "handlebars": "^4.7.8",
     "htmlparser2": "8.0.2",
-    "iconv-lite": "^0.6.3",
+    "iconv-lite": "^0.7.2",
     "json2csv": "^6.0.0-alpha.2",
     "node-fetch": "^2.7.0",
     "nypm": "^0.6.2",