@speedkit/cli 3.5.1 → 3.6.0-beta.1

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+# [3.6.0-beta.1](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.5.1...v3.6.0-beta.1) (2025-09-02)
+
+
+### Features
+
+* **onboarding:** implement dynamic encoding for documentHandling ([290cf82](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/290cf8267208eeb577698001b3a5e9d48524eb66))
+* **onboarding:** overwrite content-type header for docHandlerRequest ([cfba8d6](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/cfba8d69db47dcfb193c2d796451783d01f72f59))
+
 ## [3.5.1](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v3.5.0...v3.5.1) (2025-08-26)
 
 

package/README.md

@@ -21,7 +21,7 @@ $ npm install -g @speedkit/cli
 $ sk COMMAND
 running command...
 $ sk (--version)
-@speedkit/cli/3.5.1 linux-x64 node-v20.19.4
+@speedkit/cli/3.6.0-beta.1 linux-x64 node-v20.19.4
 $ sk --help [COMMAND]
 USAGE
   $ sk COMMAND

package/dist/services/document-handler-runtime/server/document-handler-response.d.ts

@@ -2,7 +2,7 @@ export declare class DocumentHandlerResponse {
     headers: {
         [name: string]: string;
     };
-    body: string;
+    body: string | Buffer;
     resolve: (resolvable: unknown) => void;
     header(key: string, value: string): void;
     setHeader(key: string, value: string): void;

package/dist/services/document-handler-runtime/server/request.d.ts

@@ -4,11 +4,13 @@ type query = {
     headers: string;
 };
 export declare class Request {
+    private headers;
     query: query;
     body: string;
     constructor(url: string, body: string, variation: string, headers: {
         [name: string]: string;
     });
     get(key: string): string | null;
+    header(key: string): string | null;
 }
 export {};

package/dist/services/document-handler-runtime/server/request.js

@@ -2,18 +2,19 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Request = void 0;
 class Request {
+    headers;
     query;
     body;
     constructor(url, body, variation, headers) {
+        this.headers = headers;
         this.query = { variation, headers: JSON.stringify(headers), url };
         this.body = body;
     }
     get(key) {
-        const headers = JSON.parse(this.query.headers);
-        const foundKey = Object.keys(headers).find((headerKey) => {
-            return headerKey.toLowerCase().includes(key.toLowerCase());
-        });
-        return this.query.headers[foundKey];
+        return this.headers?.[key] || null;
+    }
+    header(key) {
+        return this.get(key);
     }
 }
 exports.Request = Request;

package/dist/services/onboarding/browser/baqend-response.d.ts

@@ -6,7 +6,7 @@ export declare class BaqendResponse implements Partial<Protocol.Fetch.FulfillReq
     responseHeaders: Protocol.Fetch.HeaderEntry[];
     private readonly contentType;
     private readonly contentLength;
-    constructor(body: string, contentType?: string, headers?: Protocol.Fetch.HeaderEntry[], status?: number);
+    constructor(body: string | Buffer, contentType?: string, headers?: Protocol.Fetch.HeaderEntry[], status?: number);
     setHeader(name: string, value: string): void;
     protected addCustomHeaders(headers?: Protocol.Fetch.HeaderEntry[]): Protocol.Fetch.HeaderEntry[];
 }

package/dist/services/onboarding/browser/baqend-response.js

@@ -22,7 +22,9 @@ class BaqendResponse {
             ...DEFAULT_HEADERS,
             ...(headers || []),
         ]);
-        this.body = Buffer.from(body).toString("base64");
+        this.body = Buffer.isBuffer(body)
+            ? body.toString("base64")
+            : Buffer.from(body).toString("base64");
     }
     setHeader(name, value) {
         for (const header of this.responseHeaders) {

package/dist/services/onboarding/browser/extension/extension-downloader.d.ts

@@ -4,7 +4,7 @@ export declare class ExtensionDownloader {
     private userConfig;
     private cli;
     constructor(userConfig: UserCliConfig, cli: CliServiceInterface);
-    downloadExtension(): Promise<string | false>;
+    downloadExtension(): Promise<string>;
     private updateLocalDevtools;
     isInstalled(): boolean;
     private checkInstalledVersion;

package/dist/services/onboarding/browser/extension/extension-downloader.js

@@ -8,6 +8,7 @@ const semver_1 = tslib_1.__importDefault(require("semver"));
 const extract_zip_1 = tslib_1.__importDefault(require("extract-zip"));
 const cli_1 = require("../../../cli");
 const safe_1 = require("../../../../helpers/safe");
+const validate_extension_error_1 = require("../../error/validate-extension-error");
 const devtools = "integration-devtools";
 class ExtensionDownloader {
     userConfig;
@@ -26,14 +27,14 @@ class ExtensionDownloader {
             if (semver_1.default.compare(remoteVersion, localVersion) > 0) {
                 const response = await this.cli.confirm(`integration-devtools update available from ${localVersion} to ${remoteVersion}  Install?`, true);
                 if (!response) {
-                    return false;
+                    throw new validate_extension_error_1.ValidateExtensionError(localVersion, remoteVersion);
                 }
             }
         }
         else {
             const response = await this.cli.confirm("Do you want to install devtools-extension?", true);
             if (!response) {
-                return false;
+                return;
             }
         }
         this.cli.startAction("update integration-devtools");
@@ -41,7 +42,7 @@ class ExtensionDownloader {
         if (response.success === false) {
             this.cli.endAction(cli_1.CliActionStatus.FAILED);
             this.cli.writeError(response.error);
-            return false;
+            return;
         }
         this.cli.endAction(cli_1.CliActionStatus.COMPLETED);
         return response.data;

package/dist/services/onboarding/browser/extension/extension-validator.js

@@ -60,18 +60,27 @@ class ExtensionValidator {
         return result.data;
     }
     async handleDevtoolsExtension(validExtensions) {
+        // check if devtools extension can be find in current  extensionPaths
         if (validExtensions.some((config) => config.name.toLowerCase().includes("devtools extension"))) {
             return;
         }
-        const path = await this.extensionDownloader.downloadExtension();
-        if (path === false) {
+        // download or update extension
+        const result = await (0, safe_1.safe)(this.extensionDownloader.downloadExtension());
+        if (result.success !== true) {
+            return;
+        }
+        if (result.data?.length > 0) {
+            return;
+        }
+        // get extensionDetails from manifestFile
+        const extensionConfigResult = await (0, safe_1.safe)(this.getManifestConfig(result.data));
+        if (extensionConfigResult.success !== true) {
             return;
         }
-        const extensionConfig = await this.getManifestConfig(path);
         validExtensions.push({
-            path: path,
-            name: extensionConfig.name,
-            version: extensionConfig.version,
+            path: result.data,
+            name: extensionConfigResult.data.name,
+            version: extensionConfigResult.data.version,
         });
     }
 }

package/dist/services/onboarding/browser/origin-response.d.ts

@@ -1,8 +1,8 @@
 import Protocol from "devtools-protocol";
 export declare class OriginResponse implements Partial<Protocol.Fetch.FulfillRequestRequest> {
-    readonly body: string;
     readonly responseCode: number;
     readonly responseHeaders: Protocol.Fetch.HeaderEntry[];
-    constructor(body: string, headers: Protocol.Fetch.HeaderEntry[], responseCode: number);
+    readonly body: string;
+    constructor(body: string | Buffer, headers: Protocol.Fetch.HeaderEntry[], responseCode: number);
     private addCustomHeaders;
 }

package/dist/services/onboarding/browser/origin-response.js

@@ -2,14 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OriginResponse = void 0;
 class OriginResponse {
-    body;
     responseCode;
     responseHeaders;
+    body;
     constructor(body, headers, responseCode) {
-        this.body = body;
         this.responseCode = responseCode;
         this.responseHeaders = this.addCustomHeaders(headers);
-        this.body = Buffer.from(body).toString("base64");
+        this.body = Buffer.isBuffer(body)
+            ? body.toString("base64")
+            : Buffer.from(body).toString("base64");
     }
     addCustomHeaders(headers) {
         headers.push({ name: "x-served-via", value: "sk-onboarding" }, { name: "Service-Worker-Allowed", value: "/" });

package/dist/services/onboarding/dashboard/diff-against-current-page.js

@@ -27,9 +27,17 @@ class DiffAgainstCurrentPage {
         if (transform) {
             // todo check if the currentHTML is already transformed
             const currentDocumentHandlerResponse = await this.documentHandlerRuntime.transform(currentHtml, variant, currentUrl, {});
-            currentHtml = currentDocumentHandlerResponse.body;
+            const currentResponse = currentDocumentHandlerResponse.body;
+            if (Buffer.isBuffer(currentResponse)) {
+                throw new TypeError("cant diff body as buffer");
+            }
+            currentHtml = currentResponse;
             const originDocumentHandlerResponse = await this.documentHandlerRuntime.transform(originHtml, variant, currentUrl, {});
-            originHtml = originDocumentHandlerResponse.body;
+            const originResponse = originDocumentHandlerResponse.body;
+            if (Buffer.isBuffer(originResponse)) {
+                throw new TypeError("cant diff body as buffer");
+            }
+            originHtml = originResponse;
         }
         this.cli.endAction(cli_1.CliActionStatus.COMPLETED);
         const remoteFilePath = await this.diffService.writeContentToTemporalFile(`remote-${variant}.html`, originHtml);

package/dist/services/onboarding/dashboard/request-diff-service.js

@@ -41,9 +41,15 @@ class RequestDiffService {
         const replaceRegEx = /<meta[^>]*name="baqend:asset_url"[^>]*>/;
         // <meta name="baqend:asset_url" content="https://www.kik.de/c/damen/damen-bekleidung-shirts-tops?page=7">
         const transformedHtmlWithParameterResponse = await this.documentHandlerRuntime.transform(htmlWithParameter, "", urlWithParameter, {});
+        if (Buffer.isBuffer(transformedHtmlWithParameterResponse.body)) {
+            throw new TypeError("Can't diff body if it's a buffer");
+        }
         let transformedHtmlWithParameter = transformedHtmlWithParameterResponse.body;
         transformedHtmlWithParameter = transformedHtmlWithParameter.replace(replaceRegEx, "");
         const transformedHtmlWithoutParameterResponse = await this.documentHandlerRuntime.transform(htmlWithoutParameter, "", urlWithParameter, {});
+        if (Buffer.isBuffer(transformedHtmlWithoutParameterResponse.body)) {
+            throw new TypeError("Can't diff body if it's a buffer");
+        }
         let transformedHtmlWithoutParameter = transformedHtmlWithoutParameterResponse.body;
         transformedHtmlWithoutParameter = transformedHtmlWithoutParameter.replace(replaceRegEx, "");
         if (checkIsEqual) {

package/dist/services/onboarding/error/validate-extension-error.d.ts

@@ -0,0 +1,5 @@
+import ApplicationError from "../../error-handling/error/application-error";
+export declare class ValidateExtensionError extends ApplicationError {
+    readonly code = "VALIDATE_EXTENSION_ERROR";
+    constructor(localVersion: string, remoteVersion: string);
+}

package/dist/services/onboarding/error/validate-extension-error.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidateExtensionError = void 0;
+const tslib_1 = require("tslib");
+const application_error_1 = tslib_1.__importDefault(require("../../error-handling/error/application-error"));
+class ValidateExtensionError extends application_error_1.default {
+    code = "VALIDATE_EXTENSION_ERROR";
+    constructor(localVersion, remoteVersion) {
+        super(`Unable to validate Versions: "${localVersion}" - "${remoteVersion}"`);
+    }
+}
+exports.ValidateExtensionError = ValidateExtensionError;

package/dist/services/onboarding/fetch-events/customer-domain-document-response.js

@@ -5,7 +5,7 @@ const tslib_1 = require("tslib");
 const onboarding_model_1 = require("../onboarding-model");
 const origin_response_1 = require("../browser/origin-response");
 const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
-const SUPPORTED_CHARSETS = new Set(["ascii", "utf8", "utf16le", "latin1"]);
+const encoding_japanese_1 = tslib_1.__importDefault(require("encoding-japanese"));
 /**
  * Intercepts any request to the customersOrigin and equivalent requests to baqendAssetApi
  * @implements {FetchRequestPausedEventHandler}
@@ -62,33 +62,31 @@ class CustomerDomainDocumentResponse {
         if (!this.isHtmlContentHeader(event.responseHeaders)) {
             return;
         }
-        const contentType = event.responseHeaders.find((headerRecord) => {
-            return headerRecord.name.toLowerCase() === "content-type";
-        })?.["value"] || "";
-        const charset = contentType.split("charset=")?.[1]?.trim() || "utf-8";
-        let text = await this.getHtmlContent(client, event, charset);
+        let text = await this.getHtmlContent(client, event);
         if (!this.isValidHtmlContent(text)) {
             return;
         }
+        const headers = this.removeContentSecurityPolicy
+            ? this.removeContentSecurityHeaderFromResponse(event.responseHeaders)
+            : event.responseHeaders;
         if (this.customerConfig.forceInstall) {
             text = this.rewriteInstallResource(text);
         }
         if (this.isSpeedKitResponse(text)) {
             text = this.rewriteHtmlToLocalConfig(text);
         }
-        const headers = this.removeContentSecurityPolicy
-            ? this.removeContentSecurityHeaderFromResponse(event.responseHeaders)
-            : event.responseHeaders;
         const content = this.removeContentSecurityPolicy
             ? this.removeContentSecurityMetaTag(text)
             : text;
         // replace origin contentTypeCharset with utf-8 as this is what it's converted to now
+        let detectedEncoding = "utf-8";
         for (const header of headers) {
-            if (header.name.toLowerCase().includes("content-type")) {
-                header.value = header.value.replace(charset, "utf-8");
+            if (header.name.includes("x-sk-detected-encoding")) {
+                detectedEncoding = header.value;
             }
         }
-        return new origin_response_1.OriginResponse(content, headers, event.responseStatusCode);
+        const encodedContent = iconv_lite_1.default.encode(content, detectedEncoding);
+        return new origin_response_1.OriginResponse(encodedContent, headers, event.responseStatusCode);
     }
     isValidHtmlContent(text) {
         return text.includes("<html");
@@ -105,20 +103,41 @@ class CustomerDomainDocumentResponse {
         })?.["value"];
         return contentType && contentType.includes("text/html");
     }
-    async getHtmlContent(client, event, charset) {
+    async getHtmlContent(client, event) {
         const { body, base64Encoded } = await client.send("Fetch.getResponseBody", {
             requestId: event.requestId,
         });
+        const contentType = event.responseHeaders.find((headerRecord) => {
+            return headerRecord.name.toLowerCase() === "content-type";
+        })?.["value"] || "";
+        const charsetRegex = /charset=([^\s"(),./:;<=>?@[\]]*)/i;
+        const encodingFromHeader = contentType && charsetRegex.test(contentType)
+            ? charsetRegex.exec(contentType)[1]
+            : null;
         // create text from base64Encoded
         if (base64Encoded) {
-            // make use of iconv if charset is not supported by node:buffer
-            if (!SUPPORTED_CHARSETS.has(charset)) {
-                return iconv_lite_1.default
-                    .decode(Buffer.from(body, "base64"), charset.toLowerCase())
-                    .toString();
+            const buffer = Buffer.from(body, "base64");
+            const encodingFromBuffer = encoding_japanese_1.default.detect(buffer);
+            const encodedBody = iconv_lite_1.default
+                .decode(buffer, encodingFromHeader ?? encodingFromBuffer)
+                .toString();
+            const metaSelector = /<meta http-equiv="content-type" content="text\/html;charset=([^\s"(),./:;<=>?@[\]]*)">/is;
+            const encodingFromBodyMeta = metaSelector.test(encodedBody)
+                ? metaSelector.exec(encodedBody)[1]
+                : "";
+            const detectedEncoding = encodingFromHeader ||
+                encodingFromBodyMeta ||
+                encodingFromBuffer ||
+                "utf-8";
+            if (!iconv_lite_1.default.encodingExists(detectedEncoding)) {
+                console.error(`Encoding not supported: "${detectedEncoding}" on url: ${event.request.url}`);
+                return encodedBody;
             }
-            // @ts-expect-error string is not valid
-            return Buffer.from(body, "base64").toString(charset);
+            event.responseHeaders.push({
+                name: "x-sk-detected-encoding",
+                value: detectedEncoding,
+            });
+            return iconv_lite_1.default.decode(buffer, detectedEncoding);
         }
         return body;
     }

package/dist/services/onboarding/virtual-orestes-app/index.js

@@ -6,6 +6,8 @@ const baqend_response_1 = require("../browser/baqend-response");
 const safe_1 = require("../../../helpers/safe");
 const onboarding_model_1 = require("../onboarding-model");
 const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
+const encoding_japanese_1 = tslib_1.__importDefault(require("encoding-japanese"));
+const iconv_lite_2 = tslib_1.__importDefault(require("iconv-lite"));
 class VirtualOrestesApp {
     customerConfig;
     crawler;
@@ -46,25 +48,29 @@ class VirtualOrestesApp {
         if (response.status > 300 && response.status < 400) {
             return this.returnRedirect(customHeaders, response, event);
         }
-        const responseContent = await this.getResponseText(response);
+        const { responseContent, responseEncoding } = await this.getResponseText(response);
         if (!this.isHtmlContentHeader(response.headers) ||
             !this.isValidHtmlContent(responseContent)) {
-            const customResponse = new baqend_response_1.BaqendResponse(responseContent, response.headers.get("Content-Type") || null, customHeaders);
+            const customResponse = new baqend_response_1.BaqendResponse(responseContent, `text/html; charset=${responseEncoding}`, customHeaders);
             this.cache.addEntry(event.request.url, customResponse);
             return customResponse;
         }
         const convertedResponseHeaders = this.convertResponseHeadersToOrestesFormat(response);
-        const customResponse = await (0, safe_1.safe)(this.documentHandler.transform(responseContent, variation, originUrl, convertedResponseHeaders));
+        const customResponse = await (0, safe_1.safe)(this.documentHandler.transform(responseContent, variation, originUrl, {
+            ...convertedResponseHeaders,
+            "content-type": `text/html;charset=${responseEncoding}`,
+        }));
         if (customResponse.success === false) {
+            console.error("[DOC_HANDLER_RESPONSE]", customResponse);
             return this.returnErrorResponse(customResponse, customHeaders);
         }
         const skResponse = customResponse.data;
-        if (this.customerConfig.forceInstall) {
+        if (!Buffer.isBuffer(skResponse.body) && this.customerConfig.forceInstall) {
             skResponse.body = this.rewriteInstallResource(skResponse.body);
         }
         for (const key in skResponse.headers) {
             customHeaders.push({
-                name: `x-debug-${key}`,
+                name: key,
                 value: skResponse.headers[key],
             });
         }
@@ -73,22 +79,37 @@ class VirtualOrestesApp {
         response.headers.forEach((value, key) => {
             originHeaders.push({ name: `x-origin-${key}`, value: value });
         });
-        const baqendResponse = new baqend_response_1.BaqendResponse(skResponse.body, null, [
-            ...customHeaders,
-            ...originHeaders,
-        ]);
+        customHeaders.push({
+            name: "x-sk-detected-encoding",
+            value: responseEncoding,
+        });
+        const baqendResponse = new baqend_response_1.BaqendResponse(skResponse.body, response.headers.get("Content-Type"), [...customHeaders, ...originHeaders]);
         this.cache.addEntry(event.request.url, baqendResponse);
         return baqendResponse;
     }
     async getResponseText(response) {
+        const buffer = Buffer.from(await response.arrayBuffer());
         const contentType = response.headers.get("content-type") || "";
-        const charset = contentType.split("charset=")?.[1]?.trim() || "utf-8";
-        if (charset.toLowerCase() !== "utf-8") {
-            return iconv_lite_1.default
-                .decode(Buffer.from(await response.arrayBuffer()), charset.toLowerCase())
-                .toString();
-        }
-        return await response.text();
+        const charsetRegex = /charset=([^\s"(),./:;<=>?@[\]]*)/i;
+        const encodingFromHeader = contentType && charsetRegex.test(contentType)
+            ? charsetRegex.exec(contentType)[1]
+            : null;
+        const encodingFromBuffer = encoding_japanese_1.default.detect(buffer);
+        const encodedBody = iconv_lite_2.default
+            .decode(buffer, encodingFromHeader ?? encodingFromBuffer)
+            .toString();
+        const metaSelector = /<meta http-equiv="content-type" content="text\/html;charset=([^\s"(),./:;<=>?@[\]]*)">/is;
+        const encodingFromBodyMeta = metaSelector.test(encodedBody)
+            ? metaSelector.exec(encodedBody)[1]
+            : "";
+        const detectedEncoding = encodingFromHeader ||
+            encodingFromBodyMeta ||
+            encodingFromBuffer ||
+            "utf-8";
+        return {
+            responseContent: iconv_lite_1.default.decode(Buffer.from(buffer), detectedEncoding),
+            responseEncoding: detectedEncoding,
+        };
     }
     returnErrorResponse(customResponse, customHeaders) {
         const message = `Could not transform html: - ${customResponse.error}`;

package/oclif.manifest.json

@@ -732,5 +732,5 @@
       ]
     }
   },
-  "version": "3.5.1"
+  "version": "3.6.0-beta.1"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@speedkit/cli",
   "description": "Speed Kit CLI",
-  "version": "3.5.1",
+  "version": "3.6.0-beta.1",
   "author": {
     "name": "Baqend.com",
     "email": "info@baqend.com"
@@ -81,12 +81,15 @@
     "dom-serializer": "2.0.0",
     "domhandler": "5.0.3",
     "domutils": "^3.1",
-    "fluent-ffmpeg": "^2.1.3",
     "dotenv": "^16.4.7",
+    "encoding-japanese": "^2.2.0",
+    "esbuild": "^0.20.2",
     "extract-zip": "^2.0.1",
+    "fluent-ffmpeg": "^2.1.3",
     "fs-extra": "^11.2.0",
     "handlebars": "^4.7.8",
     "htmlparser2": "8.0.2",
+    "iconv-lite": "^0.6.3",
     "json2csv": "^6.0.0-alpha.2",
     "node-fetch": "^2.7.0",
     "parse5": "7.1",
@@ -95,10 +98,8 @@
     "puppeteer-extra": "^3.3.6",
     "semver": "^7.6.3",
     "strip-comments": "^2.0.1",
-    "uuid": "^9.0.1",
-    "esbuild": "^0.20.2",
     "ts-node": "^10.9.2",
-    "iconv-lite": "^0.6.3"
+    "uuid": "^9.0.1"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.8.1",