@speedkit/cli 2.27.0 → 2.29.0

package/dist/services/onboarding/onboarding-model.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OnboardingContext = exports.BrowserContext = void 0;
+exports.USER_AGENT = exports.OnboardingContext = exports.BrowserContext = void 0;
 const core_1 = require("@oclif/core");
 const cli_parameters_1 = require("../../models/cli-parameters");
 const DEFAULT_BROWSER_ARGS = [
@@ -12,13 +12,15 @@ class BrowserContext {
     domain;
     chromeFlags;
     userConfig;
+    userAgent;
     browserArgs;
     chromePath;
     startParameterDashboard = false;
-    constructor(domain, chromeFlags, userConfig) {
+    constructor(domain, chromeFlags, userConfig, userAgent) {
         this.domain = domain;
         this.chromeFlags = chromeFlags;
         this.userConfig = userConfig;
+        this.userAgent = userAgent;
         this.domain = domain;
         this.chromeFlags = chromeFlags || [];
         this.chromePath = userConfig.chromePath;
@@ -74,3 +76,7 @@ class OnboardingContext {
     }
 }
 exports.OnboardingContext = OnboardingContext;
+exports.USER_AGENT = {
+    DESKTOP: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 (compatible; SpeedKit/1.0)",
+    MOBILE: "Mozilla/5.0 (Linux; Android 7.1.1; ONEPLUS A5000 Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36 (compatible; SpeedKit/1.0)",
+};

package/dist/services/onboarding/onboarding-service-factory.d.ts

@@ -10,6 +10,7 @@ export declare class OnboardingServiceFactory {
     private getAthenaClient;
     private getConfigApi;
     private getFetchEventHandler;
+    private getSpeedKitServerConfig;
     private getIntegrationFiles;
     private getUrlForTest;
 }

package/dist/services/onboarding/onboarding-service-factory.js

@@ -11,6 +11,7 @@ const config_api_context_1 = tslib_1.__importDefault(require("../config-api/cont
 const config_api_service_factory_1 = tslib_1.__importDefault(require("../config-api/factory/config-api-service-factory"));
 const diff_1 = require("../diff");
 const document_handler_runtime_service_factory_1 = tslib_1.__importDefault(require("../document-handler-runtime/factory/document-handler-runtime-service-factory"));
+const integration_api_1 = require("../integration-api");
 const install_file_recipe_1 = require("./config-renderer/install-file-recipe");
 const speed_kit_install_context_1 = require("./config-renderer/speed-kit-install-context");
 const dashboard_1 = require("./dashboard");
@@ -32,7 +33,8 @@ const cache_1 = require("./request-cache/cache");
 const install_speed_kit_html_template_1 = require("./templates/install-speed-kit-html-template");
 const install_speed_kit_js_template_1 = require("./templates/install-speed-kit-js-template");
 const todo_collector_1 = require("./todo-collector");
-const integration_api_1 = require("../integration-api");
+const crawler_1 = require("./virtual-orestes-app/crawler");
+const virtual_orestes_app_1 = require("./virtual-orestes-app");
 class OnboardingServiceFactory {
     context;
     cliConfig;
@@ -89,6 +91,7 @@ class OnboardingServiceFactory {
     }
     async getFetchEventHandler(files, customerConfig, documentHandler, cache, cli, browserContext) {
         const configApi = this.getConfigApi(customerConfig.app);
+        const serverConfig = await this.getSpeedKitServerConfig(configApi, cli);
         const parameterQueryBuilder = new parameter_query_builder_1.ParameterQueryBuilder(customerConfig, files.getByName(files_1.INTEGRATION_FILES.CONFIG.SPEED_KIT));
         const DiffService = new diff_1.DiffServiceFactory(new diff_1.DiffContext(this.cliConfig.diffExec, this.cliConfig.diffExecTemplate)).getService();
         const requestDiffService = new request_diff_service_1.RequestDiffService(DiffService, documentHandler);
@@ -100,20 +103,26 @@ class OnboardingServiceFactory {
             new speed_kit_service_worker_js_1.SpeedKitServiceWorkerJs(customerConfig, files.getByName(files_1.INTEGRATION_FILES.CUSTOM.SW)),
         ];
         if (this.context.local) {
-            handlers.push(new speed_kit_asset_request_1.SpeedKitAssetRequest(customerConfig, documentHandler, cache, cli), new speed_kit_rum_pi_request_1.SpeedKitRumPiRequest(customerConfig, cache, cli));
+            const crawler = new crawler_1.Crawler(cli, serverConfig);
+            const orestesApp = new virtual_orestes_app_1.VirtualOrestesApp(customerConfig, crawler, documentHandler, cache, cli);
+            handlers.push(new speed_kit_asset_request_1.SpeedKitAssetRequest(customerConfig, orestesApp), new speed_kit_rum_pi_request_1.SpeedKitRumPiRequest(customerConfig, cache, cli));
         }
         const athenaClient = await this.getAthenaClient();
         if (athenaClient) {
             handlers.push(new dashboard_request_1.DashboardRequest(new dashboard_1.Dashboard(customerConfig, parameterQueryBuilder, athenaClient, requestDiffService, cache)));
-            /**
-             *   todo check error `Cannot read properties of undefined (reading 'mainFrame')`
-             *   until this is fixed, we need to deaktivate the parameter dashboard
-             *   See ticket: https://www.notion.so/baqend/d3749c5796d647f5892aa5ebfa2fd75f?v=306744f97b924ce29f59a2a3779a045c&p=728aae13a9a94c10807f368143b7dabf&pm=s
-             **/
-            // browserContext.startParameterDashboard = true;
         }
         return new fetch_event_handler_1.FetchEventHandler(handlers, customerConfig, cli, configApi);
     }
+    async getSpeedKitServerConfig(configApi, cli) {
+        cli.startAction("get server config from app");
+        const serverConfigResult = await (0, safe_1.safe)(configApi.getServerConfig());
+        if (serverConfigResult.success !== true) {
+            cli.endAction(cli_1.CliActionStatus.FAILED);
+            return;
+        }
+        cli.endAction(cli_1.CliActionStatus.COMPLETED);
+        return JSON.parse(serverConfigResult.data);
+    }
     async getIntegrationFiles() {
         const integrationApiContext = new integration_api_1.IntegrationApiContext(this.context.customerPath, this.context.configName, []);
         const integrationApi = new integration_api_1.IntegrationApiFactory(integrationApiContext, this.cliConfig).buildService();

package/dist/services/onboarding/onboarding-service.js

@@ -63,8 +63,8 @@ class OnboardingService {
         await this.prepareCleanState(page);
         this.registerFileWatcher(page);
         this.registerReinstallSpeedKitOnNavigate(page);
-        // inject open Todos from configFiles
         page.on("framenavigated", async (_frame) => {
+            // inject open Todos from configFiles
             const tasks = this.taskCollector.getOpenTasks();
             await (0, safe_1.safe)(page.evaluate(`window.skOnboarding = ${JSON.stringify(tasks)}`));
         });

package/dist/services/onboarding/request-cache/cache.js

@@ -22,7 +22,15 @@ class Cache {
         this.inMemoryCache.set(requestUri, response);
     }
     removeEntry(requestUri) {
-        this.inMemoryCache.delete(requestUri);
+        const keysToDelete = [];
+        for (const key of this.inMemoryCache.keys()) {
+            if (key.includes(requestUri)) {
+                keysToDelete.push(key);
+            }
+        }
+        for (const key of keysToDelete) {
+            this.inMemoryCache.delete(key);
+        }
     }
     clear() {
         this.cli.writeWarning("clear request-caches");

package/dist/services/onboarding/virtual-orestes-app/crawler.d.ts

@@ -0,0 +1,10 @@
+import { SpeedKitServerConfig } from "../onboarding-model";
+import { CliService } from "../../cli";
+export declare class Crawler {
+    private cli;
+    private speedKitServerConfig?;
+    constructor(cli: CliService, speedKitServerConfig?: SpeedKitServerConfig);
+    fetchRemote(originUrl: string, variation: string): Promise<Response>;
+    private setDefaultUserAgent;
+    private prepareOriginRequest;
+}

package/dist/services/onboarding/virtual-orestes-app/crawler.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Crawler = void 0;
+const tslib_1 = require("tslib");
+const onboarding_model_1 = require("../onboarding-model");
+const safe_1 = require("../../../helpers/safe");
+const node_https_1 = require("node:https");
+const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
+const agent = new node_https_1.Agent({
+    rejectUnauthorized: false,
+    keepAlive: true,
+    secureOptions: node_crypto_1.default.constants.SSL_OP_LEGACY_SERVER_CONNECT,
+});
+class Crawler {
+    cli;
+    speedKitServerConfig;
+    constructor(cli, speedKitServerConfig) {
+        this.cli = cli;
+        this.speedKitServerConfig = speedKitServerConfig;
+    }
+    async fetchRemote(originUrl, variation) {
+        const userAgent = this.setDefaultUserAgent(variation);
+        const headers = { "User-Agent": userAgent };
+        const requestInit = {
+            ...this.prepareOriginRequest(variation, {
+                headers,
+            }),
+            agent,
+        };
+        const fetchResponse = await (0, safe_1.safe)(fetch(originUrl, requestInit));
+        if (fetchResponse.success === true) {
+            return fetchResponse.data;
+        }
+        this.cli.writeError(`error while fetching Asset: ${originUrl}`);
+        this.cli.writeError(fetchResponse.error);
+        this.cli.writeError(JSON.stringify(fetchResponse.errorObj));
+    }
+    setDefaultUserAgent(variation) {
+        if (variation.includes("MOBILE")) {
+            return onboarding_model_1.USER_AGENT.MOBILE;
+        }
+        return onboarding_model_1.USER_AGENT.DESKTOP;
+    }
+    prepareOriginRequest(variationParameter, init) {
+        if (!this.speedKitServerConfig || !variationParameter) {
+            return init;
+        }
+        for (const variationKey in this.speedKitServerConfig.speedKit.variations) {
+            if (variationParameter === variationKey.toUpperCase()) {
+                const variationConfig = this.speedKitServerConfig.speedKit.variations[variationKey];
+                init.headers = { ...init.headers, ...variationConfig.headers };
+                return init;
+            }
+        }
+        return init;
+    }
+}
+exports.Crawler = Crawler;

package/dist/services/onboarding/virtual-orestes-app/index.d.ts

@@ -0,0 +1,26 @@
+import Protocol from "devtools-protocol";
+import { AbortResponse } from "../browser/abort-response";
+import { DocumentHandlerServer } from "../../document-handler-runtime/document-handler-server";
+import { Cache } from "../request-cache/cache";
+import { CliService } from "../../cli";
+import { Crawler } from "./crawler";
+import { CustomerConfig } from "../../integration-api";
+export declare class VirtualOrestesApp {
+    private customerConfig;
+    private crawler;
+    private documentHandler;
+    private cache;
+    private cli;
+    constructor(customerConfig: CustomerConfig, crawler: Crawler, documentHandler: DocumentHandlerServer, cache: Cache, cli: CliService);
+    fetchAssetResponse(event: Protocol.Fetch.RequestPausedEvent): Promise<Partial<Protocol.Fetch.FulfillRequestRequest> | AbortResponse>;
+    private returnErrorResponse;
+    private convertResponseHeadersToOrestesFormat;
+    private returnRedirect;
+    private handleCdnRedirect;
+    private getAssetUrl;
+    private getCachedResponse;
+    private prepareCustomHeaders;
+    private isHtmlContentHeader;
+    private isValidHtmlContent;
+    private rewriteInstallResource;
+}

package/dist/services/onboarding/virtual-orestes-app/index.js

@@ -0,0 +1,190 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VirtualOrestesApp = void 0;
+const baqend_response_1 = require("../browser/baqend-response");
+const safe_1 = require("../../../helpers/safe");
+class VirtualOrestesApp {
+    customerConfig;
+    crawler;
+    documentHandler;
+    cache;
+    cli;
+    constructor(customerConfig, crawler, documentHandler, cache, cli) {
+        this.customerConfig = customerConfig;
+        this.crawler = crawler;
+        this.documentHandler = documentHandler;
+        this.cache = cache;
+        this.cli = cli;
+    }
+    async fetchAssetResponse(event) {
+        const originUrl = this.getAssetUrl(event.request.url);
+        const UrlObject = new URL(event.request.url);
+        const variation = UrlObject.searchParams.has("bqvariation")
+            ? UrlObject.searchParams.get("bqvariation").trim().toUpperCase()
+            : "DEFAULT";
+        const cachedResponse = this.getCachedResponse(event.request.url);
+        if (cachedResponse instanceof baqend_response_1.BaqendResponse) {
+            return cachedResponse;
+        }
+        const response = await this.crawler.fetchRemote(originUrl, variation);
+        // todo check if we can handle more then htmlResponses
+        // only handle html responses
+        if (!response.headers.get("Content-Type").includes("html")) {
+            return;
+        }
+        const customHeaders = this.prepareCustomHeaders(originUrl);
+        // CDN Redirect
+        if (originUrl !== response.url) {
+            return this.handleCdnRedirect(customHeaders, response, event);
+        }
+        if (response.status > 300 && response.status < 400) {
+            return this.returnRedirect(customHeaders, response, event);
+        }
+        const responseContent = await response.text();
+        if (!this.isHtmlContentHeader(response.headers) ||
+            !this.isValidHtmlContent(responseContent)) {
+            const customResponse = new baqend_response_1.BaqendResponse(responseContent, response.headers.get("Content-Type") || null, customHeaders);
+            this.cache.addEntry(event.request.url, customResponse);
+            return customResponse;
+        }
+        const convertedResponseHeaders = this.convertResponseHeadersToOrestesFormat(response);
+        const customResponse = await (0, safe_1.safe)(this.documentHandler.transform(responseContent, variation, originUrl, convertedResponseHeaders));
+        if (customResponse.success === false) {
+            return this.returnErrorResponse(customResponse, customHeaders);
+        }
+        const skResponse = customResponse.data;
+        if (this.customerConfig.forceInstall) {
+            skResponse.body = this.rewriteInstallResource(skResponse.body);
+        }
+        for (const key in skResponse.headers) {
+            customHeaders.push({
+                name: `x-debug-${key}`,
+                value: skResponse.headers[key],
+            });
+        }
+        const originHeaders = [];
+        // eslint-disable-next-line unicorn/no-array-for-each
+        response.headers.forEach((value, key) => {
+            originHeaders.push({ name: `x-origin-${key}`, value: value });
+        });
+        const baqendResponse = new baqend_response_1.BaqendResponse(skResponse.body, null, [
+            ...customHeaders,
+            ...originHeaders,
+        ]);
+        this.cache.addEntry(event.request.url, baqendResponse);
+        return baqendResponse;
+    }
+    returnErrorResponse(customResponse, customHeaders) {
+        let message = `could not transform html - ${customResponse.error}`;
+        if (customResponse.errorObj) {
+            message = customResponse.errorObj.message;
+        }
+        this.cli.writeError(message);
+        this.cli.writeError(JSON.stringify(customResponse.errorObj));
+        customHeaders.push({ name: "x-error", value: message });
+        return new baqend_response_1.BaqendResponse(message, null, customHeaders, 500);
+    }
+    convertResponseHeadersToOrestesFormat(response) {
+        const convertedResponseHeaders = {};
+        // eslint-disable-next-line unicorn/no-array-for-each
+        response.headers.forEach((value, key) => {
+            convertedResponseHeaders[key] = value;
+        });
+        return convertedResponseHeaders;
+    }
+    returnRedirect(customHeaders, response, event) {
+        customHeaders.push({
+            name: "location",
+            value: response.headers.get("location") || "",
+        });
+        const redirectResponse = new baqend_response_1.BaqendResponse("", null, customHeaders, response.status);
+        this.cache.addEntry(event.request.url, redirectResponse);
+        return redirectResponse;
+    }
+    handleCdnRedirect(customHeaders, response, event) {
+        customHeaders.push({
+            name: "x-debug-response-url",
+            value: response.url,
+        }, {
+            name: "x-debug-redirect",
+            value: "response url differs from request url",
+        });
+        const serverTimingHeader = customHeaders.find((header) => {
+            return header.name === "server-timing";
+        });
+        serverTimingHeader.value += ",error;desc=202";
+        const redirectResponse = new baqend_response_1.BaqendResponse("", null, customHeaders, 400);
+        this.cache.addEntry(event.request.url, redirectResponse);
+        return redirectResponse;
+    }
+    getAssetUrl(requestUrl) {
+        const url = new URL(requestUrl);
+        const parametersToDelete = [];
+        for (const [name] of url.searchParams.entries()) {
+            if (name.startsWith("bq")) {
+                parametersToDelete.push(name);
+            }
+        }
+        for (const parameter of parametersToDelete) {
+            url.searchParams.delete(parameter);
+        }
+        const cleanedUrl = url.toString();
+        return cleanedUrl.split("app.baqend.com/v1/asset/")[1];
+    }
+    getCachedResponse(requestUrl) {
+        const response = this.cache.getEntry(requestUrl);
+        if (!(response instanceof baqend_response_1.BaqendResponse)) {
+            return;
+        }
+        response.setHeader("x-cache", "POP, HIT");
+        response.setHeader("x-served-by", "cache-local");
+        response.setHeader("server-timing", "pop;desc=LOCAL;dur=1,cache;desc=CACHE,hotness;desc=1,proto;desc=h3");
+        return response;
+    }
+    prepareCustomHeaders(originUrl) {
+        const time = new Date().toUTCString();
+        const originDomain = new URL(originUrl).origin;
+        return [
+            { name: "origin", value: originDomain },
+            { name: "Access-Control-Allow-Credentials", value: "true" },
+            {
+                name: "strict-transport-security",
+                value: "max-age=31536000; includeSubDomains",
+            },
+            { name: "timing-allow-origin", value: originDomain },
+            {
+                name: "Cache-Control",
+                value: "public,browser-ttl=0,sw-max-age=31536000,max-age=31469674",
+            },
+            {
+                name: "Alt-Svc",
+                value: 'h3=":443";ma=86400,h3-29=":443";ma=86400,h3-27=":443";ma=86400',
+            },
+            { name: "x-frame-options", value: "SAMEORIGIN" },
+            { name: "x-cache", value: "STREAM, HIT" },
+            { name: "x-cache-hits", value: "1" },
+            { name: "x-served-by", value: "stream-local" },
+            { name: "last-modified", value: time },
+            { name: "Baqend-Created-At", value: time },
+            { name: "Age", value: "0" },
+            {
+                name: "server-timing",
+                value: "pop;desc=LOCAL;dur=1,cache;desc=STREAM,hotness;desc=1,proto;desc=h3",
+            },
+        ];
+    }
+    isHtmlContentHeader(responseHeaders) {
+        return (responseHeaders.has("content-type") &&
+            responseHeaders.get("content-type").includes("text/html"));
+    }
+    isValidHtmlContent(text) {
+        return text.includes("<html");
+    }
+    rewriteInstallResource(text) {
+        text = text.replace(/<script(?:.(?!<\/script>))*app\.baqend\.com\/v1\/speedkit\/install.js(?:.|(?!<\/script>))*<\/script>/, "");
+        return text.replace(/<\s*head\b[^>]*>/, (head) => {
+            return `${head}\n<script src="${this.customerConfig.installPath}" ${this.customerConfig.installParams}></script>`;
+        });
+    }
+}
+exports.VirtualOrestesApp = VirtualOrestesApp;

package/oclif.manifest.json

@@ -667,5 +667,5 @@
       ]
     }
   },
-  "version": "2.27.0"
+  "version": "2.29.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@speedkit/cli",
   "description": "Speed Kit CLI",
-  "version": "2.27.0",
+  "version": "2.29.0",
   "author": {
     "name": "Baqend.com",
     "email": "info@baqend.com"