@speedkit/cli 4.50.4 → 4.50.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## [4.50.5](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.50.4...v4.50.5) (2026-11-03)
2
+
3
+
4
+ ### Bug Fixes
5
+
6
+ * **onboarding:** detect the HTML charset like the Baqend server ([78ce6d7](https://gitlab.orestes.info/baqend/speed-kit-cli/commit/78ce6d7ac5794c385212d16ee33bc137a278208f))
7
+
1
8
  ## [4.50.4](https://gitlab.orestes.info/baqend/speed-kit-cli/compare/v4.50.3...v4.50.4) (2026-11-02)
2
9
 
3
10
 
@@ -1,8 +1,6 @@
1
- import { META_CONTENT_TYPE_REGEX, } from "../onboarding-model.js";
2
1
  import { OriginResponse } from "../browser/origin-response.js";
3
2
  import iconv from "iconv-lite";
4
- import Encoding from "encoding-japanese";
5
- import { UnsupportedEncodingError } from "../error/unsupported-encoding-error.js";
3
+ import { decodeHtml } from "../html-charset.js";
6
4
  /**
7
5
  * Intercepts any request to the customersOrigin and equivalent requests to baqendAssetApi
8
6
  * @implements {FetchRequestPausedEventHandler}
@@ -93,33 +91,14 @@ export class CustomerDomainDocumentResponse {
93
91
  const contentType = event.responseHeaders.find((headerRecord) => {
94
92
  return headerRecord.name.toLowerCase() === "content-type";
95
93
  })?.["value"] || "";
96
- const charsetRegex = /charset=([^\s"(),./:;<=>?@[\]]*)/i;
97
- const encodingFromHeader = contentType && charsetRegex.test(contentType)
98
- ? charsetRegex.exec(contentType)[1]
99
- : null;
100
94
  // create text from base64Encoded
101
95
  if (base64Encoded) {
102
- const buffer = Buffer.from(body, "base64");
103
- const encodingFromBuffer = Encoding.detect(buffer);
104
- const encodedBody = iconv
105
- .decode(buffer, encodingFromHeader ?? "utf-8")
106
- .toString();
107
- const metaSelector = META_CONTENT_TYPE_REGEX;
108
- const encodingFromBodyMeta = metaSelector.test(encodedBody)
109
- ? metaSelector.exec(encodedBody)[1]
110
- : "";
111
- const detectedEncoding = encodingFromHeader ||
112
- encodingFromBodyMeta ||
113
- encodingFromBuffer ||
114
- "utf-8";
115
- if (!iconv.encodingExists(detectedEncoding)) {
116
- throw new UnsupportedEncodingError(`encoding is not supported by cli: "${detectedEncoding}"`);
117
- }
96
+ const { text, charset } = decodeHtml(contentType, Buffer.from(body, "base64"));
118
97
  event.responseHeaders.push({
119
98
  name: "x-sk-detected-encoding",
120
- value: detectedEncoding,
99
+ value: charset,
121
100
  });
122
- return iconv.decode(buffer, detectedEncoding);
101
+ return text;
123
102
  }
124
103
  return body;
125
104
  }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Picks the charset to decode an origin HTML document with.
3
+ * It follows the rules of the Baqend server (`detectHtmlCharset`), so onboarding sees the same text as production.
4
+ *
5
+ * - A header that declares UTF-8 wins, and the body is not read.
6
+ * - Otherwise the first 1024 bytes decide: BOM, XML declaration, `<meta charset>` or `<meta http-equiv>`.
7
+ * - A charset found in the body replaces a non-UTF-8 header charset, because origins often send a wrong one.
8
+ * - Without a hint in the body, the header charset stays, else UTF-8.
9
+ *
10
+ * There is no byte-frequency guess, because the server makes none.
11
+ *
12
+ * @param contentType - The `content-type` header value, or an empty string.
13
+ * @param body - The decompressed body bytes.
14
+ * @returns The lower-case charset name.
15
+ */
16
+ export declare function detectHtmlCharset(contentType: string, body: Uint8Array): string;
17
+ /**
18
+ * Decodes an origin HTML document with the charset of {@link detectHtmlCharset}.
19
+ *
20
+ * @param contentType - The `content-type` header value, or an empty string.
21
+ * @param body - The decompressed body bytes.
22
+ * @returns The text, and the charset to encode it back with.
23
+ * @throws {UnsupportedEncodingError} If iconv-lite does not know the charset.
24
+ */
25
+ export declare function decodeHtml(contentType: string, body: Buffer): {
26
+ text: string;
27
+ charset: string;
28
+ };
@@ -0,0 +1,57 @@
1
+ import { ResultType, Sniffer } from "encoding-sniffer/sniffer";
2
+ import iconv from "iconv-lite";
3
+ import { UnsupportedEncodingError } from "./error/unsupported-encoding-error.js";
4
+ const SNIFF_BYTES = 1024;
5
+ const HTML_CONTENT_TYPE_RE = /^\s*text\/html\b/i;
6
+ const UTF8_CHARSET_RE = /;\s*charset=["']?utf-?8["']?/i;
7
+ const HEADER_CHARSET_RE = /charset=["']?([^\s"'(),./:;<=>?@[\]]+)/i;
8
+ const UTF8_RE = /^utf-?8$/i;
9
+ /**
10
+ * Picks the charset to decode an origin HTML document with.
11
+ * It follows the rules of the Baqend server (`detectHtmlCharset`), so onboarding sees the same text as production.
12
+ *
13
+ * - A header that declares UTF-8 wins, and the body is not read.
14
+ * - Otherwise the first 1024 bytes decide: BOM, XML declaration, `<meta charset>` or `<meta http-equiv>`.
15
+ * - A charset found in the body replaces a non-UTF-8 header charset, because origins often send a wrong one.
16
+ * - Without a hint in the body, the header charset stays, else UTF-8.
17
+ *
18
+ * There is no byte-frequency guess, because the server makes none.
19
+ *
20
+ * @param contentType - The `content-type` header value, or an empty string.
21
+ * @param body - The decompressed body bytes.
22
+ * @returns The lower-case charset name.
23
+ */
24
+ export function detectHtmlCharset(contentType, body) {
25
+ const fromHeader = HEADER_CHARSET_RE.exec(contentType)?.[1]?.toLowerCase();
26
+ if (!HTML_CONTENT_TYPE_RE.test(contentType) ||
27
+ UTF8_CHARSET_RE.test(contentType)) {
28
+ return fromHeader ?? "utf-8";
29
+ }
30
+ const sniffer = new Sniffer({
31
+ maxBytes: SNIFF_BYTES,
32
+ defaultEncoding: "utf-8",
33
+ });
34
+ sniffer.write(body.subarray(0, SNIFF_BYTES));
35
+ // A default result means the body carries no hint, so it must not override the header.
36
+ if (sniffer.resultType === ResultType.DEFAULT || !sniffer.encoding) {
37
+ return fromHeader ?? "utf-8";
38
+ }
39
+ return UTF8_RE.test(sniffer.encoding)
40
+ ? "utf-8"
41
+ : sniffer.encoding.toLowerCase();
42
+ }
43
+ /**
44
+ * Decodes an origin HTML document with the charset of {@link detectHtmlCharset}.
45
+ *
46
+ * @param contentType - The `content-type` header value, or an empty string.
47
+ * @param body - The decompressed body bytes.
48
+ * @returns The text, and the charset to encode it back with.
49
+ * @throws {UnsupportedEncodingError} If iconv-lite does not know the charset.
50
+ */
51
+ export function decodeHtml(contentType, body) {
52
+ const charset = detectHtmlCharset(contentType, body);
53
+ if (!iconv.encodingExists(charset)) {
54
+ throw new UnsupportedEncodingError(`encoding is not supported by cli: "${charset}"`);
55
+ }
56
+ return { text: iconv.decode(body, charset), charset };
57
+ }
@@ -206,4 +206,3 @@ export type DevelopmentToolsMessage = {
206
206
  data: LocalCacheError[];
207
207
  };
208
208
  export declare const SpeedKitInstallRegex: RegExp;
209
- export declare const META_CONTENT_TYPE_REGEX: RegExp;
@@ -282,4 +282,3 @@ export const DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-
282
282
  export const DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
283
283
  export const DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
284
284
  export const SpeedKitInstallRegex = /<script[^>]*src[^>]*app\.baqend\.com\/v1\/speedkit\/install.js[^>]*>(?:(?!<\/script>).)*<\/script>/s;
285
- export const META_CONTENT_TYPE_REGEX = /<meta[^"]*http-equiv="content-type"[^"]*content="text\/html;[^"]*charset=([^\s"(),./:;<=>?@[\]]*)"/is;
@@ -0,0 +1,39 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import iconv from "iconv-lite";
4
+ import { CustomerDomainDocumentResponse } from "../../fetch-events/customer-domain-document-response.js";
5
+ const TITLE = "掃除機";
6
+ const identity = {
7
+ headers: (headers) => headers,
8
+ html: (html) => html,
9
+ };
10
+ async function serve(contentType, body) {
11
+ const client = {
12
+ send: async () => ({ body: body.toString("base64"), base64Encoded: true }),
13
+ };
14
+ const event = {
15
+ requestId: "1",
16
+ responseStatusCode: 200,
17
+ responseHeaders: [{ name: "Content-Type", value: contentType }],
18
+ };
19
+ const handler = new CustomerDomainDocumentResponse({}, identity);
20
+ const response = await handler.handle(event, client);
21
+ const encoding = response.responseHeaders.find((header) => header.name === "x-sk-detected-encoding").value;
22
+ return { response, encoding };
23
+ }
24
+ describe("CustomerDomainDocumentResponse", () => {
25
+ it("decodes a page that declares its charset only in <meta charset>", async () => {
26
+ const html = `<html><head><meta charset="shift_jis"><title>${TITLE}</title></head></html>`;
27
+ const body = iconv.encode(html, "shift_jis");
28
+ const { response, encoding } = await serve("text/html", body);
29
+ expect(encoding).to.equal("shift_jis");
30
+ // The page reaches Chrome in its own charset, byte for byte.
31
+ expect(Buffer.from(response.body, "base64")).to.deep.equal(body);
32
+ });
33
+ it("decodes a page without any charset hint as UTF-8", async () => {
34
+ const body = Buffer.from(`<html><title>${TITLE}</title></html>`);
35
+ const { response, encoding } = await serve("text/html", body);
36
+ expect(encoding).to.equal("utf-8");
37
+ expect(Buffer.from(response.body, "base64")).to.deep.equal(body);
38
+ });
39
+ });
@@ -0,0 +1,58 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import iconv from "iconv-lite";
4
+ import { decodeHtml, detectHtmlCharset } from "../html-charset.js";
5
+ import { UnsupportedEncodingError } from "../error/unsupported-encoding-error.js";
6
+ const TITLE = "掃除機";
7
+ function page(head, charset) {
8
+ return iconv.encode(`<!doctype html><html><head>${head}<title>${TITLE}</title></head></html>`, charset);
9
+ }
10
+ describe("html-charset", () => {
11
+ it("takes <meta charset> when the header has no charset", () => {
12
+ const body = page('<meta charset="shift_jis">', "shift_jis");
13
+ expect(decodeHtml("text/html", body)).to.deep.equal({
14
+ text: iconv.decode(body, "shift_jis"),
15
+ charset: "shift_jis",
16
+ });
17
+ expect(decodeHtml("text/html", body).text).to.contain(TITLE);
18
+ });
19
+ it("lets the body charset replace a non-UTF-8 header charset", () => {
20
+ const body = page('<meta charset="shift_jis">', "shift_jis");
21
+ const { text, charset } = decodeHtml("text/html; charset=MS932", body);
22
+ expect(charset).to.equal("shift_jis");
23
+ expect(text).to.contain(TITLE);
24
+ });
25
+ it("trusts a UTF-8 header and ignores a conflicting meta tag", () => {
26
+ const body = page('<meta charset="shift_jis">', "utf-8");
27
+ const { text, charset } = decodeHtml("text/html; charset=UTF-8", body);
28
+ expect(charset).to.equal("utf-8");
29
+ expect(text).to.contain(TITLE);
30
+ });
31
+ it("falls back to UTF-8 without any hint", () => {
32
+ expect(detectHtmlCharset("text/html", page("", "utf-8"))).to.equal("utf-8");
33
+ expect(detectHtmlCharset("", page("", "utf-8"))).to.equal("utf-8");
34
+ });
35
+ it("keeps a non-UTF-8 header charset when the body has no hint", () => {
36
+ expect(detectHtmlCharset("text/html; charset=EUC-JP", page("", "euc-jp"))).to.equal("euc-jp");
37
+ });
38
+ it("reads the http-equiv form of the meta tag", () => {
39
+ const body = page('<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">', "shift_jis");
40
+ const { text, charset } = decodeHtml("text/html", body);
41
+ expect(charset).to.equal("shift_jis");
42
+ expect(text).to.contain(TITLE);
43
+ });
44
+ it("reads a BOM", () => {
45
+ const body = Buffer.concat([
46
+ Buffer.from([0xef, 0xbb, 0xbf]),
47
+ page("", "utf-8"),
48
+ ]);
49
+ expect(detectHtmlCharset("text/html; charset=iso-8859-1", body)).to.equal("utf-8");
50
+ });
51
+ it("does not sniff a non-HTML content type", () => {
52
+ const body = page('<meta charset="shift_jis">', "shift_jis");
53
+ expect(detectHtmlCharset("application/xhtml+xml", body)).to.equal("utf-8");
54
+ });
55
+ it("throws UnsupportedEncodingError for a charset iconv-lite does not know", () => {
56
+ expect(() => decodeHtml("text/html; charset=x-unknown", page("", "utf-8"))).to.throw(UnsupportedEncodingError, "x-unknown");
57
+ });
58
+ });
@@ -0,0 +1,19 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import iconv from "iconv-lite";
4
+ import { VirtualOrestesApp } from "../../virtual-orestes-app/index.js";
5
+ describe("VirtualOrestesApp", () => {
6
+ it("decodes the Speed Kit document by its <meta charset>", async () => {
7
+ const html = '<html><head><meta charset="shift_jis"><title>掃除機</title></head></html>';
8
+ const body = iconv.encode(html, "shift_jis");
9
+ // getResponseText needs no collaborator, so the constructor is skipped.
10
+ const app = Object.create(VirtualOrestesApp.prototype);
11
+ const result = await app["getResponseText"](new Response(new Uint8Array(body), {
12
+ headers: { "content-type": "text/html" },
13
+ }));
14
+ expect(result).to.deep.equal({
15
+ responseContent: html,
16
+ textEncoding: "shift_jis",
17
+ });
18
+ });
19
+ });
@@ -2,9 +2,8 @@ import { BaqendResponse } from "../browser/baqend-response.js";
2
2
  import { ABORT_SERVER_TIMING, ABORT_STATUS_CODE, isDocumentHandlerAbort, isLocalPrerendererUnreachable, LOCAL_PRERENDERER_PORT, prerenderFailure, } from "../../document-handler-runtime/document-handler-abort.js";
3
3
  import { isBlockedOriginStatus, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "../../origin-request/index.js";
4
4
  import { safe } from "../../../helpers/safe.js";
5
- import { META_CONTENT_TYPE_REGEX, } from "../onboarding-model.js";
6
5
  import iconv from "iconv-lite";
7
- import { UnsupportedEncodingError } from "../error/unsupported-encoding-error.js";
6
+ import { decodeHtml } from "../html-charset.js";
8
7
  import { removeContentSecurityMetaTag, rewriteInstallResource, } from "../html-rewrites.js";
9
8
  // for none encodable chars use a space, so the broken char is invisible.
10
9
  // We are using the same trick in our node.js server
@@ -191,30 +190,8 @@ export class VirtualOrestesApp {
191
190
  return baqendResponse;
192
191
  }
193
192
  async getResponseText(response) {
194
- const buffer = Buffer.from(await response.arrayBuffer());
195
- const contentType = response?.headers.get("content-type") || "";
196
- const charsetRegex = /charset=([^\s"(),./:;<=>?@[\]]*)/i;
197
- const encodingFromHeader = (contentType && charsetRegex.test(contentType)
198
- ? charsetRegex.exec(contentType)[1]
199
- : null) ?? "utf-8";
200
- if (!iconv.encodingExists(encodingFromHeader)) {
201
- throw new UnsupportedEncodingError(`encoding is not supported by cli: "${encodingFromHeader}"`);
202
- }
203
- let encodedBody = iconv.decode(buffer, encodingFromHeader);
204
- const metaSelector = META_CONTENT_TYPE_REGEX;
205
- const encodingFromBodyMeta = metaSelector.test(encodedBody)
206
- ? metaSelector.exec(encodedBody)[1]
207
- : null;
208
- if (encodingFromBodyMeta && encodingFromBodyMeta !== encodingFromHeader) {
209
- if (!iconv.encodingExists(encodingFromBodyMeta)) {
210
- throw new UnsupportedEncodingError(`encoding is not supported by cli: "${encodingFromBodyMeta}"`);
211
- }
212
- encodedBody = iconv.decode(buffer, encodingFromBodyMeta);
213
- }
214
- return {
215
- responseContent: encodedBody,
216
- textEncoding: encodingFromBodyMeta ?? encodingFromHeader,
217
- };
193
+ const { text, charset } = decodeHtml(response?.headers.get("content-type") || "", Buffer.from(await response.arrayBuffer()));
194
+ return { responseContent: text, textEncoding: charset };
218
195
  }
219
196
  localCacheErrorKey(url, variation) {
220
197
  return `${url}|${variation}`;
@@ -2113,5 +2113,5 @@
2113
2113
  ]
2114
2114
  }
2115
2115
  },
2116
- "version": "4.50.4"
2116
+ "version": "4.50.5"
2117
2117
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@speedkit/cli",
3
3
  "description": "Speed Kit CLI",
4
- "version": "4.50.4",
4
+ "version": "4.50.5",
5
5
  "author": {
6
6
  "name": "Baqend.com",
7
7
  "email": "info@baqend.com"
@@ -124,7 +124,7 @@
124
124
  "domhandler": "^5.0.3",
125
125
  "domutils": "^3.1.0",
126
126
  "dotenv": "^17.4.2",
127
- "encoding-japanese": "^2.2.0",
127
+ "encoding-sniffer": "^0.2.1",
128
128
  "esbuild": "^0.27.3",
129
129
  "extract-zip": "^2.0.1",
130
130
  "fetch-socks": "^1.3.3",