@theia/core 1.77.0-next.89 → 1.77.0-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/browser/encoding-registry.js +1 -1
- package/lib/browser/encoding-registry.js.map +1 -1
- package/lib/common/encoding-service.d.ts +5 -0
- package/lib/common/encoding-service.d.ts.map +1 -1
- package/lib/common/encoding-service.js +20 -3
- package/lib/common/encoding-service.js.map +1 -1
- package/lib/common/encoding-service.spec.d.ts +2 -0
- package/lib/common/encoding-service.spec.d.ts.map +1 -0
- package/lib/common/encoding-service.spec.js +40 -0
- package/lib/common/encoding-service.spec.js.map +1 -0
- package/package.json +4 -4
- package/src/browser/encoding-registry.ts +1 -1
- package/src/common/encoding-service.spec.ts +40 -0
- package/src/common/encoding-service.ts +24 -3
|
@@ -55,7 +55,7 @@ let EncodingRegistry = class EncodingRegistry {
|
|
|
55
55
|
if (!fileEncoding || !this.encodingService.exists(fileEncoding)) {
|
|
56
56
|
return encodings_1.UTF8; // the default is UTF 8
|
|
57
57
|
}
|
|
58
|
-
return
|
|
58
|
+
return fileEncoding;
|
|
59
59
|
}
|
|
60
60
|
getEncodingOverride(resource) {
|
|
61
61
|
if (this.encodingOverrides && this.encodingOverrides.length) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoding-registry.js","sourceRoot":"","sources":["../../src/browser/encoding-registry.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,yCAAyC;AACzC,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF;;;gGAGgG;;;;AAEhG,gKAAgK;AAEhK,yCAA+C;AAE/C,qDAAkD;AAClD,iEAAgF;AAChF,mDAA2C;AAC3C,iEAA6D;AAUtD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAAtB;QAEgB,sBAAiB,GAAuB,EAAE,CAAC;IAyDlE,CAAC;IAjDG,gBAAgB,CAAC,QAA0B;QACvC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,uBAAU,CAAC,MAAM,CAAC,GAAG,EAAE;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC5C,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,sBAAsB,CAAC,QAAa,EAAE,iBAA0B;QAC5D,IAAI,YAAoB,CAAC;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,QAAQ,EAAE,CAAC;YACX,YAAY,GAAG,QAAQ,CAAC,CAAC,gCAAgC;QAC7D,CAAC;aAAM,IAAI,iBAAiB,EAAE,CAAC;YAC3B,YAAY,GAAG,iBAAiB,CAAC,CAAC,kCAAkC;QACxE,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,SAAS,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9D,OAAO,gBAAI,CAAC,CAAC,uBAAuB;QACxC,CAAC;QAED,OAAO,
|
|
1
|
+
{"version":3,"file":"encoding-registry.js","sourceRoot":"","sources":["../../src/browser/encoding-registry.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,yCAAyC;AACzC,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF;;;gGAGgG;;;;AAEhG,gKAAgK;AAEhK,yCAA+C;AAE/C,qDAAkD;AAClD,iEAAgF;AAChF,mDAA2C;AAC3C,iEAA6D;AAUtD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAAtB;QAEgB,sBAAiB,GAAuB,EAAE,CAAC;IAyDlE,CAAC;IAjDG,gBAAgB,CAAC,QAA0B;QACvC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,uBAAU,CAAC,MAAM,CAAC,GAAG,EAAE;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC5C,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,sBAAsB,CAAC,QAAa,EAAE,iBAA0B;QAC5D,IAAI,YAAoB,CAAC;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,QAAQ,EAAE,CAAC;YACX,YAAY,GAAG,QAAQ,CAAC,CAAC,gCAAgC;QAC7D,CAAC;aAAM,IAAI,iBAAiB,EAAE,CAAC;YAC3B,YAAY,GAAG,iBAAiB,CAAC,CAAC,kCAAkC;QACxE,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,SAAS,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9D,OAAO,gBAAI,CAAC,CAAC,uBAAuB;QACxC,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAES,mBAAmB,CAAC,QAAa;QACvC,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAC1D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC/D,OAAO,QAAQ,CAAC,QAAQ,CAAC;gBAC7B,CAAC;gBAED,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;oBACvE,OAAO,QAAQ,CAAC,QAAQ,CAAC;gBAC7B,CAAC;gBAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACzD,OAAO,QAAQ,CAAC,QAAQ,CAAC;gBAC7B,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CAEJ,CAAA;AA3DY,4CAAgB;AAKN;IADlB,IAAA,kBAAM,EAAC,kCAAe,CAAC;;qDACwB;AAG7B;IADlB,IAAA,kBAAM,EAAC,kCAAe,CAAC;sCACY,kCAAe;yDAAC;2BAR3C,gBAAgB;IAD5B,IAAA,sBAAU,GAAE;GACA,gBAAgB,CA2D5B"}
|
|
@@ -24,6 +24,11 @@ export declare class EncodingService {
|
|
|
24
24
|
decode(value: BinaryBuffer, encoding?: string): string;
|
|
25
25
|
exists(encoding: string): boolean;
|
|
26
26
|
toIconvEncoding(encoding?: string): string;
|
|
27
|
+
/**
|
|
28
|
+
* Transforms jschardet encoding names into encoding ids. It's a port of VS Code's `toIconvLiteEncoding` from:
|
|
29
|
+
* https://github.com/microsoft/vscode/blob/a7e1fc08139956da66cc7fb6333a390b06b61876/src/vs/workbench/services/textfile/common/encoding.ts#L404
|
|
30
|
+
*/
|
|
31
|
+
protected fromJschardetEncoding(encodingName: string): string;
|
|
27
32
|
toResourceEncoding(encoding: string, options: {
|
|
28
33
|
overwriteEncoding?: boolean;
|
|
29
34
|
read: (length: number) => Promise<Uint8Array>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoding-service.d.ts","sourceRoot":"","sources":["../../src/common/encoding-service.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC1F,OAAO,EAAQ,aAAa,EAAE,OAAO,EAAE,OAAO,EAAsC,MAAM,aAAa,CAAC;AACxG,OAAO,EAAsB,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"encoding-service.d.ts","sourceRoot":"","sources":["../../src/common/encoding-service.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAC1F,OAAO,EAAQ,aAAa,EAAE,OAAO,EAAE,OAAO,EAAsC,MAAM,aAAa,CAAC;AACxG,OAAO,EAAsB,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAmBxE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,OAAO,CAAA;CACxB;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,4BAA4B,CAAC,EAAE,MAAM,CAAC;IAEtC,iBAAiB,CAAC,gBAAgB,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC5E;AACD,MAAM,WAAW,kBAAkB;IAC/B,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IAC/B,QAAQ,EAAE,gBAAgB,CAAC;CAC9B;AAED,qBACa,eAAe;IAExB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,YAAY;IAW/D,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAMtD,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAKjC,eAAe,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAO1C;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM;IAOvD,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE;QAChD,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;KAChD,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwBvB,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,iBAAiB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoEhG,SAAS,CAAC,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,aAAa,GAAG,OAAO,OAAO,GAAG,OAAO,OAAO,GAAG,SAAS;cAgC9H,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAqBlF,YAAY,CAAC,MAAM,EAAE,0BAA0B,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAkF3G,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,oBAAoB,CAAC;IACxH,YAAY,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,oBAAoB,GAAG,SAAS,CAAC;CA0DxI"}
|
|
@@ -39,6 +39,10 @@ const AUTO_ENCODING_GUESS_MAX_BYTES = 512 * 128; // set an upper limit for the n
|
|
|
39
39
|
// - UTF-16: we have our own detection logic for UTF-16
|
|
40
40
|
// - UTF-32: we do not support this encoding in VSCode
|
|
41
41
|
const IGNORE_ENCODINGS = ['ascii', 'utf-16', 'utf-32'];
|
|
42
|
+
const JSCHARDET_TO_ICONV_ENCODINGS = {
|
|
43
|
+
'ibm866': 'cp866',
|
|
44
|
+
'big5': 'cp950'
|
|
45
|
+
};
|
|
42
46
|
let EncodingService = class EncodingService {
|
|
43
47
|
encode(value, options) {
|
|
44
48
|
let encoding = options?.encoding;
|
|
@@ -65,6 +69,15 @@ let EncodingService = class EncodingService {
|
|
|
65
69
|
}
|
|
66
70
|
return encoding;
|
|
67
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Transforms jschardet encoding names into encoding ids. It's a port of VS Code's `toIconvLiteEncoding` from:
|
|
74
|
+
* https://github.com/microsoft/vscode/blob/a7e1fc08139956da66cc7fb6333a390b06b61876/src/vs/workbench/services/textfile/common/encoding.ts#L404
|
|
75
|
+
*/
|
|
76
|
+
fromJschardetEncoding(encodingName) {
|
|
77
|
+
const normalizedEncodingName = encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
|
|
78
|
+
const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName];
|
|
79
|
+
return mapped || normalizedEncodingName;
|
|
80
|
+
}
|
|
68
81
|
async toResourceEncoding(encoding, options) {
|
|
69
82
|
// Some encodings come with a BOM automatically
|
|
70
83
|
if (encoding === encodings_1.UTF16be || encoding === encodings_1.UTF16le || encoding === encodings_1.UTF8_with_bom) {
|
|
@@ -139,7 +152,7 @@ let EncodingService = class EncodingService {
|
|
|
139
152
|
}
|
|
140
153
|
// Auto guess encoding if configured
|
|
141
154
|
if (autoGuessEncoding && !seemsBinary && !encoding && buffer) {
|
|
142
|
-
const guessedEncoding = await this.guessEncodingByBuffer(buffer.
|
|
155
|
+
const guessedEncoding = await this.guessEncodingByBuffer(buffer.subarray(0, bytesRead));
|
|
143
156
|
return {
|
|
144
157
|
seemsBinary: false,
|
|
145
158
|
encoding: guessedEncoding
|
|
@@ -173,7 +186,11 @@ let EncodingService = class EncodingService {
|
|
|
173
186
|
}
|
|
174
187
|
async guessEncodingByBuffer(buffer) {
|
|
175
188
|
const jschardet = await Promise.resolve().then(() => tslib_1.__importStar(require('jschardet')));
|
|
176
|
-
|
|
189
|
+
// jschardet only converts its input to a binary string if it is an instance of the global `Buffer`,
|
|
190
|
+
// which does not exist in the frontend, so we convert it ourselves: https://github.com/aadsm/jschardet/blob/v2.3.0/src/index.js#L49-L53
|
|
191
|
+
const guessed = jschardet.detect(
|
|
192
|
+
// Use `subarray` to ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
|
|
193
|
+
buffer.subarray(0, AUTO_ENCODING_GUESS_MAX_BYTES).toString('binary'));
|
|
177
194
|
if (!guessed || !guessed.encoding) {
|
|
178
195
|
return undefined;
|
|
179
196
|
}
|
|
@@ -181,7 +198,7 @@ let EncodingService = class EncodingService {
|
|
|
181
198
|
if (0 <= IGNORE_ENCODINGS.indexOf(enc)) {
|
|
182
199
|
return undefined; // see comment above why we ignore some encodings
|
|
183
200
|
}
|
|
184
|
-
return this.
|
|
201
|
+
return this.fromJschardetEncoding(guessed.encoding);
|
|
185
202
|
}
|
|
186
203
|
decodeStream(source, options) {
|
|
187
204
|
const minBytesRequiredForDetection = options.minBytesRequiredForDetection ?? options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoding-service.js","sourceRoot":"","sources":["../../src/common/encoding-service.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,yCAAyC;AACzC,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF;;;gGAGgG;AAChG,mJAAmJ;;;;AAEnJ,oCAAoC;AAEpC,0DAAoC;AACpC,+CAAsC;AACtC,yCAAuC;AACvC,qCAA0F;AAC1F,2CAAwG;AACxG,qCAAwE;AAExE,MAAM,kCAAkC,GAAG,GAAG,CAAC,CAAG,wEAAwE;AAC1H,MAAM,2BAA2B,GAAG,GAAG,CAAC,CAAU,wEAAwE;AAC1H,MAAM,6BAA6B,GAAG,GAAG,GAAG,CAAC,CAAC,CAAI,wEAAwE;AAC1H,MAAM,6BAA6B,GAAG,GAAG,GAAG,GAAG,CAAC,CAAE,qEAAqE;AAEvH,sEAAsE;AACtE,iFAAiF;AACjF,iFAAiF;AACjF,uDAAuD;AACvD,sDAAsD;AACtD,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"encoding-service.js","sourceRoot":"","sources":["../../src/common/encoding-service.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,yCAAyC;AACzC,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF;;;gGAGgG;AAChG,mJAAmJ;;;;AAEnJ,oCAAoC;AAEpC,0DAAoC;AACpC,+CAAsC;AACtC,yCAAuC;AACvC,qCAA0F;AAC1F,2CAAwG;AACxG,qCAAwE;AAExE,MAAM,kCAAkC,GAAG,GAAG,CAAC,CAAG,wEAAwE;AAC1H,MAAM,2BAA2B,GAAG,GAAG,CAAC,CAAU,wEAAwE;AAC1H,MAAM,6BAA6B,GAAG,GAAG,GAAG,CAAC,CAAC,CAAI,wEAAwE;AAC1H,MAAM,6BAA6B,GAAG,GAAG,GAAG,GAAG,CAAC,CAAE,qEAAqE;AAEvH,sEAAsE;AACtE,iFAAiF;AACjF,iFAAiF;AACjF,uDAAuD;AACvD,sDAAsD;AACtD,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAEvD,MAAM,4BAA4B,GAA+B;IAC7D,QAAQ,EAAE,OAAO;IACjB,MAAM,EAAE,OAAO;CAClB,CAAC;AAwBK,IAAM,eAAe,GAArB,MAAM,eAAe;IAExB,MAAM,CAAC,KAAa,EAAE,OAA0B;QAC5C,IAAI,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,gBAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,qBAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,OAAO,qBAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,KAAmB,EAAE,QAAiB;QACzC,MAAM,MAAM,GAAG,qBAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,CAAC,QAAgB;QACnB,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,eAAe,CAAC,QAAiB;QAC7B,IAAI,QAAQ,KAAK,yBAAa,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,OAAO,gBAAI,CAAC,CAAC,8EAA8E;QAC/F,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACO,qBAAqB,CAAC,YAAoB;QAChD,MAAM,sBAAsB,GAAG,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACvF,MAAM,MAAM,GAAG,4BAA4B,CAAC,sBAAsB,CAAC,CAAC;QAEpE,OAAO,MAAM,IAAI,sBAAsB,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,QAAgB,EAAE,OAG1C;QACG,+CAA+C;QAC/C,IAAI,QAAQ,KAAK,mBAAO,IAAI,QAAQ,KAAK,mBAAO,IAAI,QAAQ,KAAK,yBAAa,EAAE,CAAC;YAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAED,4DAA4D;QAC5D,qDAAqD;QACrD,MAAM,iBAAiB,GAAG,OAAO,EAAE,iBAAiB,CAAC;QACrD,IAAI,CAAC,iBAAiB,IAAI,QAAQ,KAAK,gBAAI,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACD,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,oBAAQ,CAAC,MAAM,CAAC,CAAC;gBACnD,IAAI,IAAI,CAAC,6BAA6B,CAAC,qBAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,yBAAa,EAAE,CAAC;oBAC/F,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;gBACtC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,gCAAgC;YACpC,CAAC;QACL,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAkB,EAAE,iBAA2B;QAChE,MAAM,MAAM,GAAG,qBAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,wDAAwD;QACxD,IAAI,QAAQ,GAAG,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAErE,2DAA2D;QAC3D,8DAA8D;QAC9D,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,QAAQ,KAAK,mBAAO,IAAI,QAAQ,KAAK,mBAAO,IAAI,MAAM,EAAE,CAAC;YACzD,IAAI,cAAc,GAAG,IAAI,CAAC,CAAC,iBAAiB;YAC5C,IAAI,cAAc,GAAG,IAAI,CAAC,CAAC,iBAAiB;YAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAE7B,2EAA2E;YAC3E,4EAA4E;YAC5E,mEAAmE;YACnE,2EAA2E;YAC3E,4EAA4E;YAC5E,gDAAgD;YAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,IAAI,CAAC,GAAG,kCAAkC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3E,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,6CAA6C;gBAC7E,MAAM,UAAU,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBAE/C,IAAI,UAAU,EAAE,CAAC;oBACb,gBAAgB,GAAG,IAAI,CAAC;gBAC5B,CAAC;gBAED,mCAAmC;gBACnC,IAAI,cAAc,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,EAAE,CAAC;oBACzE,cAAc,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBAED,mCAAmC;gBACnC,IAAI,cAAc,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzE,cAAc,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBAED,2EAA2E;gBAC3E,IAAI,UAAU,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;oBACnD,MAAM;gBACV,CAAC;YACL,CAAC;YAED,iCAAiC;YACjC,IAAI,gBAAgB,EAAE,CAAC;gBACnB,IAAI,cAAc,EAAE,CAAC;oBACjB,QAAQ,GAAG,mBAAO,CAAC;gBACvB,CAAC;qBAAM,IAAI,cAAc,EAAE,CAAC;oBACxB,QAAQ,GAAG,mBAAO,CAAC;gBACvB,CAAC;qBAAM,CAAC;oBACJ,WAAW,GAAG,IAAI,CAAC;gBACvB,CAAC;YACL,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,IAAI,iBAAiB,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;YAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;YACxF,OAAO;gBACH,WAAW,EAAE,KAAK;gBAClB,QAAQ,EAAE,eAAe;aAC5B,CAAC;QACN,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IACrC,CAAC;IAES,6BAA6B,CAAC,MAAc,EAAE,SAAiB;QACrE,IAAI,CAAC,MAAM,IAAI,SAAS,GAAG,uBAAW,CAAC,MAAM,EAAE,CAAC;YAC5C,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAE/B,YAAY;QACZ,IAAI,EAAE,KAAK,uBAAW,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,uBAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,OAAO,mBAAO,CAAC;QACnB,CAAC;QAED,YAAY;QACZ,IAAI,EAAE,KAAK,uBAAW,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,uBAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,OAAO,mBAAO,CAAC;QACnB,CAAC;QAED,IAAI,SAAS,GAAG,oBAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAE/B,QAAQ;QACR,IAAI,EAAE,KAAK,oBAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,oBAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,oBAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,yBAAa,CAAC;QACzB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAES,KAAK,CAAC,qBAAqB,CAAC,MAAc;QAChD,MAAM,SAAS,GAAG,gEAAa,WAAW,GAAC,CAAC;QAE5C,oGAAoG;QACpG,wIAAwI;QACxI,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM;QAC5B,4GAA4G;QAC5G,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,6BAA6B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACvE,CAAC;QACF,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAChC,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,CAAC,iDAAiD;QACvE,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED,YAAY,CAAC,MAAkC,EAAE,OAA4B;QACzE,MAAM,4BAA4B,GAAG,OAAO,CAAC,4BAA4B,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,2BAA2B,CAAC;QAEjK,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,MAAM,MAAM,GAAG,IAAA,2BAAkB,EAAS,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAEvE,MAAM,cAAc,GAAmB,EAAE,CAAC;YAC1C,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,IAAI,OAAO,GAAoC,SAAS,CAAC;YAEzD,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;gBAC7B,IAAI,CAAC;oBAED,8BAA8B;oBAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,qBAAY,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;oBAEvG,0CAA0C;oBAC1C,QAAQ,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBAEvE,oCAAoC;oBACpC,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;oBACpE,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAM,CAAC,IAAI,CAAC,qBAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBACvF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAEtB,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;oBAC1B,aAAa,GAAG,CAAC,CAAC;oBAElB,uEAAuE;oBACvE,OAAO,CAAC;wBACJ,MAAM,EAAE,MAAM;wBACd,QAAQ;qBACX,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClB,CAAC;YACL,CAAC,CAAC;YAEF,kCAAkC;YAClC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAEjD,cAAc;YACd,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;gBAE5B,kDAAkD;gBAClD,IAAI,OAAO,EAAE,CAAC;oBACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,qBAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC3D,CAAC;qBAAM,CAAC;oBACJ,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC3B,aAAa,IAAI,KAAK,CAAC,UAAU,CAAC;oBAElC,6DAA6D;oBAC7D,IAAI,aAAa,IAAI,4BAA4B,EAAE,CAAC;wBAEhD,+CAA+C;wBAC/C,MAAM,CAAC,KAAK,EAAE,CAAC;wBAEf,MAAM,aAAa,EAAE,CAAC;wBAEtB,8CAA8C;wBAC9C,4CAA4C;wBAC5C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;oBACtC,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,aAAa;YACb,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;gBAExB,oDAAoD;gBACpD,oDAAoD;gBACpD,2CAA2C;gBAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;oBACX,MAAM,aAAa,EAAE,CAAC;gBAC1B,CAAC;gBAED,oDAAoD;gBACpD,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAID,KAAK,CAAC,YAAY,CAAC,KAA4C,EAAE,OAA0B;QACvF,IAAI,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,gBAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;gBAChE,qBAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,6BAAoB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAClF,CAAC;QAED,KAAK,GAAG,KAAK,IAAI,EAAE,CAAC;QACpB,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAChF,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEvD,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,IAAI,GAAG,KAAK,CAAC;QAEjB,OAAO;YACH,IAAI;gBACA,IAAI,IAAI,EAAE,CAAC;oBACP,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC5B,IAAI,GAAG,IAAI,CAAC;oBAEZ,0DAA0D;oBAC1D,4DAA4D;oBAC5D,iDAAiD;oBACjD,IAAI,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC;wBAC1B,QAAQ,QAAQ,EAAE,CAAC;4BACf,KAAK,gBAAI,CAAC;4BACV,KAAK,yBAAa;gCACd,OAAO,qBAAY,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,oBAAQ,CAAC,CAAC,CAAC;4BACxD,KAAK,mBAAO;gCACR,OAAO,qBAAY,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,uBAAW,CAAC,CAAC,CAAC;4BAC3D,KAAK,mBAAO;gCACR,OAAO,qBAAY,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,uBAAW,CAAC,CAAC,CAAC;wBAC/D,CAAC;oBACL,CAAC;oBAED,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBAChC,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACpC,YAAY,GAAG,IAAI,CAAC;wBACpB,OAAO,qBAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxC,CAAC;oBAED,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,YAAY,GAAG,IAAI,CAAC;gBAEpB,OAAO,qBAAY,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACnD,CAAC;SACJ,CAAC;IACN,CAAC;CAEJ,CAAA;AA3UY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,sBAAU,GAAE;GACA,eAAe,CA2U3B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encoding-service.spec.d.ts","sourceRoot":"","sources":["../../src/common/encoding-service.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// *****************************************************************************
|
|
3
|
+
// Copyright (C) 2026 EclipseSource and others.
|
|
4
|
+
//
|
|
5
|
+
// This program and the accompanying materials are made available under the
|
|
6
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
7
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
8
|
+
//
|
|
9
|
+
// This Source Code may also be made available under the following Secondary
|
|
10
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
11
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
12
|
+
// with the GNU Classpath Exception which is available at
|
|
13
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
14
|
+
//
|
|
15
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
16
|
+
// *****************************************************************************
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
const chai_1 = require("chai");
|
|
19
|
+
const encoding_service_1 = require("./encoding-service");
|
|
20
|
+
describe('EncodingService', () => {
|
|
21
|
+
describe('guessEncodingByBuffer', () => {
|
|
22
|
+
it('guesses the encoding when there is no global Buffer, as in the frontend bundle', async () => {
|
|
23
|
+
class TestEncodingService extends encoding_service_1.EncodingService {
|
|
24
|
+
guessEncodingByBuffer(data) {
|
|
25
|
+
return super.guessEncodingByBuffer(data);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const buffer = Buffer.from('Grüße aus München\n'.repeat(20));
|
|
29
|
+
const globalBuffer = globalThis.Buffer;
|
|
30
|
+
globalThis.Buffer = undefined;
|
|
31
|
+
try {
|
|
32
|
+
(0, chai_1.expect)(await new TestEncodingService().guessEncodingByBuffer(buffer)).to.equal('utf8');
|
|
33
|
+
}
|
|
34
|
+
finally {
|
|
35
|
+
globalThis.Buffer = globalBuffer;
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
//# sourceMappingURL=encoding-service.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encoding-service.spec.js","sourceRoot":"","sources":["../../src/common/encoding-service.spec.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,+CAA+C;AAC/C,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;;AAEhF,+BAA8B;AAC9B,yDAAqD;AAErD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAE7B,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAEnC,EAAE,CAAC,gFAAgF,EAAE,KAAK,IAAI,EAAE;YAC5F,MAAM,mBAAoB,SAAQ,kCAAe;gBACpC,qBAAqB,CAAC,IAAY;oBACvC,OAAO,KAAK,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;gBAC7C,CAAC;aACJ;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;YACtC,UAAmC,CAAC,MAAM,GAAG,SAAS,CAAC;YACxD,IAAI,CAAC;gBACD,IAAA,aAAM,EAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3F,CAAC;oBAAS,CAAC;gBACP,UAAU,CAAC,MAAM,GAAG,YAAY,CAAC;YACrC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@theia/core",
|
|
3
|
-
"version": "1.77.0-next.
|
|
3
|
+
"version": "1.77.0-next.94+3490801c0",
|
|
4
4
|
"description": "Theia is a cloud & desktop IDE framework implemented in TypeScript.",
|
|
5
5
|
"main": "lib/common/index.js",
|
|
6
6
|
"typings": "lib/common/index.d.ts",
|
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
"@lumino/virtualdom": "^2.0.5",
|
|
18
18
|
"@lumino/widgets": "2.7.5",
|
|
19
19
|
"@parcel/watcher": "^2.6.0",
|
|
20
|
-
"@theia/application-package": "1.77.0-next.
|
|
21
|
-
"@theia/request": "1.77.0-next.
|
|
20
|
+
"@theia/application-package": "1.77.0-next.94+3490801c0",
|
|
21
|
+
"@theia/request": "1.77.0-next.94+3490801c0",
|
|
22
22
|
"@types/body-parser": "^1.19.6",
|
|
23
23
|
"@types/express": "^4.17.25",
|
|
24
24
|
"@types/fs-extra": "^4.0.15",
|
|
@@ -233,5 +233,5 @@
|
|
|
233
233
|
"nyc": {
|
|
234
234
|
"extends": "../../configs/nyc.json"
|
|
235
235
|
},
|
|
236
|
-
"gitHead": "
|
|
236
|
+
"gitHead": "3490801c093594a70b74434088bc82705389bb31"
|
|
237
237
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// *****************************************************************************
|
|
2
|
+
// Copyright (C) 2026 EclipseSource and others.
|
|
3
|
+
//
|
|
4
|
+
// This program and the accompanying materials are made available under the
|
|
5
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
6
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
7
|
+
//
|
|
8
|
+
// This Source Code may also be made available under the following Secondary
|
|
9
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
10
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
11
|
+
// with the GNU Classpath Exception which is available at
|
|
12
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
13
|
+
//
|
|
14
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
15
|
+
// *****************************************************************************
|
|
16
|
+
|
|
17
|
+
import { expect } from 'chai';
|
|
18
|
+
import { EncodingService } from './encoding-service';
|
|
19
|
+
|
|
20
|
+
describe('EncodingService', () => {
|
|
21
|
+
|
|
22
|
+
describe('guessEncodingByBuffer', () => {
|
|
23
|
+
|
|
24
|
+
it('guesses the encoding when there is no global Buffer, as in the frontend bundle', async () => {
|
|
25
|
+
class TestEncodingService extends EncodingService {
|
|
26
|
+
override guessEncodingByBuffer(data: Buffer): Promise<string | undefined> {
|
|
27
|
+
return super.guessEncodingByBuffer(data);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const buffer = Buffer.from('Grüße aus München\n'.repeat(20));
|
|
31
|
+
const globalBuffer = globalThis.Buffer;
|
|
32
|
+
(globalThis as { Buffer?: unknown }).Buffer = undefined;
|
|
33
|
+
try {
|
|
34
|
+
expect(await new TestEncodingService().guessEncodingByBuffer(buffer)).to.equal('utf8');
|
|
35
|
+
} finally {
|
|
36
|
+
globalThis.Buffer = globalBuffer;
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
});
|
|
@@ -40,6 +40,11 @@ const AUTO_ENCODING_GUESS_MAX_BYTES = 512 * 128; // set an upper limit for the
|
|
|
40
40
|
// - UTF-32: we do not support this encoding in VSCode
|
|
41
41
|
const IGNORE_ENCODINGS = ['ascii', 'utf-16', 'utf-32'];
|
|
42
42
|
|
|
43
|
+
const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = {
|
|
44
|
+
'ibm866': 'cp866',
|
|
45
|
+
'big5': 'cp950'
|
|
46
|
+
};
|
|
47
|
+
|
|
43
48
|
export interface ResourceEncoding {
|
|
44
49
|
encoding: string
|
|
45
50
|
hasBOM: boolean
|
|
@@ -93,6 +98,17 @@ export class EncodingService {
|
|
|
93
98
|
return encoding;
|
|
94
99
|
}
|
|
95
100
|
|
|
101
|
+
/**
|
|
102
|
+
* Transforms jschardet encoding names into encoding ids. It's a port of VS Code's `toIconvLiteEncoding` from:
|
|
103
|
+
* https://github.com/microsoft/vscode/blob/a7e1fc08139956da66cc7fb6333a390b06b61876/src/vs/workbench/services/textfile/common/encoding.ts#L404
|
|
104
|
+
*/
|
|
105
|
+
protected fromJschardetEncoding(encodingName: string): string {
|
|
106
|
+
const normalizedEncodingName = encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
|
|
107
|
+
const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName];
|
|
108
|
+
|
|
109
|
+
return mapped || normalizedEncodingName;
|
|
110
|
+
}
|
|
111
|
+
|
|
96
112
|
async toResourceEncoding(encoding: string, options: {
|
|
97
113
|
overwriteEncoding?: boolean,
|
|
98
114
|
read: (length: number) => Promise<Uint8Array>
|
|
@@ -178,7 +194,7 @@ export class EncodingService {
|
|
|
178
194
|
|
|
179
195
|
// Auto guess encoding if configured
|
|
180
196
|
if (autoGuessEncoding && !seemsBinary && !encoding && buffer) {
|
|
181
|
-
const guessedEncoding = await this.guessEncodingByBuffer(buffer.
|
|
197
|
+
const guessedEncoding = await this.guessEncodingByBuffer(buffer.subarray(0, bytesRead));
|
|
182
198
|
return {
|
|
183
199
|
seemsBinary: false,
|
|
184
200
|
encoding: guessedEncoding
|
|
@@ -223,7 +239,12 @@ export class EncodingService {
|
|
|
223
239
|
protected async guessEncodingByBuffer(buffer: Buffer): Promise<string | undefined> {
|
|
224
240
|
const jschardet = await import('jschardet');
|
|
225
241
|
|
|
226
|
-
|
|
242
|
+
// jschardet only converts its input to a binary string if it is an instance of the global `Buffer`,
|
|
243
|
+
// which does not exist in the frontend, so we convert it ourselves: https://github.com/aadsm/jschardet/blob/v2.3.0/src/index.js#L49-L53
|
|
244
|
+
const guessed = jschardet.detect(
|
|
245
|
+
// Use `subarray` to ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
|
|
246
|
+
buffer.subarray(0, AUTO_ENCODING_GUESS_MAX_BYTES).toString('binary')
|
|
247
|
+
);
|
|
227
248
|
if (!guessed || !guessed.encoding) {
|
|
228
249
|
return undefined;
|
|
229
250
|
}
|
|
@@ -233,7 +254,7 @@ export class EncodingService {
|
|
|
233
254
|
return undefined; // see comment above why we ignore some encodings
|
|
234
255
|
}
|
|
235
256
|
|
|
236
|
-
return this.
|
|
257
|
+
return this.fromJschardetEncoding(guessed.encoding);
|
|
237
258
|
}
|
|
238
259
|
|
|
239
260
|
decodeStream(source: BinaryBufferReadableStream, options: DecodeStreamOptions): Promise<DecodeStreamResult> {
|