echogarden 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/lexicons/heteronyms.en.json +1 -1
- package/data/lexicons/words.en.json +32 -0
- package/dist/api/Synthesis.js +2 -2
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/encodings/Ascii.d.ts +1 -3
- package/dist/encodings/Ascii.d.ts.map +1 -1
- package/dist/encodings/Ascii.js +13 -9
- package/dist/encodings/Ascii.js.map +1 -1
- package/dist/encodings/Base64.d.ts.map +1 -1
- package/dist/encodings/Base64.js +9 -0
- package/dist/encodings/Base64.js.map +1 -1
- package/dist/encodings/Hex.d.ts.map +1 -1
- package/dist/encodings/Hex.js +11 -2
- package/dist/encodings/Hex.js.map +1 -1
- package/dist/encodings/LEB128.d.ts +1 -15
- package/dist/encodings/LEB128.d.ts.map +1 -1
- package/dist/encodings/LEB128.js +1 -247
- package/dist/encodings/LEB128.js.map +1 -1
- package/dist/encodings/Utf16.d.ts +1 -3
- package/dist/encodings/Utf16.d.ts.map +1 -1
- package/dist/encodings/Utf16.js +13 -9
- package/dist/encodings/Utf16.js.map +1 -1
- package/dist/encodings/Utf32.d.ts.map +1 -1
- package/dist/encodings/Utf32.js +6 -0
- package/dist/encodings/Utf32.js.map +1 -1
- package/dist/encodings/Utf8.d.ts +9 -5
- package/dist/encodings/Utf8.d.ts.map +1 -1
- package/dist/encodings/Utf8.js +36 -78
- package/dist/encodings/Utf8.js.map +1 -1
- package/dist/nlp/Segmentation.d.ts.map +1 -1
- package/dist/nlp/Segmentation.js +4 -1
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/nlp/TextNormalizer.js +8 -1
- package/dist/nlp/TextNormalizer.js.map +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js.map +1 -1
- package/dist/synthesis/EspeakTTS.js +1 -1
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/synthesis/MicrosoftEdgeTTS.js +1 -1
- package/dist/synthesis/MicrosoftEdgeTTS.js.map +1 -1
- package/dist/utilities/FileSystem.d.ts +12 -7
- package/dist/utilities/FileSystem.d.ts.map +1 -1
- package/dist/utilities/FileSystem.js +196 -79
- package/dist/utilities/FileSystem.js.map +1 -1
- package/dist/utilities/FileWriter.d.ts +1 -0
- package/dist/utilities/FileWriter.d.ts.map +1 -1
- package/dist/utilities/FileWriter.js +8 -1
- package/dist/utilities/FileWriter.js.map +1 -1
- package/dist/utilities/ObjectUtilities.d.ts.map +1 -1
- package/dist/utilities/ObjectUtilities.js +15 -13
- package/dist/utilities/ObjectUtilities.js.map +1 -1
- package/dist/utilities/PackageManager.js +2 -2
- package/dist/utilities/PackageManager.js.map +1 -1
- package/dist/utilities/PathUtilities.d.ts +1 -0
- package/dist/utilities/PathUtilities.d.ts.map +1 -1
- package/dist/utilities/PathUtilities.js +18 -0
- package/dist/utilities/PathUtilities.js.map +1 -1
- package/dist/utilities/StringUtilities.d.ts.map +1 -1
- package/dist/utilities/StringUtilities.js.map +1 -1
- package/package.json +8 -7
- package/src/api/Synthesis.ts +2 -2
- package/src/encodings/Ascii.ts +14 -10
- package/src/encodings/Base64.ts +9 -0
- package/src/encodings/Hex.ts +11 -2
- package/src/encodings/Utf16.ts +14 -10
- package/src/encodings/Utf32.ts +6 -0
- package/src/encodings/Utf8.ts +45 -82
- package/src/nlp/Segmentation.ts +5 -1
- package/src/nlp/TextNormalizer.ts +12 -1
- package/src/synthesis/AzureCognitiveServicesTTS.ts +1 -1
- package/src/synthesis/EspeakTTS.ts +1 -1
- package/src/synthesis/MicrosoftEdgeTTS.ts +1 -1
- package/src/utilities/FileSystem.ts +224 -85
- package/src/utilities/FileWriter.ts +11 -1
- package/src/utilities/ObjectUtilities.ts +19 -15
- package/src/utilities/PackageManager.ts +2 -2
- package/src/utilities/PathUtilities.ts +19 -0
- package/src/utilities/StringUtilities.ts +0 -2
- package/src/encodings/HtmlEscape.ts +0 -39
- package/src/encodings/LEB128.ts +0 -362
- package/src/utilities/StringBuilder.ts +0 -45
package/dist/encodings/LEB128.js
CHANGED
|
@@ -1,248 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
import { logToStderr } from "../utilities/Utilities.js";
|
|
3
|
-
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
4
|
-
// Encode unsigned integer
|
|
5
|
-
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
6
|
-
export function encodeUnsignedInt(value, outEncodedData) {
|
|
7
|
-
if (value < 0) {
|
|
8
|
-
throw new Error(`The negative value ${value} can't be encoded as an unsigned LEB128 integer.`);
|
|
9
|
-
}
|
|
10
|
-
if (typeof value === 'number') {
|
|
11
|
-
if (value < (2 ** 31)) {
|
|
12
|
-
return encodeUnsignedInt31(value, outEncodedData);
|
|
13
|
-
}
|
|
14
|
-
else {
|
|
15
|
-
return encodeUnsignedBigInt(BigInt(value), outEncodedData);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
else {
|
|
19
|
-
return encodeUnsignedBigInt(value, outEncodedData);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
function encodeUnsignedInt31(value, outEncodedData) {
|
|
23
|
-
value = value >>> 0;
|
|
24
|
-
if (value < (2 ** 7)) {
|
|
25
|
-
outEncodedData.add(value);
|
|
26
|
-
}
|
|
27
|
-
else if (value < (2 ** 14)) {
|
|
28
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, value >>> 7);
|
|
29
|
-
}
|
|
30
|
-
else if (value < (2 ** 21)) {
|
|
31
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >>> 7) & 0b01111111) | 0b10000000, value >>> 14);
|
|
32
|
-
}
|
|
33
|
-
else if (value < (2 ** 28)) {
|
|
34
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >>> 7) & 0b01111111) | 0b10000000, ((value >>> 14) & 0b01111111) | 0b10000000, value >>> 21);
|
|
35
|
-
}
|
|
36
|
-
else {
|
|
37
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >>> 7) & 0b01111111) | 0b10000000, ((value >>> 14) & 0b01111111) | 0b10000000, ((value >>> 21) & 0b01111111) | 0b10000000, value >>> 28);
|
|
38
|
-
}
|
|
39
|
-
return outEncodedData;
|
|
40
|
-
}
|
|
41
|
-
function encodeUnsignedBigInt(value, outEncodedData) {
|
|
42
|
-
while (true) {
|
|
43
|
-
const lowest7Bits = Number(value & 127n);
|
|
44
|
-
value = value >> 7n;
|
|
45
|
-
if (value === 0n) {
|
|
46
|
-
outEncodedData.add(lowest7Bits);
|
|
47
|
-
return outEncodedData;
|
|
48
|
-
}
|
|
49
|
-
else {
|
|
50
|
-
outEncodedData.add(lowest7Bits | 0b10000000);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
55
|
-
// Decode unsigned integer
|
|
56
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
57
|
-
export function decodeUnsignedInt31Fast(encodedData, readOffset) {
|
|
58
|
-
const byte0 = encodedData[readOffset++];
|
|
59
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
60
|
-
const decodedValue = byte0;
|
|
61
|
-
return { decodedValue, readOffset };
|
|
62
|
-
}
|
|
63
|
-
const byte1 = encodedData[readOffset++];
|
|
64
|
-
if ((byte1 & 0b10000000) === 0) {
|
|
65
|
-
const decodedValue = (byte0 & 0b01111111) |
|
|
66
|
-
(byte1 & 0b01111111) << 7;
|
|
67
|
-
return { decodedValue, readOffset };
|
|
68
|
-
}
|
|
69
|
-
const byte2 = encodedData[readOffset++];
|
|
70
|
-
if ((byte2 & 0b10000000) === 0) {
|
|
71
|
-
const decodedValue = (byte0 & 0b01111111) |
|
|
72
|
-
(byte1 & 0b01111111) << 7 |
|
|
73
|
-
(byte2 & 0b01111111) << 14;
|
|
74
|
-
return { decodedValue, readOffset };
|
|
75
|
-
}
|
|
76
|
-
const byte3 = encodedData[readOffset++];
|
|
77
|
-
if ((byte3 & 0b10000000) === 0) {
|
|
78
|
-
const decodedValue = (byte0 & 0b01111111) |
|
|
79
|
-
(byte1 & 0b01111111) << 7 |
|
|
80
|
-
(byte2 & 0b01111111) << 14 |
|
|
81
|
-
(byte3 & 0b01111111) << 21;
|
|
82
|
-
return { decodedValue, readOffset };
|
|
83
|
-
}
|
|
84
|
-
const byte4 = encodedData[readOffset++];
|
|
85
|
-
if ((byte4 & 0b10000000) === 0) {
|
|
86
|
-
const decodedValue = (byte0 & 0b01111111) |
|
|
87
|
-
(byte1 & 0b01111111) << 7 |
|
|
88
|
-
(byte2 & 0b01111111) << 14 |
|
|
89
|
-
(byte3 & 0b01111111) << 21 |
|
|
90
|
-
(byte4 & 0b01111111) << 28;
|
|
91
|
-
return { decodedValue, readOffset };
|
|
92
|
-
}
|
|
93
|
-
if (readOffset >= encodedData.length) {
|
|
94
|
-
throw new Error(`Invalid LEB128 data. Last encoded byte sequence is truncated.`);
|
|
95
|
-
}
|
|
96
|
-
else {
|
|
97
|
-
throw new Error(`LEB128 sequence can't be decoded. Encoded byte sequence represents a value that extends beyond the range of a unsigned 32 bit integer.`);
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
101
|
-
// Encode signed integer
|
|
102
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
103
|
-
export function encodeSignedInt32(value, outEncodedData) {
|
|
104
|
-
if (value < -(2 ** 31) || value > (2 ** 31)) {
|
|
105
|
-
throw new Error('Value must be between -(2^31) and 2^31 - 1');
|
|
106
|
-
}
|
|
107
|
-
while (true) {
|
|
108
|
-
const lowest7Bits = value & 0b01111111;
|
|
109
|
-
value >>= 7;
|
|
110
|
-
if ((value === 0 && (lowest7Bits & 0b01000000) === 0) ||
|
|
111
|
-
(value === -1 && (lowest7Bits & 0b01000000) !== 0)) {
|
|
112
|
-
outEncodedData.add(lowest7Bits);
|
|
113
|
-
return outEncodedData;
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
outEncodedData.add(lowest7Bits | 0b10000000);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
export function encodeSignedInt32Fast(value, outEncodedData) {
|
|
121
|
-
const absValue = Math.abs(value | 0);
|
|
122
|
-
if (absValue < (2 ** 6)) {
|
|
123
|
-
outEncodedData.add((value & 0b01111111));
|
|
124
|
-
}
|
|
125
|
-
else if (absValue < (2 ** 13)) {
|
|
126
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, (value >> 7) & 0b01111111);
|
|
127
|
-
}
|
|
128
|
-
else if (absValue < (2 ** 20)) {
|
|
129
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >> 7) & 0b01111111) | 0b10000000, (value >> 14) & 0b01111111);
|
|
130
|
-
}
|
|
131
|
-
else if (absValue < (2 ** 27)) {
|
|
132
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >> 7) & 0b01111111) | 0b10000000, ((value >> 14) & 0b01111111) | 0b10000000, (value >> 21) & 0b01111111);
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
outEncodedData.addMany((value & 0b01111111) | 0b10000000, ((value >> 7) & 0b01111111) | 0b10000000, ((value >> 14) & 0b01111111) | 0b10000000, ((value >> 21) & 0b01111111) | 0b10000000, (value >> 28) & 0b01111111);
|
|
136
|
-
}
|
|
137
|
-
return outEncodedData;
|
|
138
|
-
}
|
|
139
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
140
|
-
// Decode signed integer
|
|
141
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
142
|
-
export function decodeSignedInt32(encodedData, readOffset) {
|
|
143
|
-
let decodedValue = 0;
|
|
144
|
-
let shiftAmount = 0;
|
|
145
|
-
while (true) {
|
|
146
|
-
const encodedByte = encodedData[readOffset++];
|
|
147
|
-
const lowest7Bits = encodedByte & 0b01111111;
|
|
148
|
-
decodedValue |= lowest7Bits << shiftAmount;
|
|
149
|
-
// If 8th bit is 0, then this is the last byte in the sequence
|
|
150
|
-
if ((encodedByte & 0b10000000) === 0) {
|
|
151
|
-
// If 7th bit is 1, then the value is negative
|
|
152
|
-
if ((encodedByte & 0b01000000) !== 0) {
|
|
153
|
-
// If the value should be negative
|
|
154
|
-
// Ensure that the value is encoded as a negative number by
|
|
155
|
-
// setting all higher bits to 1
|
|
156
|
-
decodedValue |= -1 << Math.min(shiftAmount + 7, 31);
|
|
157
|
-
}
|
|
158
|
-
return { decodedValue, readOffset };
|
|
159
|
-
}
|
|
160
|
-
if (readOffset === encodedData.length) {
|
|
161
|
-
throw new Error(`Invalid LEB128 data. Last encoded byte sequence is truncated.`);
|
|
162
|
-
}
|
|
163
|
-
shiftAmount += 7;
|
|
164
|
-
if (shiftAmount > 31) {
|
|
165
|
-
throw new Error(`LEB128 sequence can't be decoded. Byte sequence extends beyond the range of a signed 32 bit integer.`);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
export function decodeSignedInt32Fast(encodedData, readOffset) {
|
|
170
|
-
const byte0 = encodedData[readOffset++];
|
|
171
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
172
|
-
let decodedValue = byte0;
|
|
173
|
-
if ((byte0 & 0b01000000) !== 0) {
|
|
174
|
-
decodedValue |= -1 << 7;
|
|
175
|
-
}
|
|
176
|
-
return { decodedValue, readOffset };
|
|
177
|
-
}
|
|
178
|
-
const byte1 = encodedData[readOffset++];
|
|
179
|
-
if ((byte1 & 0b10000000) === 0) {
|
|
180
|
-
let decodedValue = (byte0 & 0b01111111) |
|
|
181
|
-
(byte1 & 0b01111111) << 7;
|
|
182
|
-
if ((byte1 & 0b01000000) !== 0) {
|
|
183
|
-
decodedValue |= -1 << 14;
|
|
184
|
-
}
|
|
185
|
-
return { decodedValue, readOffset };
|
|
186
|
-
}
|
|
187
|
-
const byte2 = encodedData[readOffset++];
|
|
188
|
-
if ((byte2 & 0b10000000) === 0) {
|
|
189
|
-
let decodedValue = (byte0 & 0b01111111) |
|
|
190
|
-
(byte1 & 0b01111111) << 7 |
|
|
191
|
-
(byte2 & 0b01111111) << 14;
|
|
192
|
-
if ((byte2 & 0b01000000) !== 0) {
|
|
193
|
-
decodedValue |= -1 << 21;
|
|
194
|
-
}
|
|
195
|
-
return { decodedValue, readOffset };
|
|
196
|
-
}
|
|
197
|
-
const byte3 = encodedData[readOffset++];
|
|
198
|
-
if ((byte3 & 0b10000000) === 0) {
|
|
199
|
-
let decodedValue = (byte0 & 0b01111111) |
|
|
200
|
-
(byte1 & 0b01111111) << 7 |
|
|
201
|
-
(byte2 & 0b01111111) << 14 |
|
|
202
|
-
(byte3 & 0b01111111) << 21;
|
|
203
|
-
if ((byte3 & 0b01000000) !== 0) {
|
|
204
|
-
decodedValue |= -1 << 28;
|
|
205
|
-
}
|
|
206
|
-
return { decodedValue, readOffset };
|
|
207
|
-
}
|
|
208
|
-
const byte4 = encodedData[readOffset++];
|
|
209
|
-
if ((byte4 & 0b10000000) === 0) {
|
|
210
|
-
let decodedValue = (byte0 & 0b01111111) |
|
|
211
|
-
(byte1 & 0b01111111) << 7 |
|
|
212
|
-
(byte2 & 0b01111111) << 14 |
|
|
213
|
-
(byte3 & 0b01111111) << 21 |
|
|
214
|
-
(byte4 & 0b01111111) << 28;
|
|
215
|
-
if ((byte4 & 0b01000000) !== 0) {
|
|
216
|
-
decodedValue |= -1 << 31;
|
|
217
|
-
}
|
|
218
|
-
return { decodedValue, readOffset };
|
|
219
|
-
}
|
|
220
|
-
if (readOffset >= encodedData.length) {
|
|
221
|
-
throw new Error(`Invalid LEB128 data. Last encoded byte sequence is truncated.`);
|
|
222
|
-
}
|
|
223
|
-
else {
|
|
224
|
-
throw new Error(`LEB128 sequence can't be decoded. Encoded byte sequence represents a value that extends beyond the range of a signed 32 bit integer.`);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
228
|
-
// Tests
|
|
229
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
230
|
-
export function testLeb128Signed() {
|
|
231
|
-
const encodedBytes = createDynamicUint8Array();
|
|
232
|
-
function runTest(testValue) {
|
|
233
|
-
encodedBytes.clear();
|
|
234
|
-
encodeSignedInt32Fast(testValue, encodedBytes);
|
|
235
|
-
const { decodedValue } = decodeSignedInt32Fast(encodedBytes.elements, 0);
|
|
236
|
-
if (decodedValue !== testValue) {
|
|
237
|
-
throw new Error(`Expected ${testValue} but got ${decodedValue}`);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
for (let i = -(2 ** 26); i < 2 ** 26; i++) {
|
|
241
|
-
if (i % 1000000 === 0) {
|
|
242
|
-
logToStderr(i);
|
|
243
|
-
}
|
|
244
|
-
runTest(i);
|
|
245
|
-
}
|
|
246
|
-
const x = 1;
|
|
247
|
-
}
|
|
1
|
+
export {};
|
|
248
2
|
//# sourceMappingURL=LEB128.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LEB128.js","sourceRoot":"","sources":["../../src/encodings/LEB128.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"LEB128.js","sourceRoot":"","sources":["../../src/encodings/LEB128.ts"],"names":[],"mappings":""}
|
|
@@ -3,9 +3,7 @@ export declare function encodeUtf16(text: string): Uint16Array<ArrayBuffer>;
|
|
|
3
3
|
export declare function encodeUtf16Into(text: string, resultBuffer: Uint16Array): EncodeIntoResult;
|
|
4
4
|
export declare function decodeUtf16(encodedString: Uint16Array): string;
|
|
5
5
|
export declare class ChunkedUtf16Decoder {
|
|
6
|
-
private str;
|
|
7
6
|
private readonly textDecoder;
|
|
8
|
-
writeChunk(chunk: Uint16Array):
|
|
9
|
-
toString(): string;
|
|
7
|
+
writeChunk(chunk: Uint16Array): string;
|
|
10
8
|
}
|
|
11
9
|
//# sourceMappingURL=Utf16.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf16.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf16.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"Utf16.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf16.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAK3D,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,4BAMvC;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,GAAG,gBAAgB,CAYzF;AAKD,wBAAgB,WAAW,CAAC,aAAa,EAAE,WAAW,UAerD;AAED,qBAAa,mBAAmB;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAE1D,UAAU,CAAC,KAAK,EAAE,WAAW,UAI5B;CACD"}
|
package/dist/encodings/Utf16.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// UTF-16 Encoding
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
1
4
|
export function encodeUtf16(text) {
|
|
2
5
|
const resultArray = new Uint16Array(text.length);
|
|
3
6
|
const { written } = encodeUtf16Into(text, resultArray);
|
|
@@ -13,24 +16,25 @@ export function encodeUtf16Into(text, resultBuffer) {
|
|
|
13
16
|
}
|
|
14
17
|
return { read: len, written: len };
|
|
15
18
|
}
|
|
19
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
20
|
+
// UTF-16 Decoding
|
|
21
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
16
22
|
export function decodeUtf16(encodedString) {
|
|
17
23
|
const maxChunkLength = 2 ** 24;
|
|
18
|
-
const
|
|
24
|
+
const chunkedUtf16Decoder = new ChunkedUtf16Decoder();
|
|
25
|
+
let resultString = '';
|
|
19
26
|
for (let offset = 0; offset < encodedString.length; offset += maxChunkLength) {
|
|
20
|
-
const
|
|
21
|
-
|
|
27
|
+
const utf16Chunk = encodedString.subarray(offset, offset + maxChunkLength);
|
|
28
|
+
const stringChunk = chunkedUtf16Decoder.writeChunk(utf16Chunk);
|
|
29
|
+
resultString += stringChunk;
|
|
22
30
|
}
|
|
23
|
-
return
|
|
31
|
+
return resultString;
|
|
24
32
|
}
|
|
25
33
|
export class ChunkedUtf16Decoder {
|
|
26
|
-
str = '';
|
|
27
34
|
textDecoder = new TextDecoder('utf-16le');
|
|
28
35
|
writeChunk(chunk) {
|
|
29
36
|
const decodedChunk = this.textDecoder.decode(chunk, { stream: true });
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
toString() {
|
|
33
|
-
return this.str;
|
|
37
|
+
return decodedChunk;
|
|
34
38
|
}
|
|
35
39
|
}
|
|
36
40
|
//# sourceMappingURL=Utf16.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf16.js","sourceRoot":"","sources":["../../src/encodings/Utf16.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEhD,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAEtD,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,YAAyB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;IAEvB,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;IAC7E,CAAC;IAED,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC;QACzD,YAAY,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;IACvD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,aAA0B;IACrD,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAA;IAE9B,MAAM,
|
|
1
|
+
{"version":3,"file":"Utf16.js","sourceRoot":"","sources":["../../src/encodings/Utf16.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEhD,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAEtD,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,YAAyB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;IAEvB,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;IAC7E,CAAC;IAED,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC;QACzD,YAAY,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;IACvD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;AACnC,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,aAA0B;IACrD,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAA;IAE9B,MAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAA;IAErD,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,IAAI,cAAc,EAAE,CAAC;QAC9E,MAAM,UAAU,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,CAAA;QAC1E,MAAM,WAAW,GAAG,mBAAmB,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;QAE9D,YAAY,IAAI,WAAW,CAAA;IAC5B,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACd,WAAW,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAA;IAE1D,UAAU,CAAC,KAAkB;QAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAErE,OAAO,YAAY,CAAA;IACpB,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf32.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf32.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"Utf32.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf32.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAK3D,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,4BAMvC;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,GAAG,gBAAgB,CAuBzF;AAKD,wBAAgB,WAAW,CAAC,aAAa,EAAE,WAAW,UAQrD"}
|
package/dist/encodings/Utf32.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// UTF-32 Encoding
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
1
4
|
export function encodeUtf32(text) {
|
|
2
5
|
const resultArray = new Uint32Array(text.length * 2);
|
|
3
6
|
const { written } = encodeUtf32Into(text, resultArray);
|
|
@@ -22,6 +25,9 @@ export function encodeUtf32Into(text, resultBuffer) {
|
|
|
22
25
|
}
|
|
23
26
|
return { read: readOffset, written: writeOffset };
|
|
24
27
|
}
|
|
28
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
29
|
+
// UTF-32 Decoding
|
|
30
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
25
31
|
export function decodeUtf32(encodedString) {
|
|
26
32
|
let result = '';
|
|
27
33
|
for (let i = 0; i < encodedString.length; i++) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf32.js","sourceRoot":"","sources":["../../src/encodings/Utf32.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEpD,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAEtD,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,YAAyB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;IAEvB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,OAAO,UAAU,GAAG,GAAG,IAAI,WAAW,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QAE9C,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,uCAAuC,UAAU,EAAE,CAAC,CAAA;QACrE,CAAC;QAED,YAAY,CAAC,WAAW,EAAE,CAAC,GAAG,SAAS,CAAA;QAEvC,IAAI,SAAS,GAAG,MAAM,EAAE,CAAC;YACxB,UAAU,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,UAAU,IAAI,CAAC,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,aAA0B;IACrD,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IACjD,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC"}
|
|
1
|
+
{"version":3,"file":"Utf32.js","sourceRoot":"","sources":["../../src/encodings/Utf32.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEpD,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAEtD,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,YAAyB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;IAEvB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,OAAO,UAAU,GAAG,GAAG,IAAI,WAAW,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QAE9C,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,uCAAuC,UAAU,EAAE,CAAC,CAAA;QACrE,CAAC;QAED,YAAY,CAAC,WAAW,EAAE,CAAC,GAAG,SAAS,CAAA;QAEvC,IAAI,SAAS,GAAG,MAAM,EAAE,CAAC;YACxB,UAAU,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,UAAU,IAAI,CAAC,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;AAClD,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,aAA0B;IACrD,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IACjD,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC"}
|
package/dist/encodings/Utf8.d.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
import { EncodeIntoResult } from
|
|
1
|
+
import { EncodeIntoResult } from './TextEncodingsCommon.js';
|
|
2
2
|
export declare function encodeUtf8(text: string): Uint8Array<ArrayBuffer>;
|
|
3
3
|
export declare function encodeUtf8Into(text: string, outputArray: Uint8Array): EncodeIntoResult;
|
|
4
|
-
export declare
|
|
4
|
+
export declare class ChunkedUtf8Encoder {
|
|
5
|
+
private readonly textEncoder;
|
|
6
|
+
private pendingHighSurrogate;
|
|
7
|
+
writeChunk(stringChunk: string): Uint8Array;
|
|
8
|
+
finalize(): Uint8Array;
|
|
9
|
+
}
|
|
10
|
+
export declare function decodeUtf8(utf8Bytes: Uint8Array): string;
|
|
5
11
|
export declare class ChunkedUtf8Decoder {
|
|
6
|
-
private str;
|
|
7
12
|
private readonly textDecoder;
|
|
8
|
-
writeChunk(chunk: Uint8Array):
|
|
9
|
-
toString(): string;
|
|
13
|
+
writeChunk(chunk: Uint8Array): string;
|
|
10
14
|
}
|
|
11
15
|
//# sourceMappingURL=Utf8.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf8.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf8.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"Utf8.d.ts","sourceRoot":"","sources":["../../src/encodings/Utf8.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAK3D,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,2BAItC;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,GAAG,gBAAgB,CAMtF;AAED,qBAAa,kBAAkB;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,oBAAoB,CAAK;IAEjC,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,UAAU,CAgB1C;IAED,QAAQ,IAAI,UAAU,CAMrB;CACD;AAKD,wBAAgB,UAAU,CAAC,SAAS,EAAE,UAAU,UAe/C;AAED,qBAAa,kBAAkB;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IAEvD,UAAU,CAAC,KAAK,EAAE,UAAU,UAI3B;CACD"}
|
package/dist/encodings/Utf8.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// UTF-8 Encoding
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
1
4
|
export function encodeUtf8(text) {
|
|
2
5
|
const textEncoder = new TextEncoder();
|
|
3
6
|
return textEncoder.encode(text);
|
|
@@ -7,91 +10,46 @@ export function encodeUtf8Into(text, outputArray) {
|
|
|
7
10
|
const result = textEncoder.encodeInto(text, outputArray);
|
|
8
11
|
return result;
|
|
9
12
|
}
|
|
10
|
-
export
|
|
13
|
+
export class ChunkedUtf8Encoder {
|
|
14
|
+
textEncoder = new TextEncoder();
|
|
15
|
+
pendingHighSurrogate = '';
|
|
16
|
+
writeChunk(stringChunk) {
|
|
17
|
+
if (this.pendingHighSurrogate !== '') {
|
|
18
|
+
stringChunk = this.pendingHighSurrogate + stringChunk;
|
|
19
|
+
this.pendingHighSurrogate = '';
|
|
20
|
+
}
|
|
21
|
+
const lastCodeUnit = stringChunk.charCodeAt(stringChunk.length - 1);
|
|
22
|
+
if (lastCodeUnit >= 0xD800 && lastCodeUnit <= 0xDBFF) {
|
|
23
|
+
this.pendingHighSurrogate = stringChunk[stringChunk.length - 1];
|
|
24
|
+
stringChunk = stringChunk.substring(0, stringChunk.length - 1);
|
|
25
|
+
}
|
|
26
|
+
return this.textEncoder.encode(stringChunk);
|
|
27
|
+
}
|
|
28
|
+
finalize() {
|
|
29
|
+
const result = this.textEncoder.encode(this.pendingHighSurrogate);
|
|
30
|
+
this.pendingHighSurrogate = '';
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
35
|
+
// UTF-8 Decoding
|
|
36
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
37
|
+
export function decodeUtf8(utf8Bytes) {
|
|
11
38
|
const maxChunkLength = 2 ** 24;
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
39
|
+
const chunkedUtf8Decoder = new ChunkedUtf8Decoder();
|
|
40
|
+
let resultString = '';
|
|
41
|
+
for (let offset = 0; offset < utf8Bytes.length; offset += maxChunkLength) {
|
|
42
|
+
const utf8Chunk = utf8Bytes.subarray(offset, offset + maxChunkLength);
|
|
43
|
+
const stringChunk = chunkedUtf8Decoder.writeChunk(utf8Chunk);
|
|
44
|
+
resultString += stringChunk;
|
|
16
45
|
}
|
|
17
|
-
return
|
|
46
|
+
return resultString;
|
|
18
47
|
}
|
|
19
48
|
export class ChunkedUtf8Decoder {
|
|
20
|
-
str = '';
|
|
21
49
|
textDecoder = new TextDecoder('utf-8');
|
|
22
50
|
writeChunk(chunk) {
|
|
23
51
|
const decodedChunk = this.textDecoder.decode(chunk, { stream: true });
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
toString() {
|
|
27
|
-
return this.str;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
//////////////////////////////////////////////////////////////////////////////////////////////
|
|
31
|
-
// Pure JavaScript implementations
|
|
32
|
-
//////////////////////////////////////////////////////////////////////////////////////////////
|
|
33
|
-
function encodeUtf8Into_JS(str, outputArray) {
|
|
34
|
-
let readOffset = 0;
|
|
35
|
-
let writeOffset = 0;
|
|
36
|
-
while (readOffset < str.length) {
|
|
37
|
-
const charCode = str.codePointAt(readOffset++);
|
|
38
|
-
if (charCode <= 0x7f) {
|
|
39
|
-
outputArray[writeOffset++] = charCode;
|
|
40
|
-
}
|
|
41
|
-
else if (charCode <= 0x7ff) {
|
|
42
|
-
outputArray[writeOffset++] = 0xc0 | (charCode >>> 6);
|
|
43
|
-
outputArray[writeOffset++] = 0x80 | (charCode & 63);
|
|
44
|
-
}
|
|
45
|
-
else if (charCode <= 0xffff) {
|
|
46
|
-
outputArray[writeOffset++] = 0xe0 | (charCode >>> 12);
|
|
47
|
-
outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63);
|
|
48
|
-
outputArray[writeOffset++] = 0x80 | (charCode & 63);
|
|
49
|
-
}
|
|
50
|
-
else if (charCode <= 0x10ffff) {
|
|
51
|
-
outputArray[writeOffset++] = 0xf0 | (charCode >>> 18);
|
|
52
|
-
outputArray[writeOffset++] = 0x80 | ((charCode >>> 12) & 63);
|
|
53
|
-
outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63);
|
|
54
|
-
outputArray[writeOffset++] = 0x80 | (charCode & 63);
|
|
55
|
-
readOffset++;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
return { read: str.length, written: writeOffset };
|
|
59
|
-
}
|
|
60
|
-
function decodeUtf8_JS(utf8Bytes) {
|
|
61
|
-
let decodedString = '';
|
|
62
|
-
let readOffset = 0;
|
|
63
|
-
while (readOffset < utf8Bytes.length) {
|
|
64
|
-
const leadByte = utf8Bytes[readOffset++];
|
|
65
|
-
let outputCodePoint;
|
|
66
|
-
if (leadByte >>> 7 === 0) {
|
|
67
|
-
outputCodePoint = leadByte;
|
|
68
|
-
}
|
|
69
|
-
else if (leadByte >>> 5 === 6) {
|
|
70
|
-
outputCodePoint =
|
|
71
|
-
(leadByte & 31) << 6 |
|
|
72
|
-
(utf8Bytes[readOffset++] & 63);
|
|
73
|
-
}
|
|
74
|
-
else if (leadByte >>> 4 === 14) {
|
|
75
|
-
outputCodePoint =
|
|
76
|
-
(leadByte & 15) << 12 |
|
|
77
|
-
(utf8Bytes[readOffset++] & 63) << 6 |
|
|
78
|
-
(utf8Bytes[readOffset++] & 63);
|
|
79
|
-
}
|
|
80
|
-
else if (leadByte >>> 3 === 30) {
|
|
81
|
-
outputCodePoint =
|
|
82
|
-
(leadByte & 7) << 18 |
|
|
83
|
-
(utf8Bytes[readOffset++] & 63) << 12 |
|
|
84
|
-
(utf8Bytes[readOffset++] & 63) << 6 |
|
|
85
|
-
(utf8Bytes[readOffset++] & 63);
|
|
86
|
-
}
|
|
87
|
-
else {
|
|
88
|
-
throw new Error(`Invalid UTF-8 stream: An invalid lead byte value encountered at position ${readOffset}`);
|
|
89
|
-
}
|
|
90
|
-
decodedString += String.fromCodePoint(outputCodePoint);
|
|
91
|
-
}
|
|
92
|
-
if (readOffset > utf8Bytes.length) {
|
|
93
|
-
throw new Error(`UTF-8 decoding failed. Byte sequence is truncated.`);
|
|
52
|
+
return decodedChunk;
|
|
94
53
|
}
|
|
95
|
-
return decodedString;
|
|
96
54
|
}
|
|
97
55
|
//# sourceMappingURL=Utf8.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Utf8.js","sourceRoot":"","sources":["../../src/encodings/Utf8.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,UAAU,CAAC,IAAY;IACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,WAAuB;IACnE,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAExD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,
|
|
1
|
+
{"version":3,"file":"Utf8.js","sourceRoot":"","sources":["../../src/encodings/Utf8.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAC9E,MAAM,UAAU,UAAU,CAAC,IAAY;IACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,WAAuB;IACnE,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAExD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,OAAO,kBAAkB;IACb,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAExC,oBAAoB,GAAG,EAAE,CAAA;IAEjC,UAAU,CAAC,WAAmB;QAC7B,IAAI,IAAI,CAAC,oBAAoB,KAAK,EAAE,EAAE,CAAC;YACtC,WAAW,GAAG,IAAI,CAAC,oBAAoB,GAAG,WAAW,CAAA;YAErD,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAA;QAC/B,CAAC;QAED,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAEnE,IAAI,YAAY,IAAI,MAAM,IAAI,YAAY,IAAI,MAAM,EAAE,CAAC;YACtD,IAAI,CAAC,oBAAoB,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAE/D,WAAW,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC/D,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAC5C,CAAC;IAED,QAAQ;QACP,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QAEjE,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAA;QAE9B,OAAO,MAAM,CAAA;IACd,CAAC;CACD;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAC9E,MAAM,UAAU,UAAU,CAAC,SAAqB;IAC/C,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAA;IAE9B,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,EAAE,CAAA;IAEnD,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,IAAI,cAAc,EAAE,CAAC;QAC1E,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,CAAA;QACrE,MAAM,WAAW,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QAE5D,YAAY,IAAI,WAAW,CAAA;IAC5B,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,MAAM,OAAO,kBAAkB;IACb,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;IAEvD,UAAU,CAAC,KAAiB;QAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAErE,OAAO,YAAY,CAAA;IACpB,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Segmentation.d.ts","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAI3E,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;
|
|
1
|
+
{"version":3,"file":"Segmentation.d.ts","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAI3E,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAmDjE,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,MAAM,WAEpD;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,WAExC;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,WAEvC;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,WAEhD;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,WAE9C;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,WAE3C;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,WAExC;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,WAE1C;AAKD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,oBAAoB,YAoBvI;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,YAExC;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,gDAQ7D;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,YAAY,CAAC,CA8BzG;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,0BAA0B,EAAE,oBAAoB,UAUvG;AAOD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,UAAO,EAAE,eAAe,UAAO,uBAoEjJ;AAED,wBAAsB,oCAAoC,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,uBA4BxF;AAED,qBAAa,QAAQ;IACpB,OAAO,EAAE,MAAM,EAAE,CAAK;IAEtB,QAAQ,CAAC,mBAAmB,QAAO;IAEnC,IAAI,MAAM,WAA+D;IAEzE,IAAI,IAAI,WAAuF;CAC/F;AAED,qBAAa,MAAM;IAClB,KAAK,EAAE,IAAI,EAAE,CAAK;IAElB,IAAI,MAAM,WAAyD;IAEnE,IAAI,IAAI,WAAiF;IAEzF,IAAI,QAAQ,qBAMX;IAED,IAAI,mBAAmB,YAA+E;CACtG;AAED,qBAAa,IAAI;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,mBAAmB,EAAE,OAAO,CAAA;IAE5B,YAAY,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,OAAO,EAGrD;IAED,IAAI,uBAAuB,YAAmD;IAE9E,IAAI,YAAY,YAAqC;IAErD,IAAI,iBAAiB,YAEpB;IAED,IAAI,MAAM,WAA8B;CACxC;AAED,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAA;AAE9C,qBAAa,QAAQ;IACpB,QAAQ,EAAE,OAAO,EAAE,CAAK;IAExB,IAAI,MAAM,WAAgE;IAE1E,IAAI,IAAI,WAA0F;IAElG,IAAI,OAAO,YAA8B;IAEzC,IAAI,UAAU,YAA2B;IAEzC,IAAI,WAAW,wBAMd;CACD"}
|
package/dist/nlp/Segmentation.js
CHANGED
|
@@ -12,7 +12,10 @@ const includesWordCharacterRegExp = buildRegExp(includesWordCharacterPattern);
|
|
|
12
12
|
//export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
|
|
13
13
|
const includesEmojiSequencePattern = anyOf([unicodeProperty('Emoji_Modifier_Base'), possibly(unicodeProperty('Emoji_Modifier'))], unicodeProperty('Emoji_Presentation'), [unicodeProperty('Emoji'), codepoint('FE0F')]);
|
|
14
14
|
const includesEmojiSequenceRegExp = buildRegExp(includesEmojiSequencePattern);
|
|
15
|
-
const symbolWordsList = [
|
|
15
|
+
const symbolWordsList = [
|
|
16
|
+
'$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷',
|
|
17
|
+
'/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥', '∞'
|
|
18
|
+
];
|
|
16
19
|
const includesSymbolWordPattern = anyOf(...symbolWordsList);
|
|
17
20
|
const includesSymbolWordRegExp = buildRegExp(includesSymbolWordPattern);
|
|
18
21
|
const isAllSymbolWordsPattern = [inputStart, oneOrMore(includesSymbolWordPattern), inputEnd];
|