echogarden 3.0.5 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/lexicons/heteronyms.en.json +2 -2
- package/data/lexicons/words.en.json +120 -0
- package/dist/alignment/SemanticTextAlignment.js +2 -2
- package/dist/alignment/SemanticTextAlignment.js.map +1 -1
- package/dist/api/Alignment.js +2 -2
- package/dist/api/Alignment.js.map +1 -1
- package/dist/api/Synthesis.js +2 -2
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/encodings/Ascii.d.ts +1 -3
- package/dist/encodings/Ascii.d.ts.map +1 -1
- package/dist/encodings/Ascii.js +13 -9
- package/dist/encodings/Ascii.js.map +1 -1
- package/dist/encodings/Base64.d.ts.map +1 -1
- package/dist/encodings/Base64.js +9 -0
- package/dist/encodings/Base64.js.map +1 -1
- package/dist/encodings/Hex.d.ts.map +1 -1
- package/dist/encodings/Hex.js +11 -2
- package/dist/encodings/Hex.js.map +1 -1
- package/dist/encodings/LEB128.d.ts +1 -15
- package/dist/encodings/LEB128.d.ts.map +1 -1
- package/dist/encodings/LEB128.js +1 -247
- package/dist/encodings/LEB128.js.map +1 -1
- package/dist/encodings/Utf16.d.ts +1 -3
- package/dist/encodings/Utf16.d.ts.map +1 -1
- package/dist/encodings/Utf16.js +13 -9
- package/dist/encodings/Utf16.js.map +1 -1
- package/dist/encodings/Utf32.d.ts.map +1 -1
- package/dist/encodings/Utf32.js +6 -0
- package/dist/encodings/Utf32.js.map +1 -1
- package/dist/encodings/Utf8.d.ts +9 -5
- package/dist/encodings/Utf8.d.ts.map +1 -1
- package/dist/encodings/Utf8.js +36 -78
- package/dist/encodings/Utf8.js.map +1 -1
- package/dist/nlp/Segmentation.d.ts +2 -8
- package/dist/nlp/Segmentation.d.ts.map +1 -1
- package/dist/nlp/Segmentation.js +42 -19
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/nlp/TextNormalizer.d.ts.map +1 -1
- package/dist/nlp/TextNormalizer.js +29 -14
- package/dist/nlp/TextNormalizer.js.map +1 -1
- package/dist/recognition/AmazonTranscribeSTT.js +3 -3
- package/dist/recognition/AmazonTranscribeSTT.js.map +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js.map +1 -1
- package/dist/synthesis/EspeakTTS.d.ts.map +1 -1
- package/dist/synthesis/EspeakTTS.js +15 -8
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/synthesis/KokoroTTS.d.ts.map +1 -1
- package/dist/synthesis/KokoroTTS.js +0 -5
- package/dist/synthesis/KokoroTTS.js.map +1 -1
- package/dist/synthesis/MicrosoftEdgeTTS.js +1 -1
- package/dist/synthesis/MicrosoftEdgeTTS.js.map +1 -1
- package/dist/synthesis/VitsTTS.d.ts.map +1 -1
- package/dist/synthesis/VitsTTS.js +0 -5
- package/dist/synthesis/VitsTTS.js.map +1 -1
- package/dist/utilities/FileSystem.d.ts +12 -7
- package/dist/utilities/FileSystem.d.ts.map +1 -1
- package/dist/utilities/FileSystem.js +196 -79
- package/dist/utilities/FileSystem.js.map +1 -1
- package/dist/utilities/FileWriter.d.ts +1 -0
- package/dist/utilities/FileWriter.d.ts.map +1 -1
- package/dist/utilities/FileWriter.js +8 -1
- package/dist/utilities/FileWriter.js.map +1 -1
- package/dist/utilities/ObjectUtilities.d.ts.map +1 -1
- package/dist/utilities/ObjectUtilities.js +15 -13
- package/dist/utilities/ObjectUtilities.js.map +1 -1
- package/dist/utilities/PackageManager.js +2 -2
- package/dist/utilities/PackageManager.js.map +1 -1
- package/dist/utilities/PathUtilities.d.ts +1 -0
- package/dist/utilities/PathUtilities.d.ts.map +1 -1
- package/dist/utilities/PathUtilities.js +18 -0
- package/dist/utilities/PathUtilities.js.map +1 -1
- package/dist/utilities/StringUtilities.d.ts.map +1 -1
- package/dist/utilities/StringUtilities.js.map +1 -1
- package/dist/utilities/Timeline.js +2 -2
- package/dist/utilities/Timeline.js.map +1 -1
- package/dist/utilities/WikipediaReader.js +5 -5
- package/dist/utilities/WikipediaReader.js.map +1 -1
- package/package.json +13 -12
- package/src/alignment/SemanticTextAlignment.ts +2 -2
- package/src/api/Alignment.ts +2 -2
- package/src/api/Synthesis.ts +2 -2
- package/src/encodings/Ascii.ts +14 -10
- package/src/encodings/Base64.ts +9 -0
- package/src/encodings/Hex.ts +11 -2
- package/src/encodings/Utf16.ts +14 -10
- package/src/encodings/Utf32.ts +6 -0
- package/src/encodings/Utf8.ts +45 -82
- package/src/nlp/Segmentation.ts +53 -19
- package/src/nlp/TextNormalizer.ts +40 -16
- package/src/recognition/AmazonTranscribeSTT.ts +3 -3
- package/src/synthesis/AzureCognitiveServicesTTS.ts +1 -1
- package/src/synthesis/EspeakTTS.ts +17 -9
- package/src/synthesis/KokoroTTS.ts +0 -6
- package/src/synthesis/MicrosoftEdgeTTS.ts +1 -1
- package/src/synthesis/VitsTTS.ts +0 -6
- package/src/utilities/FileSystem.ts +224 -85
- package/src/utilities/FileWriter.ts +11 -1
- package/src/utilities/ObjectUtilities.ts +19 -15
- package/src/utilities/PackageManager.ts +2 -2
- package/src/utilities/PathUtilities.ts +19 -0
- package/src/utilities/StringUtilities.ts +0 -2
- package/src/utilities/Timeline.ts +2 -2
- package/src/utilities/WikipediaReader.ts +5 -5
- package/src/encodings/HtmlEscape.ts +0 -39
- package/src/encodings/LEB128.ts +0 -362
- package/src/utilities/StringBuilder.ts +0 -45
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PathUtilities.d.ts","sourceRoot":"","sources":["../../src/utilities/PathUtilities.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,UAAU,CAAA;AAGnC,wBAAgB,gBAAgB,WAI/B;AAED,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,UAE1D;AAED,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,UAQzD;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,UAiB5C;AAED,wBAAgB,QAAQ,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,UAE1C;AAED,wBAAgB,WAAW,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,UAE7C;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,UAEzC;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,eAEzC;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,UAEvC;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,UAEtC;AAED,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,UAE3D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,uBAErC"}
|
|
1
|
+
{"version":3,"file":"PathUtilities.d.ts","sourceRoot":"","sources":["../../src/utilities/PathUtilities.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,UAAU,CAAA;AAGnC,wBAAgB,gBAAgB,WAI/B;AAED,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,UAE1D;AAED,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,UAQzD;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,UAiB5C;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,UAiB5C;AAED,wBAAgB,QAAQ,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,UAE1C;AAED,wBAAgB,WAAW,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,UAE7C;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,UAEzC;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,eAEzC;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,UAEvC;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,UAEtC;AAED,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,UAE3D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,uBAErC"}
|
|
@@ -33,6 +33,24 @@ export function getAppTempDir(appName) {
|
|
|
33
33
|
}
|
|
34
34
|
return tempDir;
|
|
35
35
|
}
|
|
36
|
+
export function getAppDataDir(appName) {
|
|
37
|
+
let dataDir;
|
|
38
|
+
const platform = process.platform;
|
|
39
|
+
const homeDir = os.homedir();
|
|
40
|
+
if (platform == 'win32') {
|
|
41
|
+
dataDir = joinPath(homeDir, 'AppData', 'Local', appName);
|
|
42
|
+
}
|
|
43
|
+
else if (platform == 'darwin') {
|
|
44
|
+
dataDir = joinPath(homeDir, 'Library', 'Application Support', appName);
|
|
45
|
+
}
|
|
46
|
+
else if (platform == 'linux') {
|
|
47
|
+
dataDir = joinPath(homeDir, '.local', 'share', appName);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
throw new Error(`Unsupport platform ${platform}`);
|
|
51
|
+
}
|
|
52
|
+
return dataDir;
|
|
53
|
+
}
|
|
36
54
|
export function joinPath(...paths) {
|
|
37
55
|
return NodePath.join(...paths);
|
|
38
56
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PathUtilities.js","sourceRoot":"","sources":["../../src/utilities/PathUtilities.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,UAAU,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAA;AAE7B,MAAM,UAAU,gBAAgB;IAC/B,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IAE3E,OAAO,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,YAAoB;IAC1D,OAAO,WAAW,CAAC,gBAAgB,EAAE,EAAE,YAAY,CAAC,CAAA;AACrD,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,QAAgB;IACzD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAEpD,IAAI,kBAAkB,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,QAAQ,CAAC,SAAS,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;AAChE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC5C,IAAI,OAAe,CAAA;IAEnB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAA;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAA;IAE5B,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QACzB,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IACjE,CAAC;SAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC1D,CAAC;SAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QAChC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC/C,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAG,KAAe;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAG,KAAe;IAC7C,OAAO,QAAQ,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,CAAA;AAClC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACzC,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACzC,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACtC,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,QAAgB;IAC3D,OAAO,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACrC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAC5B,CAAC"}
|
|
1
|
+
{"version":3,"file":"PathUtilities.js","sourceRoot":"","sources":["../../src/utilities/PathUtilities.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,WAAW,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,UAAU,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAA;AAE7B,MAAM,UAAU,gBAAgB;IAC/B,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IAE3E,OAAO,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,YAAoB;IAC1D,OAAO,WAAW,CAAC,gBAAgB,EAAE,EAAE,YAAY,CAAC,CAAA;AACrD,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,QAAgB;IACzD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAEpD,IAAI,kBAAkB,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,QAAQ,CAAC,SAAS,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;AAChE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC5C,IAAI,OAAe,CAAA;IAEnB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAA;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAA;IAE5B,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QACzB,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IACjE,CAAC;SAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC1D,CAAC;SAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QAChC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC/C,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC5C,IAAI,OAAe,CAAA;IAEnB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAA;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAA;IAE5B,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QACzB,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IACzD,CAAC;SAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,OAAO,CAAC,CAAA;IACvE,CAAC;SAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QAChC,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IACxD,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAG,KAAe;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAG,KAAe;IAC7C,OAAO,QAAQ,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,CAAA;AAClC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACzC,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACzC,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACvC,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACtC,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,QAAgB;IAC3D,OAAO,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACrC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAC5B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StringUtilities.d.ts","sourceRoot":"","sources":["../../src/utilities/StringUtilities.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"StringUtilities.d.ts","sourceRoot":"","sources":["../../src/utilities/StringUtilities.ts"],"names":[],"mappings":"AAAA,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAE9D;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAU7D;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAQ9D;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,EAAE,gBAAgB,SAAM,UAEnI;AAED,wBAAgB,QAAQ,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,EAAE,gBAAgB,SAAM,UAElH;AAED,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,UAU/E;AAED,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,WAAW,SAAM,UAEhF;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM;;;EAmBxC;AAED,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,WAQnD;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,CAyCzF;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;;;EAyClE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,gBAkBnD;AAED,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAcvG;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAQnG"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StringUtilities.js","sourceRoot":"","sources":["../../src/utilities/StringUtilities.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"StringUtilities.js","sourceRoot":"","sources":["../../src/utilities/StringUtilities.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,UAAoB;IAC9D,OAAO,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;AAC1C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,UAAoB;IAC7D,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAEpC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YAChB,OAAO,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAC,CAAA;AACV,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW,EAAE,QAAkB;IAC9D,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAkF,EAAE,gBAAgB,GAAG,GAAG;IACnI,OAAO,GAAG,6BAA6B,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,6BAA6B,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,6BAA6B,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,gBAAgB,GAAG,6BAA6B,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAA;AAClP,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,MAAkE,EAAE,gBAAgB,GAAG,GAAG;IAClH,OAAO,GAAG,6BAA6B,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,6BAA6B,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,gBAAgB,GAAG,6BAA6B,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAA;AAC5L,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,GAAW,EAAE,aAAqB;IAC/E,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAErB,IAAI,WAAW,GAAG,GAAG,GAAG,EAAE,CAAA;IAE1B,OAAO,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAC3C,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAChC,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,OAAiB,EAAE,WAAW,GAAG,GAAG;IAChF,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,GAAG,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC3E,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,MAAM,gBAAgB,GAAa,EAAE,CAAA;IAErC,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,SAAS,IAAI,GAAG,EAAE,CAAC;QAC7B,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAClC,CAAC;QAED,UAAU,IAAI,CAAC,CAAA;IAChB,CAAC;IAED,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAEjC,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,GAAW;IACnD,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YACnC,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,IAAY,EAAE,cAAsB;IAC9E,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAA;IACxE,CAAC;IAED,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAE1C,8EAA8E;IAC9E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,CAAA;IACd,CAAC;IAED,8BAA8B;IAC9B,MAAM,MAAM,GAAa,EAAE,CAAA;IAE3B,gCAAgC;IAChC,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,8BAA8B;IAC9B,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QACvB,qCAAqC;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEjD,iFAAiF;QACjF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,CAAA;QAE1D,wDAAwD;QACxD,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,iEAAiE;IACjE,CAAC;QACA,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE/C,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAA2B;IAClE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEpD,MAAM,yBAAyB,GAAa,CAAC,CAAC,CAAC,CAAA;IAE/C,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,IAAI,cAAc,EAAE,WAAW,EAAE,EAAE,CAAC;QACxE,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,KAAK,IAAI,SAAS,GAAG,WAAW,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC1E,MAAM,cAAc,GAAG,SAAS,GAAG,WAAW,CAAA;YAE9C,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClD,MAAK;YACN,CAAC;YAED,UAAU,IAAI,CAAC,CAAA;QAChB,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAA;QAExB,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC;IAED,IAAI,kBAAkB,GAAG,CAAC,QAAQ,CAAA;IAClC,IAAI,2BAA2B,GAAG,CAAC,QAAQ,CAAA;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,yBAAyB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5D,MAAM,WAAW,GAAG,yBAAyB,CAAC,CAAC,CAAC,CAAA;QAEhD,IAAI,WAAW,GAAG,kBAAkB,EAAE,CAAC;YACtC,kBAAkB,GAAG,WAAW,CAAA;QACjC,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAExC,IAAI,UAAU,GAAG,2BAA2B,EAAE,CAAC;YAC9C,2BAA2B,GAAG,UAAU,CAAA;QACzC,CAAC;IACF,CAAC;IAED,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,CAAA;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IACnD,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAA;IAEnD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE;QAC7B,QAAQ,EAAE,KAAK;QAEf,SAAS,EAAE;YACV,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;YAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;YACnC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACjD,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACjD,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACjD,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACjD,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,EAAE;SAC/D;KACD,CAAC,CAAA;IAEF,OAAO,IAAI,IAAI,EAAE,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,IAAY,EAAE,kBAA0C;IACvG,IAAI,UAAU,GAAG,EAAE,CAAA;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,IAAI,YAAY,CAAA;QAC3B,CAAC;aAAM,CAAC;YACP,UAAU,IAAI,IAAI,CAAA;QACnB,CAAC;IACF,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAAY,EAAE,kBAA0C;IACnG,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,YAAY,CAAA;IACpB,CAAC;SAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { addMissingPunctuationWordsToWordSequence, segmentWordSequence, WordSequence } from '@echogarden/text-segmentation';
|
|
2
|
-
import { applyWhitespaceProcessing,
|
|
2
|
+
import { applyWhitespaceProcessing, includesWordCharacter, isWordOrEmojiOrSymbolWord } from '../nlp/Segmentation.js';
|
|
3
3
|
import { deepClone } from './ObjectUtilities.js';
|
|
4
4
|
import { getUTF32Chars } from './StringUtilities.js';
|
|
5
5
|
import { roundToDigits } from './Utilities.js';
|
|
@@ -150,7 +150,7 @@ function replaceSentenceEndersWithinWordsWithMaskingCharacter(transcript, wordTi
|
|
|
150
150
|
const sentenceEnders = ['.', '。', '?', '？', '!', '！', '|'];
|
|
151
151
|
for (const wordEntry of wordTimeline) {
|
|
152
152
|
const wordText = wordEntry.text;
|
|
153
|
-
if (!
|
|
153
|
+
if (!includesWordCharacter(wordText)) {
|
|
154
154
|
continue;
|
|
155
155
|
}
|
|
156
156
|
let newWordText = '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Timeline.js","sourceRoot":"","sources":["../../src/utilities/Timeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AAE3H,OAAO,EAAE,yBAAyB,EAAE,
|
|
1
|
+
{"version":3,"file":"Timeline.js","sourceRoot":"","sources":["../../src/utilities/Timeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AAE3H,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,yBAAyB,EAAqB,MAAM,wBAAwB,CAAA;AACvI,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAE9C,MAAM,UAAU,uBAAuB,CAAC,cAAwB,EAAE,UAAkB;IACnF,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAE7C,KAAK,MAAM,oBAAoB,IAAI,WAAW,EAAE,CAAC;QAChD,oBAAoB,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;QACzF,oBAAoB,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,OAAO,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;QAErF,IAAI,oBAAoB,CAAC,QAAQ,EAAE,CAAC;YACnC,oBAAoB,CAAC,QAAQ,GAAG,uBAAuB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,cAAwB,EAAE,MAAc;IAChF,MAAM,WAAW,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAE7C,KAAK,MAAM,oBAAoB,IAAI,WAAW,EAAE,CAAC;QAChD,oBAAoB,CAAC,SAAS,GAAG,oBAAoB,CAAC,SAAS,GAAG,MAAM,CAAA;QACxE,oBAAoB,CAAC,OAAO,GAAG,oBAAoB,CAAC,OAAO,GAAG,MAAM,CAAA;QAEpE,IAAI,oBAAoB,CAAC,QAAQ,EAAE,CAAC;YACnC,oBAAoB,CAAC,QAAQ,GAAG,wBAAwB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,cAAwB,EAAE,aAAa,GAAG,CAAC;IAClF,MAAM,eAAe,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAEjD,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;QAChE,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;QAC5D,CAAC;QAED,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;QAClE,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpB,KAAK,CAAC,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QACzD,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CAAC,uBAAiC,EAAE,UAAkB,EAAE,QAAgB,EAAE,eAAe,GAAuB,QAAQ,EAAE,UAAU,GAAyB,UAAU;IACjO,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAEvC,KAAK,MAAM,SAAS,IAAI,uBAAuB,EAAE,CAAC;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,gBAAiB,CAAA;QACnD,MAAM,aAAa,GAAG,SAAS,CAAC,cAAe,CAAA;QAE/C,MAAM,aAAa,GAAG,CAAC,yBAAyB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAEhE,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,aAAa,CAAC,CAAA;IACrE,CAAC;IAED,MAAM,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,GAAG,wCAAwC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IAEvI,MAAM,qBAAqB,GAAG,MAAM,mBAAmB,CAAC,2BAA2B,CAAC,CAAA;IAEpF,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,qBAAqB,CAAA;IAEzE,MAAM,eAAe,GAAa,EAAE,CAAA;IACpC,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,IAAI,kBAAkB,GAAG,uBAAuB,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAA;IAEpE,IAAI,WAAW,GAAG,EAAE,CAAA;IACpB,IAAI,gBAAgB,GAAa,EAAE,CAAA;IAEnC,KAAK,MAAM,oBAAoB,IAAI,qBAAqB,EAAE,CAAC;QAC1D,KAAK,IAAI,aAAa,GAAG,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,oBAAoB,CAAC,GAAG,EAAE,aAAa,EAAE,EAAE,CAAC;YAChH,MAAM,aAAa,GAAG,qBAAqB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;YACpE,MAAM,YAAY,GAAa,EAAE,CAAA;YAEjC,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC7C,MAAM,iBAAiB,GAAG,2BAA2B,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBAEpE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACrC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,CAAA;oBAEpE,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;oBAEpC,kBAAkB,GAAG,iBAAiB,CAAC,OAAO,IAAI,kBAAkB,CAAA;gBACrE,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAA;YAEvC,MAAM,qBAAqB,GAAkB;gBAC5C,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,YAAY;gBAClB,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,kBAAkB;gBAC3D,OAAO,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,kBAAkB;gBAE7E,QAAQ,EAAE,YAAY;aACtB,CAAA;YAED,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;YAC5C,WAAW,IAAI,YAAY,CAAA;QAC5B,CAAC;QAED,IAAI,eAAe,KAAK,QAAQ;YAC/B,oBAAoB,CAAC,GAAG,KAAK,qBAAqB,CAAC,SAAS,CAAC,MAAM;YACnE,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAElC,MAAM,oBAAoB,GAAkB;gBAC3C,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,yBAAyB,CAAC,WAAW,EAAE,UAAU,CAAC;gBACxD,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,kBAAkB;gBAC/D,OAAO,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,kBAAkB;gBAErF,QAAQ,EAAE,gBAAgB;aAC1B,CAAA;YAED,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;YAE1C,WAAW,GAAG,EAAE,CAAA;YAChB,gBAAgB,GAAG,EAAE,CAAA;QACtB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,eAAe,EAAE,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,QAAkB,EAAE,IAAY;IACnF,MAAM,EAAE,gBAAgB,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;IAEhD,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,SAAS,eAAe,CAAC,QAAkB;QAC1C,IAAI,aAAa,GAAG,CAAC,CAAA;QAErB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC3B,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;gBAErB,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;gBAE1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;gBAEjC,IAAI,WAA+B,CAAA;gBACnC,IAAI,SAA6B,CAAA;gBAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;oBAE7B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;oBAE5D,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;wBAC3B,SAAQ;oBACT,CAAC;oBAED,aAAa,GAAG,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;oBAEpD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;wBACb,WAAW,GAAG,cAAc,CAAA;oBAC7B,CAAC;oBAED,SAAS,GAAG,aAAa,CAAA;gBAC1B,CAAC;gBAED,KAAK,CAAC,gBAAgB,GAAG,WAAW,IAAI,aAAa,CAAA;gBACrD,KAAK,CAAC,cAAc,GAAG,SAAS,IAAI,aAAa,CAAA;gBAEjD,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;gBACjE,KAAK,CAAC,cAAc,GAAG,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;gBAE7D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,aAAa,GAAG,SAAS,CAAA;gBAC1B,CAAC;YACF,CAAC;iBAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC3B,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAA;AACjC,CAAC;AAED,SAAS,oDAAoD,CAAC,UAAkB,EAAE,YAAsB,EAAE,gBAAwB;IACjI,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;IACzD,CAAC;IAED,IAAI,kBAAkB,GAAG,UAAU,CAAA;IAEnC,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAE1D,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAA;QAE/B,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtC,SAAQ;QACT,CAAC;QAED,IAAI,WAAW,GAAG,EAAE,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,SAAS,KAAK,CAAC,CAAA;YACnC,MAAM,UAAU,GAAG,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAA;YAC9D,MAAM,iBAAiB,GAAG,WAAW,IAAI,UAAU,CAAA;YAEnD,IAAI,CAAC,UAAU,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,WAAW,IAAI,gBAAgB,CAAA;gBAChC,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,WAAW,IAAI,IAAI,CAAA;YACpB,CAAC;YAED,SAAS,IAAI,IAAI,CAAC,MAAM,CAAA;QACzB,CAAC;QAED,IAAI,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,eAAe,GAAG,SAAS,CAAC,gBAAiB,CAAA;YACnD,MAAM,aAAa,GAAG,SAAS,CAAC,cAAe,CAAA;YAE/C,kBAAkB;gBACjB,kBAAkB,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC;oBAChD,WAAW;oBACX,kBAAkB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAkB,EAAE,SAA4C;IAC9F,MAAM,mBAAmB,GAAoB,EAAE,CAAA;IAE/C,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js';
|
|
2
2
|
export async function fetchAndParseWikipediaArticle(articleName, language, callbacks) {
|
|
3
|
-
const { default:
|
|
4
|
-
const document = await
|
|
3
|
+
const { default: wtfWikipedia } = await import('wtf_wikipedia');
|
|
4
|
+
const document = await wtfWikipedia.fetch(articleName, language);
|
|
5
5
|
if (!document) {
|
|
6
6
|
throw new Error('Error fetching Wikipedia article');
|
|
7
7
|
}
|
|
@@ -9,13 +9,13 @@ export async function fetchAndParseWikipediaArticle(articleName, language, callb
|
|
|
9
9
|
const sectionsText = [];
|
|
10
10
|
for (const section of sections) {
|
|
11
11
|
const sectionTitle = section.title();
|
|
12
|
-
if (
|
|
12
|
+
if (isWordOrEmojiOrSymbolWord(sectionTitle)) {
|
|
13
13
|
sectionsText.push(sectionTitle);
|
|
14
14
|
}
|
|
15
15
|
const sectionParagraphs = splitToParagraphs(section.text(), 'single', 'preserve');
|
|
16
16
|
for (const paragraph of sectionParagraphs) {
|
|
17
17
|
const paragraphText = paragraph;
|
|
18
|
-
if (
|
|
18
|
+
if (isWordOrEmojiOrSymbolWord(paragraphText)) {
|
|
19
19
|
sectionsText.push(paragraphText);
|
|
20
20
|
}
|
|
21
21
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WikipediaReader.js","sourceRoot":"","sources":["../../src/utilities/WikipediaReader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"WikipediaReader.js","sourceRoot":"","sources":["../../src/utilities/WikipediaReader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAErF,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,WAAmB,EAAE,QAAgB,EAAE,SAA6B;IACvH,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAA;IAE/D,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IAEhE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IACpC,MAAM,YAAY,GAAa,EAAE,CAAA;IAEjC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,EAAE,CAAA;QAEpC,IAAI,yBAAyB,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAChC,CAAC;QAED,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAA;QAEjF,KAAK,MAAM,SAAS,IAAI,iBAAiB,EAAE,CAAC;YAC3C,MAAM,aAAa,GAAG,SAAS,CAAA;YAE/B,IAAI,yBAAyB,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC9C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;YACjC,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "3.0
|
|
3
|
+
"version": "3.2.0",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "MIT AND GPL-3.0",
|
|
@@ -52,8 +52,8 @@
|
|
|
52
52
|
"echogarden": "./dist/cli/CLILauncher.js"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@aws-sdk/client-polly": "~3.
|
|
56
|
-
"@aws-sdk/client-transcribe-streaming": "~3.
|
|
55
|
+
"@aws-sdk/client-polly": "~3.1106.0",
|
|
56
|
+
"@aws-sdk/client-transcribe-streaming": "~3.1106.0",
|
|
57
57
|
"@echogarden/audio-io": "~0.4.2",
|
|
58
58
|
"@echogarden/fasttext-wasm": "~0.1.0",
|
|
59
59
|
"@echogarden/flite-wasi": "~0.1.1",
|
|
@@ -69,30 +69,31 @@
|
|
|
69
69
|
"@echogarden/wave-codec": "~0.4.0",
|
|
70
70
|
"@echogarden/whisper.cpp-binding": "~0.2.2",
|
|
71
71
|
"@mozilla/readability": "~0.6.0",
|
|
72
|
-
"chalk": "~
|
|
72
|
+
"chalk": "~6.0.0",
|
|
73
73
|
"command-exists": "~1.2.9",
|
|
74
|
+
"easier-http-request": "~0.1.0",
|
|
74
75
|
"fs-extra": "~11.4.0",
|
|
75
76
|
"graceful-fs": "~4.2.11",
|
|
76
77
|
"html-to-text": "~10.0.0",
|
|
77
78
|
"import-meta-resolve": "~4.2.0",
|
|
78
79
|
"jieba-wasm": "~2.4.0",
|
|
79
|
-
"jsdom": "~
|
|
80
|
+
"jsdom": "~30.0.1",
|
|
80
81
|
"kuromoji": "~0.1.2",
|
|
81
82
|
"microsoft-cognitiveservices-speech-sdk": "~1.51.0",
|
|
82
83
|
"msgpack-lite": "~0.1.26",
|
|
83
84
|
"onnxruntime-node": "~1.21.1",
|
|
84
|
-
"openai": "~
|
|
85
|
+
"openai": "~7.4.0",
|
|
85
86
|
"quick-json5": "~0.4.3",
|
|
86
|
-
"
|
|
87
|
-
"regexp-composer": "~0.3.0",
|
|
87
|
+
"regexp-composer": "~0.4.0",
|
|
88
88
|
"sam-js": "~0.3.1",
|
|
89
|
+
"standard-html-escaper": "~0.2.1",
|
|
89
90
|
"strip-ansi": "~7.2.0",
|
|
90
91
|
"tar": "~7.5.22",
|
|
91
92
|
"tiktoken": "~1.0.22",
|
|
92
93
|
"tinyld": "~1.3.4",
|
|
93
94
|
"wasm-feature-detect": "~1.8.0",
|
|
94
95
|
"wasm-heap-manager": "~0.4.0",
|
|
95
|
-
"ws": "~8.21.
|
|
96
|
+
"ws": "~8.21.3",
|
|
96
97
|
"wtf_wikipedia": "~10.4.2"
|
|
97
98
|
},
|
|
98
99
|
"peerDependencies": {
|
|
@@ -115,9 +116,9 @@
|
|
|
115
116
|
"@echogarden/gnuspeech-wasm": "~0.3.0",
|
|
116
117
|
"@types/fs-extra": "~11.0.4",
|
|
117
118
|
"@types/graceful-fs": "~4.1.9",
|
|
118
|
-
"@types/jsdom": "~
|
|
119
|
+
"@types/jsdom": "~30.0.0",
|
|
119
120
|
"@types/msgpack-lite": "~0.1.12",
|
|
120
|
-
"@types/node": "~26.
|
|
121
|
+
"@types/node": "~26.2.0",
|
|
121
122
|
"@types/ws": "~8.18.1",
|
|
122
123
|
"ts-json-schema-generator": "~2.9.0",
|
|
123
124
|
"typescript": "~7.0.2"
|
|
@@ -127,7 +128,7 @@
|
|
|
127
128
|
"adm-zip": "~0.6.0",
|
|
128
129
|
"uuid": "~11.1.1",
|
|
129
130
|
"brace-expansion": "~5.0.9",
|
|
130
|
-
"undici": "~7.
|
|
131
|
+
"undici": "~7.29.0"
|
|
131
132
|
},
|
|
132
133
|
"allowScripts": {
|
|
133
134
|
"onnxruntime-node": true,
|
|
@@ -3,7 +3,7 @@ import { Logger } from '../utilities/Logger.js'
|
|
|
3
3
|
import { loadPackage } from '../utilities/PackageManager.js'
|
|
4
4
|
import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
|
|
5
5
|
import { cosineDistance } from '../math/VectorMath.js'
|
|
6
|
-
import { includesPunctuation,
|
|
6
|
+
import { includesPunctuation, includesWordCharacter, parseText } from '../nlp/Segmentation.js'
|
|
7
7
|
import { Timeline, extractEntries } from '../utilities/Timeline.js'
|
|
8
8
|
import { TimelineTranslationAlignmentCallbacks } from '../api/TimelineTranslationAlignment.js'
|
|
9
9
|
import { OperationCallbacks } from '../api/Common.js'
|
|
@@ -20,7 +20,7 @@ export async function alignTimelineToTextSemantically(timeline: Timeline, text:
|
|
|
20
20
|
|
|
21
21
|
for (const sentenceEntry of timelineSentenceEntries) {
|
|
22
22
|
const wordEntryGroup = sentenceEntry.timeline!
|
|
23
|
-
.filter(wordEntry =>
|
|
23
|
+
.filter(wordEntry => includesWordCharacter(wordEntry.text))
|
|
24
24
|
|
|
25
25
|
timelineWordEntryGroups.push(wordEntryGroup)
|
|
26
26
|
timelineWordGroups.push(wordEntryGroup.map(wordEntry => wordEntry.text))
|
package/src/api/Alignment.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
|
|
|
14
14
|
import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
|
|
15
15
|
import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
|
|
16
16
|
import { type EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
|
|
17
|
-
import {
|
|
17
|
+
import { includesWordCharacter } from '../nlp/Segmentation.js'
|
|
18
18
|
|
|
19
19
|
const log = logToStderr
|
|
20
20
|
|
|
@@ -264,7 +264,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
|
|
|
264
264
|
logger.log('')
|
|
265
265
|
|
|
266
266
|
// Remove non-word entries from recognition timeline
|
|
267
|
-
recognitionTimeline = recognitionTimeline.filter(entry =>
|
|
267
|
+
recognitionTimeline = recognitionTimeline.filter(entry => includesWordCharacter(entry.text))
|
|
268
268
|
|
|
269
269
|
// Synthesize the ground-truth transcript and get its timeline
|
|
270
270
|
logger.start('Synthesize ground-truth transcript with eSpeak')
|
package/src/api/Synthesis.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { type RubberbandOptions } from '../dsp/Rubberband.js'
|
|
|
15
15
|
import { loadLexiconsForLanguage } from '../nlp/Lexicon.js'
|
|
16
16
|
|
|
17
17
|
import { Timeline, TimelineEntry, addTimeOffsetToTimeline, multiplyTimelineByFactor } from '../utilities/Timeline.js'
|
|
18
|
-
import {
|
|
18
|
+
import { ensureDir, existsSync, isFileIsUpToDate, readAndParseJsonFile, writeFileSafe } from '../utilities/FileSystem.js'
|
|
19
19
|
import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode, defaultDialectForLanguageCode, normalizeIdentifierToLanguageCode } from '../utilities/Locale.js'
|
|
20
20
|
import { loadPackage } from '../utilities/PackageManager.js'
|
|
21
21
|
import { EngineMetadata, appName } from './Common.js'
|
|
@@ -27,7 +27,7 @@ import { type DeepgramTTSOptions } from '../synthesis/DeepgramTTS.js'
|
|
|
27
27
|
import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
|
|
28
28
|
import { simplifyPunctuationCharacters } from '../nlp/TextNormalizer.js'
|
|
29
29
|
import { convertHtmlToText } from '../utilities/StringUtilities.js'
|
|
30
|
-
import { joinPath, resolvePath } from '../utilities/PathUtilities.js'
|
|
30
|
+
import { getAppDataDir, joinPath, resolvePath } from '../utilities/PathUtilities.js'
|
|
31
31
|
import { Timer } from '../utilities/Timer.js'
|
|
32
32
|
|
|
33
33
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
package/src/encodings/Ascii.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { EncodeIntoResult } from './TextEncodingsCommon.js'
|
|
2
2
|
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
4
|
+
// ASCII encoding
|
|
5
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
3
6
|
export function encodeAscii(asciiString: string) {
|
|
4
7
|
const charCount = asciiString.length
|
|
5
8
|
|
|
@@ -30,31 +33,32 @@ export function encodeAsciiInto(asciiString: string, resultBuffer: Uint8Array):
|
|
|
30
33
|
return { read: len, written: len }
|
|
31
34
|
}
|
|
32
35
|
|
|
36
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
37
|
+
// ASCII decoding
|
|
38
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
33
39
|
export function decodeAscii(encodedString: Uint8Array) {
|
|
34
40
|
const maxChunkLength = 2 ** 24
|
|
35
41
|
|
|
36
|
-
const
|
|
42
|
+
const chunkedAsciiDecoder = new ChunkedAsciiDecoder()
|
|
43
|
+
|
|
44
|
+
let resultString = ''
|
|
37
45
|
|
|
38
46
|
for (let offset = 0; offset < encodedString.length; offset += maxChunkLength) {
|
|
39
|
-
const
|
|
47
|
+
const asciiChunk = encodedString.subarray(offset, offset + maxChunkLength)
|
|
48
|
+
const stringChunk = chunkedAsciiDecoder.writeChunk(asciiChunk)
|
|
40
49
|
|
|
41
|
-
|
|
50
|
+
resultString += stringChunk
|
|
42
51
|
}
|
|
43
52
|
|
|
44
|
-
return
|
|
53
|
+
return resultString
|
|
45
54
|
}
|
|
46
55
|
|
|
47
56
|
export class ChunkedAsciiDecoder {
|
|
48
|
-
private str = ''
|
|
49
57
|
private readonly textDecoder = new TextDecoder('windows-1252')
|
|
50
58
|
|
|
51
59
|
writeChunk(chunk: Uint8Array) {
|
|
52
60
|
const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
|
|
53
61
|
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
toString() {
|
|
58
|
-
return this.str
|
|
62
|
+
return decodedChunk
|
|
59
63
|
}
|
|
60
64
|
}
|
package/src/encodings/Base64.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { decodeAscii } from './Ascii.js'
|
|
2
2
|
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
4
|
+
// Base64 encoding
|
|
5
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
3
6
|
export function encodeBase64(inputBytes: Uint8Array,
|
|
4
7
|
paddingCharacter: string | undefined = '=',
|
|
5
8
|
charCodeMap?: Uint8Array): string {
|
|
@@ -88,6 +91,9 @@ export function encodeBase64AsAsciiBuffer(
|
|
|
88
91
|
return charCodes.subarray(0, writeOffset)
|
|
89
92
|
}
|
|
90
93
|
|
|
94
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
95
|
+
// Base64 decoding
|
|
96
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
91
97
|
export function decodeBase64(
|
|
92
98
|
base64String: string,
|
|
93
99
|
outputBuffer?: Uint8Array,
|
|
@@ -153,5 +159,8 @@ export function decodeBase64(
|
|
|
153
159
|
return outputBuffer.subarray(0, writeOffset)
|
|
154
160
|
}
|
|
155
161
|
|
|
162
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
163
|
+
// Lookup tables
|
|
164
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
156
165
|
export const defaultBase64CharCodeMap: Uint8Array = new Uint8Array([65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 43, 47])
|
|
157
166
|
export const defaultBase64ReverseCharCodeMap: Uint8Array = new Uint8Array([255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 0, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255])
|
package/src/encodings/Hex.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import { decodeAscii } from
|
|
1
|
+
import { decodeAscii } from './Ascii.js'
|
|
2
2
|
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
4
|
+
// Hexadecimal encoding
|
|
5
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
3
6
|
export function encodeHex(buffer: Uint8Array) {
|
|
4
7
|
const asciiBuffer = encodeHexAsAsciiBuffer(buffer)
|
|
5
8
|
|
|
@@ -27,11 +30,14 @@ export function encodeHexAsAsciiBuffer(buffer: Uint8Array) {
|
|
|
27
30
|
return charCodes
|
|
28
31
|
}
|
|
29
32
|
|
|
33
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
34
|
+
// Hexadecimal decoding
|
|
35
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
30
36
|
export function decodeHex(hexString: string) {
|
|
31
37
|
const hexLength = hexString.length
|
|
32
38
|
|
|
33
39
|
if (hexLength % 2 !== 0) {
|
|
34
|
-
throw new Error(`Hexadecimal string doesn't have an even number of characters
|
|
40
|
+
throw new Error(`Hexadecimal string doesn't have an even number of characters.`)
|
|
35
41
|
}
|
|
36
42
|
|
|
37
43
|
const buffer = new Uint8Array(hexLength / 2)
|
|
@@ -63,5 +69,8 @@ function hexCharCodeToValue(hexCharCode: number) {
|
|
|
63
69
|
}
|
|
64
70
|
}
|
|
65
71
|
|
|
72
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
73
|
+
// Lookup tables
|
|
74
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
66
75
|
const hexCharLookup: string[] = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
|
|
67
76
|
const hexCharCodeLookup = new Uint8Array([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 97, 98, 99, 100, 101, 102])
|
package/src/encodings/Utf16.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { EncodeIntoResult } from "./TextEncodingsCommon.js"
|
|
2
2
|
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
4
|
+
// UTF-16 Encoding
|
|
5
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
3
6
|
export function encodeUtf16(text: string) {
|
|
4
7
|
const resultArray = new Uint16Array(text.length)
|
|
5
8
|
|
|
@@ -22,31 +25,32 @@ export function encodeUtf16Into(text: string, resultBuffer: Uint16Array): Encode
|
|
|
22
25
|
return { read: len, written: len }
|
|
23
26
|
}
|
|
24
27
|
|
|
28
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
29
|
+
// UTF-16 Decoding
|
|
30
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
25
31
|
export function decodeUtf16(encodedString: Uint16Array) {
|
|
26
32
|
const maxChunkLength = 2 ** 24
|
|
27
33
|
|
|
28
|
-
const
|
|
34
|
+
const chunkedUtf16Decoder = new ChunkedUtf16Decoder()
|
|
35
|
+
|
|
36
|
+
let resultString = ''
|
|
29
37
|
|
|
30
38
|
for (let offset = 0; offset < encodedString.length; offset += maxChunkLength) {
|
|
31
|
-
const
|
|
39
|
+
const utf16Chunk = encodedString.subarray(offset, offset + maxChunkLength)
|
|
40
|
+
const stringChunk = chunkedUtf16Decoder.writeChunk(utf16Chunk)
|
|
32
41
|
|
|
33
|
-
|
|
42
|
+
resultString += stringChunk
|
|
34
43
|
}
|
|
35
44
|
|
|
36
|
-
return
|
|
45
|
+
return resultString
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
export class ChunkedUtf16Decoder {
|
|
40
|
-
private str = ''
|
|
41
49
|
private readonly textDecoder = new TextDecoder('utf-16le')
|
|
42
50
|
|
|
43
51
|
writeChunk(chunk: Uint16Array) {
|
|
44
52
|
const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
|
|
45
53
|
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
toString() {
|
|
50
|
-
return this.str
|
|
54
|
+
return decodedChunk
|
|
51
55
|
}
|
|
52
56
|
}
|
package/src/encodings/Utf32.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { EncodeIntoResult } from "./TextEncodingsCommon.js"
|
|
2
2
|
|
|
3
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
4
|
+
// UTF-32 Encoding
|
|
5
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
3
6
|
export function encodeUtf32(text: string) {
|
|
4
7
|
const resultArray = new Uint32Array(text.length * 2)
|
|
5
8
|
|
|
@@ -33,6 +36,9 @@ export function encodeUtf32Into(text: string, resultBuffer: Uint32Array): Encode
|
|
|
33
36
|
return { read: readOffset, written: writeOffset }
|
|
34
37
|
}
|
|
35
38
|
|
|
39
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
40
|
+
// UTF-32 Decoding
|
|
41
|
+
//////////////////////////////////////////////////////////////////////////////
|
|
36
42
|
export function decodeUtf32(encodedString: Uint32Array) {
|
|
37
43
|
let result = ''
|
|
38
44
|
|