echogarden 0.12.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -14
- package/data/schemas/options.json +398 -111
- package/dist/alignment/DTWMfccSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWMfccSequenceAlignment.js +8 -8
- package/dist/alignment/DTWSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignment.js +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.js +2 -2
- package/dist/alignment/LevenshteinSequenceAlignment.d.ts +1 -1
- package/dist/alignment/LevenshteinSequenceAlignment.js +1 -1
- package/dist/alignment/SpeechAlignment.d.ts +9 -10
- package/dist/alignment/SpeechAlignment.js +136 -105
- package/dist/alignment/SpeechAlignment.js.map +1 -1
- package/dist/api/API.d.ts +13 -12
- package/dist/api/API.js +14 -13
- package/dist/api/API.js.map +1 -1
- package/dist/api/APIOptions.d.ts +5 -4
- package/dist/api/Alignment.d.ts +15 -9
- package/dist/api/Alignment.js +88 -74
- package/dist/api/Alignment.js.map +1 -1
- package/dist/api/Common.js +1 -1
- package/dist/api/Denoising.d.ts +6 -6
- package/dist/api/Denoising.js +23 -23
- package/dist/api/Denoising.js.map +1 -1
- package/dist/api/LanguageDetection.d.ts +19 -12
- package/dist/api/LanguageDetection.js +88 -38
- package/dist/api/LanguageDetection.js.map +1 -1
- package/dist/api/Recognition.d.ts +16 -6
- package/dist/api/Recognition.js +129 -55
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/SourceSeparation.d.ts +17 -0
- package/dist/api/SourceSeparation.js +61 -0
- package/dist/api/SourceSeparation.js.map +1 -0
- package/dist/api/Synthesis.d.ts +18 -18
- package/dist/api/Synthesis.js +191 -164
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/api/Translation.d.ts +19 -8
- package/dist/api/Translation.js +132 -35
- package/dist/api/Translation.js.map +1 -1
- package/dist/api/Vad.d.ts +10 -5
- package/dist/api/Vad.js +76 -38
- package/dist/api/Vad.js.map +1 -1
- package/dist/audio/AudioBufferConversion.d.ts +1 -1
- package/dist/audio/AudioBufferConversion.js +4 -4
- package/dist/audio/AudioPlayer.d.ts +1 -1
- package/dist/audio/AudioPlayer.js +26 -26
- package/dist/audio/AudioPlayer.js.map +1 -1
- package/dist/audio/AudioRecorder.d.ts +1 -1
- package/dist/audio/AudioRecorder.js +5 -5
- package/dist/audio/AudioUtilities.d.ts +13 -9
- package/dist/audio/AudioUtilities.js +86 -24
- package/dist/audio/AudioUtilities.js.map +1 -1
- package/dist/cli/CLI.d.ts +3 -3
- package/dist/cli/CLI.js +271 -162
- package/dist/cli/CLI.js.map +1 -1
- package/dist/cli/CLIConfigFile.js +8 -8
- package/dist/cli/CLILauncher.js +6 -6
- package/dist/cli/CLIOptionsSchema.js +2 -2
- package/dist/cli/CLIParser.js +5 -5
- package/dist/cli/CLIStarter.js +4 -4
- package/dist/codecs/FFMpegTranscoder.d.ts +2 -2
- package/dist/codecs/FFMpegTranscoder.js +37 -37
- package/dist/codecs/FFMpegTranscoder.js.map +1 -1
- package/dist/codecs/TIMITCodec.js +5 -5
- package/dist/codecs/WaveCodec.d.ts +1 -1
- package/dist/codecs/WaveCodec.js +22 -22
- package/dist/denoising/RNNoise.d.ts +1 -1
- package/dist/denoising/RNNoise.js +9 -9
- package/dist/dsp/BiquadFilter.d.ts +3 -2
- package/dist/dsp/BiquadFilter.js +18 -11
- package/dist/dsp/BiquadFilter.js.map +1 -1
- package/dist/dsp/DecayingPeakEstimator.d.ts +16 -0
- package/dist/dsp/DecayingPeakEstimator.js +23 -0
- package/dist/dsp/DecayingPeakEstimator.js.map +1 -0
- package/dist/dsp/FFT.d.ts +8 -4
- package/dist/dsp/FFT.js +76 -30
- package/dist/dsp/FFT.js.map +1 -1
- package/dist/dsp/KWeightingFilter.d.ts +9 -0
- package/dist/dsp/KWeightingFilter.js +40 -0
- package/dist/dsp/KWeightingFilter.js.map +1 -0
- package/dist/dsp/LoudnessEstimator.d.ts +21 -0
- package/dist/dsp/LoudnessEstimator.js +47 -0
- package/dist/dsp/LoudnessEstimator.js.map +1 -0
- package/dist/dsp/MFCC.d.ts +2 -2
- package/dist/dsp/MFCC.js +15 -15
- package/dist/dsp/MelSpectogram.d.ts +1 -1
- package/dist/dsp/MelSpectogram.js +6 -6
- package/dist/dsp/Rubberband.d.ts +11 -11
- package/dist/dsp/Rubberband.js +27 -27
- package/dist/dsp/Sonic.d.ts +1 -1
- package/dist/dsp/Sonic.js +3 -3
- package/dist/dsp/SpeexResampler.d.ts +1 -1
- package/dist/dsp/SpeexResampler.js +2 -2
- package/dist/math/VectorMath.d.ts +12 -8
- package/dist/math/VectorMath.js +35 -32
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/ChineseSegmentation.js +2 -2
- package/dist/nlp/CompromiseNLP.js +3 -3
- package/dist/nlp/EspeakPhonemizer.js +30 -30
- package/dist/nlp/IPA.js +20 -20
- package/dist/nlp/JapaneseSegmentation.js +6 -6
- package/dist/nlp/Lexicon.d.ts +1 -1
- package/dist/nlp/Lexicon.js +7 -7
- package/dist/nlp/Segmentation.d.ts +3 -0
- package/dist/nlp/Segmentation.js +21 -14
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/nlp/TextNormalizer.js +16 -16
- package/dist/recognition/AmazonTranscribeSTT.d.ts +2 -2
- package/dist/recognition/AmazonTranscribeSTT.js +13 -14
- package/dist/recognition/AmazonTranscribeSTT.js.map +1 -1
- package/dist/recognition/AzureCognitiveServicesSTT.js +5 -6
- package/dist/recognition/AzureCognitiveServicesSTT.js.map +1 -1
- package/dist/recognition/GoogleCloudSTT.d.ts +3 -3
- package/dist/recognition/GoogleCloudSTT.js +18 -18
- package/dist/recognition/OpenAICloudSTT.d.ts +19 -0
- package/dist/recognition/OpenAICloudSTT.js +81 -0
- package/dist/recognition/OpenAICloudSTT.js.map +1 -0
- package/dist/recognition/SileroSTT.d.ts +2 -2
- package/dist/recognition/SileroSTT.js +25 -25
- package/dist/recognition/VoskSTT.d.ts +2 -2
- package/dist/recognition/VoskSTT.js +8 -8
- package/dist/recognition/WhisperCppSTT.d.ts +88 -0
- package/dist/recognition/WhisperCppSTT.js +332 -0
- package/dist/recognition/WhisperCppSTT.js.map +1 -0
- package/dist/recognition/WhisperSTT.d.ts +49 -25
- package/dist/recognition/WhisperSTT.js +626 -481
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/server/Client.d.ts +1 -1
- package/dist/server/Client.js +22 -22
- package/dist/server/Server.js +9 -9
- package/dist/server/Server.js.map +1 -1
- package/dist/server/Worker.d.ts +22 -22
- package/dist/server/Worker.js +36 -36
- package/dist/server/Worker.js.map +1 -1
- package/dist/server/WorkerStarter.js +2 -2
- package/dist/source-separation/MDXNetSourceSeparation.d.ts +11 -0
- package/dist/source-separation/MDXNetSourceSeparation.js +161 -0
- package/dist/source-separation/MDXNetSourceSeparation.js.map +1 -0
- package/dist/speech-language-detection/SileroLanguageDetection.d.ts +1 -1
- package/dist/speech-language-detection/SileroLanguageDetection.js +7 -7
- package/dist/subtitles/Subtitles.d.ts +10 -0
- package/dist/subtitles/Subtitles.js +2 -2
- package/dist/subtitles/Subtitles.js.map +1 -1
- package/dist/synthesis/AwsPollyTTS.d.ts +1 -1
- package/dist/synthesis/AwsPollyTTS.js +12 -12
- package/dist/synthesis/AzureCognitiveServicesTTS.js +7 -7
- package/dist/synthesis/CoquiServerTTS.js +10 -10
- package/dist/synthesis/CoquiServerTTS.js.map +1 -1
- package/dist/synthesis/ElevenlabsTTS.d.ts +23 -0
- package/dist/synthesis/ElevenlabsTTS.js +103 -0
- package/dist/synthesis/ElevenlabsTTS.js.map +1 -0
- package/dist/synthesis/EspeakTTS.d.ts +6 -5
- package/dist/synthesis/EspeakTTS.js +81 -69
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/synthesis/FliteTTS.d.ts +3 -3
- package/dist/synthesis/FliteTTS.js +154 -154
- package/dist/synthesis/FliteTTS.js.map +1 -1
- package/dist/synthesis/GoogleCloudTTS.d.ts +3 -3
- package/dist/synthesis/GoogleCloudTTS.js +17 -17
- package/dist/synthesis/GoogleCloudTTS.js.map +1 -1
- package/dist/synthesis/GoogleTranslateTTS.d.ts +1 -1
- package/dist/synthesis/GoogleTranslateTTS.js +103 -103
- package/dist/synthesis/MicrosoftEdgeTTS.d.ts +2 -2
- package/dist/synthesis/MicrosoftEdgeTTS.js +74 -74
- package/dist/synthesis/OpenAICloudTTS.d.ts +13 -0
- package/dist/synthesis/OpenAICloudTTS.js +169 -0
- package/dist/synthesis/OpenAICloudTTS.js.map +1 -0
- package/dist/synthesis/SamTTS.js +3 -3
- package/dist/synthesis/SapiTTS.d.ts +3 -3
- package/dist/synthesis/SapiTTS.js +26 -26
- package/dist/synthesis/StreamlabsPollyTTS.d.ts +2 -2
- package/dist/synthesis/StreamlabsPollyTTS.js +27 -27
- package/dist/synthesis/SvoxPicoTTS.d.ts +2 -2
- package/dist/synthesis/SvoxPicoTTS.js +65 -65
- package/dist/synthesis/SvoxPicoTTS.js.map +1 -1
- package/dist/synthesis/VitsTTS.d.ts +3 -3
- package/dist/synthesis/VitsTTS.js +378 -378
- package/dist/synthesis/VitsTTS.js.map +1 -1
- package/dist/tests/Test.js +2 -2
- package/dist/utilities/Compression.d.ts +5 -0
- package/dist/utilities/Compression.js +29 -13
- package/dist/utilities/Compression.js.map +1 -1
- package/dist/utilities/FileDownloader.d.ts +1 -1
- package/dist/utilities/FileDownloader.js +16 -16
- package/dist/utilities/FileSystem.js +7 -7
- package/dist/utilities/Locale.d.ts +7 -7
- package/dist/utilities/Locale.js +15 -15
- package/dist/utilities/Logger.js +3 -3
- package/dist/utilities/ObjectUtilities.js +19 -19
- package/dist/utilities/OpenPromise.js +2 -2
- package/dist/utilities/OpenPromise.js.map +1 -1
- package/dist/utilities/PackageManager.js +31 -0
- package/dist/utilities/PackageManager.js.map +1 -1
- package/dist/utilities/PathUtilities.js +8 -8
- package/dist/utilities/RandomGenerator.js +2 -2
- package/dist/utilities/SmoothEstimator.d.ts +8 -0
- package/dist/utilities/SmoothEstimator.js +25 -0
- package/dist/utilities/SmoothEstimator.js.map +1 -0
- package/dist/utilities/TarballMaker.js +8 -8
- package/dist/utilities/Timeline.d.ts +3 -2
- package/dist/utilities/Timeline.js +11 -11
- package/dist/utilities/Timeline.js.map +1 -1
- package/dist/utilities/Timer.js +4 -4
- package/dist/utilities/Utilities.d.ts +4 -0
- package/dist/utilities/Utilities.js +38 -15
- package/dist/utilities/Utilities.js.map +1 -1
- package/dist/utilities/WasmMemoryManager.js +7 -7
- package/dist/utilities/WebReader.js +23 -23
- package/dist/utilities/WikipediaReader.js +2 -2
- package/dist/voice-activity-detection/AdaptiveGateVAD.d.ts +28 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js +138 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js.map +1 -0
- package/dist/voice-activity-detection/SileroVAD.d.ts +1 -1
- package/dist/voice-activity-detection/SileroVAD.js +5 -5
- package/dist/voice-activity-detection/SileroVAD.js.map +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.d.ts +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.js +4 -4
- package/docs/API.md +29 -11
- package/docs/CLI.md +31 -7
- package/docs/Contributing.md +38 -0
- package/docs/Development.md +93 -19
- package/docs/Engines.md +28 -16
- package/docs/Licenses.md +4 -1
- package/docs/Options.md +158 -78
- package/docs/Releases.md +262 -0
- package/docs/Server.md +7 -7
- package/docs/Tasklist.md +95 -76
- package/docs/Technical.md +4 -4
- package/package.json +13 -14
- package/src/alignment/DTWMfccSequenceAlignment.ts +9 -9
- package/src/alignment/DTWSequenceAlignment.ts +2 -2
- package/src/alignment/DTWSequenceAlignmentWindowed.ts +3 -3
- package/src/alignment/LevenshteinSequenceAlignment.ts +2 -2
- package/src/alignment/SpeechAlignment.ts +204 -119
- package/src/api/API.ts +14 -13
- package/src/api/APIOptions.ts +12 -11
- package/src/api/Alignment.ts +147 -90
- package/src/api/Common.ts +1 -1
- package/src/api/Denoising.ts +28 -28
- package/src/api/LanguageDetection.ts +135 -48
- package/src/api/Recognition.ts +198 -59
- package/src/api/SourceSeparation.ts +99 -0
- package/src/api/Synthesis.ts +217 -181
- package/src/api/Translation.ts +193 -40
- package/src/api/Vad.ts +110 -41
- package/src/audio/AudioBufferConversion.ts +4 -4
- package/src/audio/AudioPlayer.ts +27 -27
- package/src/audio/AudioRecorder.ts +5 -5
- package/src/audio/AudioUtilities.ts +107 -24
- package/src/cli/CLI.ts +313 -164
- package/src/cli/CLIConfigFile.ts +8 -8
- package/src/cli/CLILauncher.ts +6 -6
- package/src/cli/CLIOptionsSchema.ts +2 -2
- package/src/cli/CLIParser.ts +5 -5
- package/src/cli/CLIStarter.ts +4 -4
- package/src/codecs/FFMpegTranscoder.ts +38 -38
- package/src/codecs/TIMITCodec.ts +5 -5
- package/src/codecs/WaveCodec.ts +22 -22
- package/src/denoising/RNNoise.ts +9 -9
- package/src/dsp/BiquadFilter.ts +19 -11
- package/src/dsp/DecayingPeakEstimator.ts +35 -0
- package/src/dsp/FFT.ts +103 -35
- package/src/dsp/KWeightingFilter.ts +43 -0
- package/src/dsp/LoudnessEstimator.ts +74 -0
- package/src/dsp/MFCC.ts +15 -15
- package/src/dsp/MelSpectogram.ts +7 -7
- package/src/dsp/Rubberband.ts +38 -38
- package/src/dsp/Sonic.ts +4 -4
- package/src/dsp/SpeexResampler.ts +2 -2
- package/src/math/VectorMath.ts +42 -33
- package/src/nlp/ChineseSegmentation.ts +3 -3
- package/src/nlp/CompromiseNLP.ts +3 -3
- package/src/nlp/EspeakPhonemizer.ts +30 -30
- package/src/nlp/IPA.ts +20 -20
- package/src/nlp/JapaneseSegmentation.ts +6 -6
- package/src/nlp/Lexicon.ts +8 -8
- package/src/nlp/Segmentation.ts +23 -14
- package/src/nlp/TextNormalizer.ts +16 -16
- package/src/recognition/AmazonTranscribeSTT.ts +16 -17
- package/src/recognition/AzureCognitiveServicesSTT.ts +8 -6
- package/src/recognition/GoogleCloudSTT.ts +21 -21
- package/src/recognition/OpenAICloudSTT.ts +142 -0
- package/src/recognition/SileroSTT.ts +26 -26
- package/src/recognition/VoskSTT.ts +10 -10
- package/src/recognition/WhisperCppSTT.ts +555 -0
- package/src/recognition/WhisperSTT.ts +760 -507
- package/src/server/Client.ts +23 -23
- package/src/server/Server.ts +9 -9
- package/src/server/Worker.ts +53 -53
- package/src/server/WorkerStarter.ts +2 -2
- package/src/source-separation/MDXNetSourceSeparation.ts +228 -0
- package/src/speech-language-detection/SileroLanguageDetection.ts +8 -8
- package/src/subtitles/Subtitles.ts +3 -3
- package/src/synthesis/AwsPollyTTS.ts +14 -14
- package/src/synthesis/AzureCognitiveServicesTTS.ts +10 -10
- package/src/synthesis/CoquiServerTTS.ts +10 -10
- package/src/synthesis/ElevenlabsTTS.ts +137 -0
- package/src/synthesis/EspeakTTS.ts +90 -71
- package/src/synthesis/FliteTTS.ts +157 -157
- package/src/synthesis/GoogleCloudTTS.ts +19 -19
- package/src/synthesis/GoogleTranslateTTS.ts +104 -104
- package/src/synthesis/MicrosoftEdgeTTS.ts +80 -80
- package/src/synthesis/OpenAICloudTTS.ts +196 -0
- package/src/synthesis/SamTTS.ts +3 -3
- package/src/synthesis/SapiTTS.ts +29 -29
- package/src/synthesis/StreamlabsPollyTTS.ts +29 -29
- package/src/synthesis/SvoxPicoTTS.ts +67 -67
- package/src/synthesis/VitsTTS.ts +380 -380
- package/src/tests/Test.ts +4 -4
- package/src/utilities/Compression.ts +34 -13
- package/src/utilities/FileDownloader.ts +19 -19
- package/src/utilities/FileSystem.ts +7 -7
- package/src/utilities/Locale.ts +22 -22
- package/src/utilities/Logger.ts +4 -4
- package/src/utilities/ObjectUtilities.ts +19 -19
- package/src/utilities/OpenPromise.ts +2 -2
- package/src/utilities/PackageManager.ts +40 -0
- package/src/utilities/PathUtilities.ts +8 -8
- package/src/utilities/RandomGenerator.ts +3 -3
- package/src/utilities/SmoothEstimator.ts +35 -0
- package/src/utilities/TarballMaker.ts +9 -9
- package/src/utilities/Timeline.ts +15 -13
- package/src/utilities/Timer.ts +4 -4
- package/src/utilities/Utilities.ts +49 -15
- package/src/utilities/WasmMemoryManager.ts +7 -7
- package/src/utilities/WebReader.ts +23 -23
- package/src/utilities/WikipediaReader.ts +2 -2
- package/src/voice-activity-detection/AdaptiveGateVAD.ts +202 -0
- package/src/voice-activity-detection/SileroVAD.ts +5 -5
- package/src/voice-activity-detection/WebRtcVAD.ts +5 -5
- package/dist/synthesis/ElevenLabsTTS.d.ts +0 -8
- package/dist/synthesis/ElevenLabsTTS.js +0 -82
- package/dist/synthesis/ElevenLabsTTS.js.map +0 -1
- package/src/synthesis/ElevenLabsTTS.ts +0 -104
package/src/math/VectorMath.ts
CHANGED
|
@@ -2,7 +2,7 @@ import createMedianFilter from 'moving-median'
|
|
|
2
2
|
|
|
3
3
|
export function covarianceMatrixOfSamples(samples: number[][], weights?: number[], biased = false) {
|
|
4
4
|
if (samples.length == 0) {
|
|
5
|
-
throw new Error(
|
|
5
|
+
throw new Error('No vectors given')
|
|
6
6
|
}
|
|
7
7
|
|
|
8
8
|
const { centeredVectors: centeredSamples, mean } = centerVectors(samples, weights)
|
|
@@ -22,7 +22,7 @@ export function covarianceMatrixOfCenteredSamples(centeredSamples: number[][], b
|
|
|
22
22
|
const sampleCount = centeredSamples.length
|
|
23
23
|
|
|
24
24
|
if (sampleCount == 0) {
|
|
25
|
-
throw new Error(
|
|
25
|
+
throw new Error('No vectors given')
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
const sampleSizeMetric = biased || sampleCount == 1 ? sampleCount : sampleCount - 1
|
|
@@ -64,7 +64,7 @@ export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: numbe
|
|
|
64
64
|
const sampleCount = centeredSamples.length
|
|
65
65
|
|
|
66
66
|
if (sampleCount == 0) {
|
|
67
|
-
throw new Error(
|
|
67
|
+
throw new Error('No vectors given')
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
const featureCount = centeredSamples[0].length
|
|
@@ -165,7 +165,7 @@ export function scaleToSumTo1(vector: number[]) {
|
|
|
165
165
|
}
|
|
166
166
|
|
|
167
167
|
if (sum == Infinity) {
|
|
168
|
-
throw new Error(
|
|
168
|
+
throw new Error('Vector sum is infinite')
|
|
169
169
|
}
|
|
170
170
|
|
|
171
171
|
for (let i = 0; i < vector.length; i++) {
|
|
@@ -177,9 +177,9 @@ export function scaleToSumTo1(vector: number[]) {
|
|
|
177
177
|
return scaledVector
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
export function normalizeVector(vector: number[], kind:
|
|
180
|
+
export function normalizeVector(vector: number[], kind: 'population' | 'sample' = 'population') {
|
|
181
181
|
if (vector.length == 0) {
|
|
182
|
-
throw new Error(
|
|
182
|
+
throw new Error('Vector is empty')
|
|
183
183
|
}
|
|
184
184
|
|
|
185
185
|
const mean = meanOfVector(vector)
|
|
@@ -196,7 +196,7 @@ export function normalizeVector(vector: number[], kind: "population" | "sample"
|
|
|
196
196
|
return { normalizedVector, mean, stdDeviation }
|
|
197
197
|
}
|
|
198
198
|
|
|
199
|
-
export function normalizeVectors(vectors: number[][], kind:
|
|
199
|
+
export function normalizeVectors(vectors: number[][], kind: 'population' | 'sample' = 'population') {
|
|
200
200
|
const vectorCount = vectors.length
|
|
201
201
|
|
|
202
202
|
if (vectorCount == 0) {
|
|
@@ -296,18 +296,18 @@ export function weightedMeanOfVectors(vectors: number[][], weights: number[]) {
|
|
|
296
296
|
return result
|
|
297
297
|
}
|
|
298
298
|
|
|
299
|
-
export function stdDeviationOfVectors(vectors: number[][], kind:
|
|
299
|
+
export function stdDeviationOfVectors(vectors: number[][], kind: 'population' | 'sample' = 'population', mean?: number[]) {
|
|
300
300
|
return varianceOfVectors(vectors, kind, mean).map(v => Math.sqrt(v))
|
|
301
301
|
}
|
|
302
302
|
|
|
303
|
-
export function varianceOfVectors(vectors: number[][], kind:
|
|
303
|
+
export function varianceOfVectors(vectors: number[][], kind: 'population' | 'sample' = 'population', mean?: number[]) {
|
|
304
304
|
const vectorCount = vectors.length
|
|
305
305
|
|
|
306
306
|
if (vectorCount == 0) {
|
|
307
307
|
return []
|
|
308
308
|
}
|
|
309
309
|
|
|
310
|
-
const sampleSizeMetric = kind ==
|
|
310
|
+
const sampleSizeMetric = kind == 'population' || vectorCount == 1 ? vectorCount : vectorCount - 1
|
|
311
311
|
const featureCount = vectors[0].length
|
|
312
312
|
|
|
313
313
|
if (!mean) {
|
|
@@ -331,7 +331,7 @@ export function varianceOfVectors(vectors: number[][], kind: "population" | "sam
|
|
|
331
331
|
|
|
332
332
|
export function meanOfVector(vector: number[]) {
|
|
333
333
|
if (vector.length == 0) {
|
|
334
|
-
throw new Error(
|
|
334
|
+
throw new Error('Vector is empty')
|
|
335
335
|
}
|
|
336
336
|
|
|
337
337
|
return sumVector(vector) / vector.length
|
|
@@ -339,22 +339,22 @@ export function meanOfVector(vector: number[]) {
|
|
|
339
339
|
|
|
340
340
|
export function medianOfVector(vector: number[]) {
|
|
341
341
|
if (vector.length == 0) {
|
|
342
|
-
throw new Error(
|
|
342
|
+
throw new Error('Vector is empty')
|
|
343
343
|
}
|
|
344
344
|
|
|
345
345
|
return vector[Math.floor(vector.length / 2)]
|
|
346
346
|
}
|
|
347
347
|
|
|
348
|
-
export function stdDeviationOfVector(vector: number[], kind:
|
|
348
|
+
export function stdDeviationOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
|
|
349
349
|
return Math.sqrt(varianceOfVector(vector, kind, mean))
|
|
350
350
|
}
|
|
351
351
|
|
|
352
|
-
export function varianceOfVector(vector: number[], kind:
|
|
352
|
+
export function varianceOfVector(vector: number[], kind: 'population' | 'sample' = 'population', mean?: number) {
|
|
353
353
|
if (vector.length == 0) {
|
|
354
|
-
throw new Error(
|
|
354
|
+
throw new Error('Vector is empty')
|
|
355
355
|
}
|
|
356
356
|
|
|
357
|
-
const sampleSizeMetric = kind ==
|
|
357
|
+
const sampleSizeMetric = kind == 'population' || vector.length == 1 ? vector.length : vector.length - 1
|
|
358
358
|
|
|
359
359
|
if (mean == null) {
|
|
360
360
|
mean = meanOfVector(vector)
|
|
@@ -370,7 +370,7 @@ export function varianceOfVector(vector: number[], kind: "population" | "sample"
|
|
|
370
370
|
}
|
|
371
371
|
|
|
372
372
|
export function logOfVector(vector: number[], minVal = 1e-40) {
|
|
373
|
-
return vector.map(value => Math.log(
|
|
373
|
+
return vector.map(value => Math.log(value + minVal))
|
|
374
374
|
}
|
|
375
375
|
|
|
376
376
|
export function expOfVector(vector: number[]) {
|
|
@@ -418,7 +418,7 @@ export function movingAverageOfWindow3(vector: number[]) {
|
|
|
418
418
|
|
|
419
419
|
export function averageMeanSquaredError(actual: number[][], expected: number[][]) {
|
|
420
420
|
if (actual.length != expected.length) {
|
|
421
|
-
throw new Error(
|
|
421
|
+
throw new Error('Vectors are not the same length')
|
|
422
422
|
}
|
|
423
423
|
|
|
424
424
|
const vectorCount = actual.length
|
|
@@ -438,7 +438,7 @@ export function averageMeanSquaredError(actual: number[][], expected: number[][]
|
|
|
438
438
|
|
|
439
439
|
export function meanSquaredError(actual: number[], expected: number[]) {
|
|
440
440
|
if (actual.length != expected.length) {
|
|
441
|
-
throw new Error(
|
|
441
|
+
throw new Error('Vectors are not the same length')
|
|
442
442
|
}
|
|
443
443
|
|
|
444
444
|
const featureCount = actual.length
|
|
@@ -456,13 +456,13 @@ export function meanSquaredError(actual: number[], expected: number[]) {
|
|
|
456
456
|
return sum / featureCount
|
|
457
457
|
}
|
|
458
458
|
|
|
459
|
-
export function
|
|
460
|
-
return Math.sqrt(
|
|
459
|
+
export function euclidianDistance(vector1: number[], vector2: number[]) {
|
|
460
|
+
return Math.sqrt(squaredEuclidianDistance(vector1, vector2))
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
-
export function
|
|
463
|
+
export function squaredEuclidianDistance(vector1: number[], vector2: number[]) {
|
|
464
464
|
if (vector1.length != vector2.length) {
|
|
465
|
-
throw new Error(
|
|
465
|
+
throw new Error('Vectors are not the same length')
|
|
466
466
|
}
|
|
467
467
|
|
|
468
468
|
const elementCount = vector1.length
|
|
@@ -486,7 +486,7 @@ export function cosineDistance(vector1: number[], vector2: number[]) {
|
|
|
486
486
|
|
|
487
487
|
export function cosineSimilarity(vector1: number[], vector2: number[]) {
|
|
488
488
|
if (vector1.length != vector2.length) {
|
|
489
|
-
throw new Error(
|
|
489
|
+
throw new Error('Vectors are not the same length')
|
|
490
490
|
}
|
|
491
491
|
|
|
492
492
|
if (vector1.length == 0) {
|
|
@@ -514,7 +514,7 @@ export function cosineSimilarity(vector1: number[], vector2: number[]) {
|
|
|
514
514
|
|
|
515
515
|
export function minkowskiDistance(vector1: number[], vector2: number[], power: number) {
|
|
516
516
|
if (vector1.length != vector2.length) {
|
|
517
|
-
throw new Error(
|
|
517
|
+
throw new Error('Vectors are not the same length')
|
|
518
518
|
}
|
|
519
519
|
|
|
520
520
|
const elementCount = vector1.length
|
|
@@ -538,7 +538,7 @@ export function cosineDistancePrecomputedMagnitudes(vector1: number[], vector2:
|
|
|
538
538
|
|
|
539
539
|
export function cosineSimilarityPrecomputedMagnitudes(vector1: number[], vector2: number[], magnitude1: number, magnitude2: number) {
|
|
540
540
|
if (vector1.length != vector2.length) {
|
|
541
|
-
throw new Error(
|
|
541
|
+
throw new Error('Vectors are not the same length')
|
|
542
542
|
}
|
|
543
543
|
|
|
544
544
|
if (vector1.length == 0) {
|
|
@@ -561,7 +561,7 @@ export function cosineSimilarityPrecomputedMagnitudes(vector1: number[], vector2
|
|
|
561
561
|
|
|
562
562
|
export function subtractVectors(vector1: number[], vector2: number[]) {
|
|
563
563
|
if (vector1.length != vector2.length) {
|
|
564
|
-
throw new Error(
|
|
564
|
+
throw new Error('Vectors are not the same length')
|
|
565
565
|
}
|
|
566
566
|
|
|
567
567
|
const result = createVector(vector1.length)
|
|
@@ -585,7 +585,7 @@ export function sumVector(vector: number[]) {
|
|
|
585
585
|
|
|
586
586
|
export function dotProduct(vector1: number[], vector2: number[]) {
|
|
587
587
|
if (vector1.length != vector2.length) {
|
|
588
|
-
throw new Error(
|
|
588
|
+
throw new Error('Vectors are not the same length')
|
|
589
589
|
}
|
|
590
590
|
|
|
591
591
|
const elementCount = vector1.length
|
|
@@ -616,10 +616,14 @@ export function maxValue(vector: number[]) {
|
|
|
616
616
|
}
|
|
617
617
|
|
|
618
618
|
export function indexOfMax(vector: number[]) {
|
|
619
|
-
|
|
620
|
-
|
|
619
|
+
if (vector.length == 0) {
|
|
620
|
+
return -1
|
|
621
|
+
}
|
|
621
622
|
|
|
622
|
-
|
|
623
|
+
let maxValue = vector[0]
|
|
624
|
+
let result = 0
|
|
625
|
+
|
|
626
|
+
for (let i = 1; i < vector.length; i++) {
|
|
623
627
|
if (vector[i] > maxValue) {
|
|
624
628
|
maxValue = vector[i]
|
|
625
629
|
result = i
|
|
@@ -676,7 +680,7 @@ export function sigmoid(x: number) {
|
|
|
676
680
|
}
|
|
677
681
|
|
|
678
682
|
export function softmax(logits: number[], temperature = 1.0) {
|
|
679
|
-
if (logits.length
|
|
683
|
+
if (logits.length === 0) {
|
|
680
684
|
return []
|
|
681
685
|
}
|
|
682
686
|
|
|
@@ -789,7 +793,7 @@ export function sumExp(values: number[]) {
|
|
|
789
793
|
|
|
790
794
|
export function logSoftmax(values: number[], minVal = 1e-40) {
|
|
791
795
|
const softMaxOfValues = softmax(values)
|
|
792
|
-
|
|
796
|
+
|
|
793
797
|
return logOfVector(softMaxOfValues, minVal)
|
|
794
798
|
}
|
|
795
799
|
|
|
@@ -811,3 +815,8 @@ export class IncrementalMean {
|
|
|
811
815
|
}
|
|
812
816
|
|
|
813
817
|
export type DistanceFunction = (a: number[], b: number[]) => number
|
|
818
|
+
|
|
819
|
+
export interface ComplexNumber {
|
|
820
|
+
real: number
|
|
821
|
+
imaginary: number
|
|
822
|
+
}
|
|
@@ -4,7 +4,7 @@ export async function splitChineseTextToWords_Jieba(text: string, fineGrained =
|
|
|
4
4
|
if (!fineGrained) {
|
|
5
5
|
return jieba.cut(text, useHMM)
|
|
6
6
|
} else {
|
|
7
|
-
const results = jieba.tokenize(text,
|
|
7
|
+
const results = jieba.tokenize(text, 'search', useHMM)
|
|
8
8
|
|
|
9
9
|
const startOffsetsSet = new Set<number>()
|
|
10
10
|
const endOffsetsSet = new Set<number>()
|
|
@@ -57,10 +57,10 @@ export async function splitChineseTextToWords_Jieba(text: string, fineGrained =
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
let JiebaWasmInstance: typeof import(
|
|
60
|
+
let JiebaWasmInstance: typeof import('jieba-wasm')
|
|
61
61
|
async function getWasmInstance() {
|
|
62
62
|
if (!JiebaWasmInstance) {
|
|
63
|
-
const { default: JibeaWasm } = await import(
|
|
63
|
+
const { default: JibeaWasm } = await import('jieba-wasm')
|
|
64
64
|
JiebaWasmInstance = JibeaWasm
|
|
65
65
|
}
|
|
66
66
|
|
package/src/nlp/CompromiseNLP.ts
CHANGED
|
@@ -32,12 +32,12 @@ export async function parse(text: string): Promise<CompromiseParsedDocument> {
|
|
|
32
32
|
endOffset: term.offset.start + term.offset.length
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
if (parsedTerm.text ==
|
|
35
|
+
if (parsedTerm.text == '') {
|
|
36
36
|
if (parsedSentence.length > 0) {
|
|
37
37
|
parsedSentence[parsedSentence.length - 1].postText += parsedTerm.preText + parsedTerm.postText
|
|
38
38
|
}
|
|
39
|
-
} else if (parsedTerm.tags.includes(
|
|
40
|
-
parsedTerm.text +=
|
|
39
|
+
} else if (parsedTerm.tags.includes('Abbreviation') && parsedTerm.postText.startsWith('.')) {
|
|
40
|
+
parsedTerm.text += '.'
|
|
41
41
|
parsedTerm.endOffset += 1
|
|
42
42
|
parsedSentence.push(parsedTerm)
|
|
43
43
|
} else {
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import * as EspeakTTS from
|
|
2
|
-
import { logToStderr } from
|
|
3
|
-
import * as Segmentation from
|
|
1
|
+
import * as EspeakTTS from '../synthesis/EspeakTTS.js'
|
|
2
|
+
import { logToStderr } from '../utilities/Utilities.js'
|
|
3
|
+
import * as Segmentation from './Segmentation.js'
|
|
4
4
|
|
|
5
5
|
const log = logToStderr
|
|
6
6
|
|
|
7
7
|
export async function phonemizeSentence(sentence: string, espeakVoice: string, substitutionMap?: Map<string, string[]>, useIpa = true) {
|
|
8
8
|
const ipaString = await EspeakTTS.textToPhonemes(sentence, espeakVoice, useIpa)
|
|
9
9
|
|
|
10
|
-
const clauseStrings = ipaString.split(
|
|
10
|
+
const clauseStrings = ipaString.split(' | ')
|
|
11
11
|
|
|
12
12
|
const clauses: string[][][] = []
|
|
13
13
|
|
|
@@ -20,14 +20,14 @@ export async function phonemizeSentence(sentence: string, espeakVoice: string, s
|
|
|
20
20
|
for (let wordIndex = 0; wordIndex < wordStrings.length; wordIndex++) {
|
|
21
21
|
const word = wordStrings[wordIndex]
|
|
22
22
|
|
|
23
|
-
let wordPhonemes = word.split(
|
|
23
|
+
let wordPhonemes = word.split('_')
|
|
24
24
|
|
|
25
25
|
wordPhonemes = wordPhonemes.flatMap(phoneme => {
|
|
26
|
-
if (!phoneme || phoneme.startsWith(
|
|
26
|
+
if (!phoneme || phoneme.startsWith('(')) {
|
|
27
27
|
return []
|
|
28
|
-
} else if (phoneme.startsWith(
|
|
28
|
+
} else if (phoneme.startsWith(`ˈ`) || phoneme.startsWith(`ˌ`)) {
|
|
29
29
|
return [phoneme[0], phoneme.substring(1)]
|
|
30
|
-
} else if (phoneme.endsWith(
|
|
30
|
+
} else if (phoneme.endsWith(`ˈ`) || phoneme.endsWith(`ˌ`)) {
|
|
31
31
|
return [phoneme.substring(0, phoneme.length - 1), phoneme[phoneme.length - 1]]
|
|
32
32
|
} else {
|
|
33
33
|
return substitutionMap?.get(phoneme) || [phoneme]
|
|
@@ -49,13 +49,13 @@ export async function phonemizeSentence(sentence: string, espeakVoice: string, s
|
|
|
49
49
|
|
|
50
50
|
export async function phonemizeText(text: string, voice: string, substitutionMap?: Map<string, string[]>) {
|
|
51
51
|
text = text
|
|
52
|
-
.replaceAll(
|
|
53
|
-
.replaceAll(
|
|
54
|
-
.replaceAll(
|
|
55
|
-
.replaceAll(
|
|
56
|
-
.replaceAll(
|
|
57
|
-
.replaceAll(
|
|
58
|
-
.replaceAll(
|
|
52
|
+
.replaceAll(',', ',')
|
|
53
|
+
.replaceAll('、', ',')
|
|
54
|
+
.replaceAll('。', '.')
|
|
55
|
+
.replaceAll('(', ', ')
|
|
56
|
+
.replaceAll(')', ', ')
|
|
57
|
+
.replaceAll('«', ', ')
|
|
58
|
+
.replaceAll('»', ', ')
|
|
59
59
|
|
|
60
60
|
const segmentedText = await Segmentation.parse(text, voice)
|
|
61
61
|
const preparedClauses: string[] = []
|
|
@@ -64,7 +64,7 @@ export async function phonemizeText(text: string, voice: string, substitutionMap
|
|
|
64
64
|
for (const sentence of segmentedText) {
|
|
65
65
|
for (const clause of sentence.phrases) {
|
|
66
66
|
const words = clause.words.filter(wordObject => Segmentation.isWordOrSymbolWord(wordObject.text))
|
|
67
|
-
const preparedClauseText = words.map(word => word.text.replace(/\./g,
|
|
67
|
+
const preparedClauseText = words.map(word => word.text.replace(/\./g, ' ')).join(' ')
|
|
68
68
|
|
|
69
69
|
preparedClauses.push(preparedClauseText)
|
|
70
70
|
|
|
@@ -72,18 +72,18 @@ export async function phonemizeText(text: string, voice: string, substitutionMap
|
|
|
72
72
|
const lastChar = trimmedClauseText[trimmedClauseText.length - 1]
|
|
73
73
|
|
|
74
74
|
if (clause.isSentenceFinalizer) {
|
|
75
|
-
if (trimmedClauseText.endsWith('?') || trimmedClauseText.endsWith(
|
|
75
|
+
if (trimmedClauseText.endsWith('?') || trimmedClauseText.endsWith(`?"`)) {
|
|
76
76
|
clauseBreakers.push('?')
|
|
77
|
-
} else if (trimmedClauseText.endsWith('!') || trimmedClauseText.endsWith(
|
|
77
|
+
} else if (trimmedClauseText.endsWith('!') || trimmedClauseText.endsWith(`!"`)) {
|
|
78
78
|
clauseBreakers.push('!')
|
|
79
79
|
} else {
|
|
80
|
-
clauseBreakers.push(
|
|
80
|
+
clauseBreakers.push('.')
|
|
81
81
|
}
|
|
82
82
|
} else {
|
|
83
83
|
if (lastChar == ':' || lastChar == ';') {
|
|
84
84
|
clauseBreakers.push(lastChar)
|
|
85
85
|
} else {
|
|
86
|
-
clauseBreakers.push(
|
|
86
|
+
clauseBreakers.push(',')
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
}
|
|
@@ -97,25 +97,25 @@ export async function phonemizeClauses(clauses: string[], voice: string, clauseB
|
|
|
97
97
|
return []
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
const preparedText = clauses.join(
|
|
100
|
+
const preparedText = clauses.join('\n\n') // filter(clause => clause.trim().length > 0)
|
|
101
101
|
|
|
102
102
|
const ipaString = await EspeakTTS.textToIPA(preparedText, voice)
|
|
103
103
|
|
|
104
|
-
const ipaLines = ipaString.split(
|
|
104
|
+
const ipaLines = ipaString.split('\n')
|
|
105
105
|
|
|
106
106
|
const phonemeLines = ipaLines.map(line => {
|
|
107
|
-
line = line.replace(/_+/g,
|
|
107
|
+
line = line.replace(/_+/g, '_').replace(/ +/g, ' ')
|
|
108
108
|
|
|
109
|
-
return line.split(
|
|
110
|
-
word = word.replaceAll(
|
|
111
|
-
let wordPhonemes = word.split(
|
|
109
|
+
return line.split(' ').map(word => {
|
|
110
|
+
word = word.replaceAll('_', ' ').trim()
|
|
111
|
+
let wordPhonemes = word.split(' ')
|
|
112
112
|
|
|
113
113
|
wordPhonemes = wordPhonemes.flatMap(phoneme => {
|
|
114
|
-
if (!phoneme || phoneme.startsWith(
|
|
114
|
+
if (!phoneme || phoneme.startsWith('(')) {
|
|
115
115
|
return []
|
|
116
|
-
} else if (phoneme.startsWith(
|
|
116
|
+
} else if (phoneme.startsWith('ˈ') || phoneme.startsWith('ˌ')) {
|
|
117
117
|
return [phoneme[0], phoneme.substring(1)]
|
|
118
|
-
} else if (phoneme.endsWith(
|
|
118
|
+
} else if (phoneme.endsWith('ˈ') || phoneme.endsWith('ˌ')) {
|
|
119
119
|
return [phoneme.substring(0, phoneme.length - 1), phoneme[phoneme.length - 1]]
|
|
120
120
|
} else {
|
|
121
121
|
return [phoneme]
|
|
@@ -157,7 +157,7 @@ export function phonemizedClausesToSentences(phonemizedClauses: string[][][]) {
|
|
|
157
157
|
const lastWord = phonemizedClause[phonemizedClause.length - 1]
|
|
158
158
|
const lastPhoneme = lastWord[lastWord.length - 1]
|
|
159
159
|
|
|
160
|
-
if ([
|
|
160
|
+
if (['.', '?', '!'].includes(lastPhoneme)) {
|
|
161
161
|
phonemizedSentences.push([])
|
|
162
162
|
}
|
|
163
163
|
}
|
package/src/nlp/IPA.ts
CHANGED
|
@@ -1,32 +1,32 @@
|
|
|
1
|
-
import { logToStderr } from
|
|
1
|
+
import { logToStderr } from '../utilities/Utilities.js'
|
|
2
2
|
|
|
3
3
|
const log = logToStderr
|
|
4
4
|
|
|
5
5
|
const consonant = {
|
|
6
|
-
plosive: [
|
|
7
|
-
nasal: [
|
|
8
|
-
trill: [
|
|
9
|
-
tapOrFlap: [
|
|
10
|
-
fricative: [
|
|
11
|
-
lateralFricative: [
|
|
12
|
-
affricate: [
|
|
13
|
-
approximant: [
|
|
14
|
-
lateralApproximant: [
|
|
6
|
+
plosive: ['p', 'b', 't', 'd', 'ʈ', 'ɖ', 'c', 'ɟ', 'k', 'g', 'q', 'ɢ', 'ʔ', /* extensions */ 'ɡ'],
|
|
7
|
+
nasal: ['m', 'ɱ', 'n', 'ɳ', 'ɲ', 'ŋ', 'ɴ', 'n̩'],
|
|
8
|
+
trill: ['ʙ', 'r', 'ʀ'],
|
|
9
|
+
tapOrFlap: ['ⱱ', 'ɾ', 'ɽ'],
|
|
10
|
+
fricative: ['ɸ', 'β', 'f', 'v', 'θ', 'ð', 's', 'z', 'ʃ', 'ʒ', 'ʂ', 'ʐ', 'ç', 'ʝ', 'x', 'ɣ', 'χ', 'ʁ', 'ħ', 'ʕ', 'h', 'ɦ'],
|
|
11
|
+
lateralFricative: ['ɬ', 'ɮ'],
|
|
12
|
+
affricate: ['tʃ', 'ʈʃ', 'dʒ'], // very incomplete, there are many others
|
|
13
|
+
approximant: ['ʋ', 'ɹ', 'ɻ', 'j', 'ɰ', /* extensions */ 'w'],
|
|
14
|
+
lateralApproximant: ['l', 'ɭ', 'ʎ', 'ʟ']
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
const vowel = {
|
|
18
|
-
close: [
|
|
19
|
-
closeOther: [
|
|
20
|
-
closeMid: [
|
|
21
|
-
openMid: [
|
|
22
|
-
open: [
|
|
18
|
+
close: ['i', 'yɨ', 'ʉɯ', 'u', 'iː'],
|
|
19
|
+
closeOther: ['ɪ', 'ʏ', 'ʊ', 'ɨ', 'ᵻ'],
|
|
20
|
+
closeMid: ['e', 'ø', 'ɘ', 'ɵ', 'ɤ', 'o', 'ə', 'oː'],
|
|
21
|
+
openMid: ['ɛ', 'œ', 'ɜ', 'ɞ', 'ʌ', 'ɔ', 'ɜː', 'uː', 'ɔː', 'ɛː'],
|
|
22
|
+
open: ['æ', 'a', 'ɶ', 'ɐ', 'ɑ', 'ɒ', 'ɑː'],
|
|
23
23
|
|
|
24
|
-
rhotic: [
|
|
24
|
+
rhotic: ['◌˞', 'ɚ', 'ɝ', 'ɹ̩'],
|
|
25
25
|
|
|
26
26
|
diphtongs: [
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
'eɪ', 'əʊ', 'oʊ', 'aɪ', 'ɔɪ', 'aʊ', 'iə',
|
|
28
|
+
'ɜr', 'ɑr', 'ɔr', 'oʊr', 'oːɹ', 'ir', 'ɪɹ', 'ɔːɹ', 'ɑːɹ', 'ʊɹ', 'ʊr', 'ɛr', 'ɛɹ',
|
|
29
|
+
'əl', 'aɪɚ', 'aɪə'
|
|
30
30
|
],
|
|
31
31
|
}
|
|
32
32
|
|
|
@@ -42,7 +42,7 @@ for (const p in vowel) {
|
|
|
42
42
|
vowels = [...vowels, ...(vowel as any)[p]]
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
const all = [
|
|
45
|
+
const all = [' ', ...consonants, ...vowels]
|
|
46
46
|
|
|
47
47
|
export function getPhoneSubstitutionCost1(ipa1: string, ipa2: string) {
|
|
48
48
|
if (ipa1 == ipa2) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import path from
|
|
2
|
-
import { OpenPromise } from
|
|
3
|
-
import { resolveModuleScriptPath } from
|
|
1
|
+
import path from 'path'
|
|
2
|
+
import { OpenPromise } from '../utilities/OpenPromise.js'
|
|
3
|
+
import { resolveModuleScriptPath } from '../utilities/Utilities.js'
|
|
4
4
|
|
|
5
5
|
export async function splitJapaneseTextToWords_Kuromoji(text: string) {
|
|
6
6
|
const tokenizer = await getKuromojiTokenizer()
|
|
@@ -18,12 +18,12 @@ async function getKuromojiTokenizer() {
|
|
|
18
18
|
return kuromojiTokenizer
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
const { default: kuromoji } = await import(
|
|
21
|
+
const { default: kuromoji } = await import('kuromoji')
|
|
22
22
|
|
|
23
23
|
const resultOpenPromise = new OpenPromise<any>()
|
|
24
24
|
|
|
25
25
|
const kuromojiScriptPath = await resolveModuleScriptPath('kuromoji')
|
|
26
|
-
const dictionaryPath = path.join(path.dirname(kuromojiScriptPath),
|
|
26
|
+
const dictionaryPath = path.join(path.dirname(kuromojiScriptPath), '..', '/dict')
|
|
27
27
|
|
|
28
28
|
kuromoji.builder({ dicPath: dictionaryPath }).build(function (error: any, tokenizer: any) {
|
|
29
29
|
if (error) {
|
|
@@ -41,7 +41,7 @@ async function getKuromojiTokenizer() {
|
|
|
41
41
|
|
|
42
42
|
/*
|
|
43
43
|
export async function splitJapaneseTextToWords_Sudachi(text: string, mode: 0 | 1 | 2) {
|
|
44
|
-
const { TokenizeMode, tokenize } = await import(
|
|
44
|
+
const { TokenizeMode, tokenize } = await import('sudachi')
|
|
45
45
|
|
|
46
46
|
const resultString = tokenize(text, mode)
|
|
47
47
|
|
package/src/nlp/Lexicon.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readAndParseJsonFile } from
|
|
2
|
-
import { getShortLanguageCode } from
|
|
3
|
-
import { resolveToModuleRootDir } from
|
|
1
|
+
import { readAndParseJsonFile } from '../utilities/FileSystem.js'
|
|
2
|
+
import { getShortLanguageCode } from '../utilities/Locale.js'
|
|
3
|
+
import { resolveToModuleRootDir } from '../utilities/PathUtilities.js'
|
|
4
4
|
|
|
5
5
|
export function tryGetFirstLexiconSubstitution(sentenceWords: string[], wordIndex: number, lexicons: Lexicon[], languageCode: string) {
|
|
6
6
|
const reversedLexicons = [...lexicons].reverse() // Give precedence to later lexicons
|
|
@@ -45,8 +45,8 @@ export function tryGetLexiconSubstitution(sentenceWords: string[], wordIndex: nu
|
|
|
45
45
|
continue
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
const precedingWord = sentenceWords[wordIndex - 1] ||
|
|
49
|
-
const succeedingWord = sentenceWords[wordIndex + 1] ||
|
|
48
|
+
const precedingWord = sentenceWords[wordIndex - 1] || ''
|
|
49
|
+
const succeedingWord = sentenceWords[wordIndex + 1] || ''
|
|
50
50
|
|
|
51
51
|
const precededBy = substitutionEntry?.precededBy || []
|
|
52
52
|
const notPrecededBy = substitutionEntry?.notPrecededBy || []
|
|
@@ -76,8 +76,8 @@ export async function loadLexiconFile(jsonFilePath: string): Promise<Lexicon> {
|
|
|
76
76
|
export async function loadLexiconsForLanguage(language: string, customLexiconPaths?: string[]) {
|
|
77
77
|
const lexicons: Lexicon[] = []
|
|
78
78
|
|
|
79
|
-
if (getShortLanguageCode(language) ==
|
|
80
|
-
const heteronymsLexicon = await loadLexiconFile(resolveToModuleRootDir(
|
|
79
|
+
if (getShortLanguageCode(language) == 'en') {
|
|
80
|
+
const heteronymsLexicon = await loadLexiconFile(resolveToModuleRootDir('data/lexicons/heteronyms.en.json'))
|
|
81
81
|
lexicons.push(heteronymsLexicon)
|
|
82
82
|
}
|
|
83
83
|
|
|
@@ -118,5 +118,5 @@ export type LexiconEntry = {
|
|
|
118
118
|
example?: string
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
export type LexiconWordCase =
|
|
121
|
+
export type LexiconWordCase = 'any' | 'capitalized' | 'uppercase' | 'lowercase' | 'titlecase' | 'camelcase' | 'pascalcase'
|
|
122
122
|
export type LexiconPronunciationForLanguageCodes = { [languageCode: string]: string }
|