echogarden 2.7.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/data/lexicons/heteronyms.en.json +6 -2
  2. package/data/lexicons/words.en.json +379 -19
  3. package/dist/alignment/SemanticTextAlignment.js +6 -7
  4. package/dist/alignment/SemanticTextAlignment.js.map +1 -1
  5. package/dist/api/Alignment.js +2 -2
  6. package/dist/api/Alignment.js.map +1 -1
  7. package/dist/api/Recognition.js +2 -2
  8. package/dist/api/Recognition.js.map +1 -1
  9. package/dist/api/SpeechTranslation.js +2 -2
  10. package/dist/api/SpeechTranslation.js.map +1 -1
  11. package/dist/api/Synthesis.js +7 -7
  12. package/dist/api/Synthesis.js.map +1 -1
  13. package/dist/api/TimelineTranslationAlignment.js +2 -2
  14. package/dist/api/TimelineTranslationAlignment.js.map +1 -1
  15. package/dist/api/TranslationAlignment.js +2 -2
  16. package/dist/api/TranslationAlignment.js.map +1 -1
  17. package/dist/audio/AudioPlayer.js +2 -2
  18. package/dist/audio/AudioPlayer.js.map +1 -1
  19. package/dist/audio/AudioUtilities.d.ts +1 -1
  20. package/dist/cli/CLI.js +4 -4
  21. package/dist/cli/CLI.js.map +1 -1
  22. package/dist/denoising/RNNoise.js +5 -5
  23. package/dist/denoising/RNNoise.js.map +1 -1
  24. package/dist/dsp/FFT.js +13 -19
  25. package/dist/dsp/FFT.js.map +1 -1
  26. package/dist/dsp/Rubberband.js +5 -5
  27. package/dist/dsp/Rubberband.js.map +1 -1
  28. package/dist/dsp/Sonic.js +6 -6
  29. package/dist/dsp/Sonic.js.map +1 -1
  30. package/dist/dsp/SpeexResampler.js +11 -11
  31. package/dist/dsp/SpeexResampler.js.map +1 -1
  32. package/dist/nlp/EspeakPhonemizer.js +1 -1
  33. package/dist/nlp/EspeakPhonemizer.js.map +1 -1
  34. package/dist/nlp/Segmentation.d.ts +13 -10
  35. package/dist/nlp/Segmentation.js +129 -106
  36. package/dist/nlp/Segmentation.js.map +1 -1
  37. package/dist/nlp/TextNormalizer.js +74 -42
  38. package/dist/nlp/TextNormalizer.js.map +1 -1
  39. package/dist/recognition/AmazonTranscribeSTT.js +2 -2
  40. package/dist/recognition/AmazonTranscribeSTT.js.map +1 -1
  41. package/dist/recognition/SileroSTT.js +2 -2
  42. package/dist/recognition/SileroSTT.js.map +1 -1
  43. package/dist/recognition/WhisperSTT.js +4 -16
  44. package/dist/recognition/WhisperSTT.js.map +1 -1
  45. package/dist/subtitles/Subtitles.js +52 -7
  46. package/dist/subtitles/Subtitles.js.map +1 -1
  47. package/dist/synthesis/ElevenLabsTTS.js +11 -10
  48. package/dist/synthesis/ElevenLabsTTS.js.map +1 -1
  49. package/dist/synthesis/EspeakTTS.js +8 -8
  50. package/dist/synthesis/EspeakTTS.js.map +1 -1
  51. package/dist/synthesis/KokoroTTS.js +21 -10
  52. package/dist/synthesis/KokoroTTS.js.map +1 -1
  53. package/dist/synthesis/SvoxPicoTTS.js +24 -23
  54. package/dist/synthesis/SvoxPicoTTS.js.map +1 -1
  55. package/dist/text-translation/DeepLTextTranslation.js +4 -3
  56. package/dist/text-translation/DeepLTextTranslation.js.map +1 -1
  57. package/dist/text-translation/NLLBTextTranslation.js +7 -6
  58. package/dist/text-translation/NLLBTextTranslation.js.map +1 -1
  59. package/dist/utilities/StringUtilities.d.ts +1 -1
  60. package/dist/utilities/StringUtilities.js +5 -5
  61. package/dist/utilities/StringUtilities.js.map +1 -1
  62. package/dist/utilities/Timeline.d.ts +2 -3
  63. package/dist/utilities/Timeline.js +96 -147
  64. package/dist/utilities/Timeline.js.map +1 -1
  65. package/dist/utilities/WikipediaReader.js +3 -3
  66. package/dist/utilities/WikipediaReader.js.map +1 -1
  67. package/dist/voice-activity-detection/WebRtcVAD.js +4 -4
  68. package/dist/voice-activity-detection/WebRtcVAD.js.map +1 -1
  69. package/docs/Tasklist.md +1 -6
  70. package/package.json +13 -11
  71. package/src/alignment/SemanticTextAlignment.ts +6 -7
  72. package/src/api/Alignment.ts +2 -2
  73. package/src/api/Recognition.ts +2 -2
  74. package/src/api/SpeechTranslation.ts +2 -2
  75. package/src/api/Synthesis.ts +7 -7
  76. package/src/api/TimelineTranslationAlignment.ts +2 -2
  77. package/src/api/TranslationAlignment.ts +2 -2
  78. package/src/audio/AudioPlayer.ts +2 -2
  79. package/src/cli/CLI.ts +4 -4
  80. package/src/denoising/RNNoise.ts +5 -5
  81. package/src/dsp/FFT.ts +22 -19
  82. package/src/dsp/Rubberband.ts +5 -5
  83. package/src/dsp/Sonic.ts +6 -6
  84. package/src/dsp/SpeexResampler.ts +11 -11
  85. package/src/nlp/EspeakPhonemizer.ts +1 -1
  86. package/src/nlp/Segmentation.ts +146 -111
  87. package/src/nlp/TextNormalizer.ts +94 -52
  88. package/src/recognition/AmazonTranscribeSTT.ts +2 -2
  89. package/src/recognition/SileroSTT.ts +2 -2
  90. package/src/recognition/WhisperSTT.ts +4 -18
  91. package/src/subtitles/Subtitles.ts +69 -8
  92. package/src/synthesis/ElevenLabsTTS.ts +12 -11
  93. package/src/synthesis/EspeakTTS.ts +9 -8
  94. package/src/synthesis/KokoroTTS.ts +24 -12
  95. package/src/synthesis/SvoxPicoTTS.ts +26 -23
  96. package/src/text-translation/DeepLTextTranslation.ts +4 -3
  97. package/src/text-translation/NLLBTextTranslation.ts +8 -6
  98. package/src/typings/Fillers.d.ts +0 -16
  99. package/src/utilities/StringUtilities.ts +5 -6
  100. package/src/utilities/Timeline.ts +100 -170
  101. package/src/utilities/WikipediaReader.ts +3 -3
  102. package/src/voice-activity-detection/WebRtcVAD.ts +5 -4
  103. package/dist/nlp/CompromiseNLP.d.ts +0 -15
  104. package/dist/nlp/CompromiseNLP.js +0 -79
  105. package/dist/nlp/CompromiseNLP.js.map +0 -1
  106. package/dist/utilities/WasmMemoryManager.d.ts +0 -153
  107. package/dist/utilities/WasmMemoryManager.js +0 -440
  108. package/dist/utilities/WasmMemoryManager.js.map +0 -1
  109. package/src/nlp/CompromiseNLP.ts +0 -119
  110. package/src/utilities/WasmMemoryManager.ts +0 -566
@@ -1,6 +1,6 @@
1
1
  import { cloneRawAudio } from '../audio/AudioUtilities.js';
2
2
  import { concatFloat32Arrays, isWasmSimdSupported } from '../utilities/Utilities.js';
3
- import { WasmMemoryManager } from '../utilities/WasmMemoryManager.js';
3
+ import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager';
4
4
  let speexResamplerInstance;
5
5
  export async function resampleAudioSpeex(rawAudio, outSampleRate, quality = 0) {
6
6
  const channelCount = rawAudio.audioChannels.length;
@@ -17,14 +17,14 @@ export async function resampleAudioSpeex(rawAudio, outSampleRate, quality = 0) {
17
17
  };
18
18
  }
19
19
  const m = await getSpeexResamplerInstance();
20
- const wasmMemory = new WasmMemoryManager(m);
20
+ const wasmHeap = wrapEmscriptenModuleHeap(m);
21
21
  function speexResultCodeToString(resultCode) {
22
22
  const errorStrPtr = m._speex_resampler_strerror(resultCode);
23
- const errorStrRef = wasmMemory.wrapUint8Array(errorStrPtr, 1024);
24
- const message = errorStrRef.readAsNullTerminatedUtf8String();
23
+ const messageRef = wasmHeap.wrapNullTerminatedUtf8String(errorStrPtr);
24
+ const message = messageRef.value;
25
25
  return message;
26
26
  }
27
- const initErrRef = wasmMemory.allocInt32();
27
+ const initErrRef = wasmHeap.allocInt32();
28
28
  const resamplerStateAddress = m._speex_resampler_init(channelCount, inSampleRate, outSampleRate, quality, initErrRef.address);
29
29
  let resultCode = initErrRef.value;
30
30
  if (resultCode != 0) {
@@ -33,10 +33,10 @@ export async function resampleAudioSpeex(rawAudio, outSampleRate, quality = 0) {
33
33
  const inputLatency = m._speex_resampler_get_input_latency(resamplerStateAddress);
34
34
  const outputLatency = m._speex_resampler_get_output_latency(resamplerStateAddress);
35
35
  const maxChunkSize = 2 ** 20;
36
- const inputChunkSampleCountRef = wasmMemory.allocInt32();
37
- const outputChunkSampleCountRef = wasmMemory.allocInt32();
38
- const inputChunkSamplesRef = wasmMemory.allocFloat32Array(maxChunkSize * 2);
39
- const outputChunkSamplesRef = wasmMemory.allocFloat32Array(Math.floor(maxChunkSize * sampleRateRatio) * 2);
36
+ const inputChunkSampleCountRef = wasmHeap.allocInt32();
37
+ const outputChunkSampleCountRef = wasmHeap.allocInt32();
38
+ const inputChunkSamplesRef = wasmHeap.allocFloat32Array(maxChunkSize * 2);
39
+ const outputChunkSamplesRef = wasmHeap.allocFloat32Array(Math.floor(maxChunkSize * sampleRateRatio) * 2);
40
40
  const resampledAudioChunksForChannels = [];
41
41
  for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) {
42
42
  resampledAudioChunksForChannels.push([]);
@@ -46,7 +46,7 @@ export async function resampleAudioSpeex(rawAudio, outSampleRate, quality = 0) {
46
46
  const isLastChunk = readOffset + maxChunkSize >= totalSampleCount;
47
47
  const inputPaddingSize = isLastChunk ? inputLatency : 0;
48
48
  const maxSamplesToRead = Math.min(maxChunkSize, totalSampleCount - readOffset) + inputPaddingSize;
49
- const maxSamplesToWrite = outputChunkSamplesRef.length;
49
+ const maxSamplesToWrite = outputChunkSamplesRef.elementCount;
50
50
  const inputChunkSamplesForChannel = rawAudio.audioChannels[channelIndex].slice(readOffset, readOffset + maxSamplesToRead);
51
51
  inputChunkSampleCountRef.value = maxSamplesToRead;
52
52
  outputChunkSampleCountRef.value = maxSamplesToWrite;
@@ -63,7 +63,7 @@ export async function resampleAudioSpeex(rawAudio, outSampleRate, quality = 0) {
63
63
  }
64
64
  }
65
65
  m._speex_resampler_destroy(resamplerStateAddress);
66
- wasmMemory.freeAll();
66
+ wasmHeap.freeAll();
67
67
  const resampledAudio = {
68
68
  audioChannels: [],
69
69
  sampleRate: outSampleRate
@@ -1 +1 @@
1
- {"version":3,"file":"SpeexResampler.js","sourceRoot":"","sources":["../../src/dsp/SpeexResampler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,aAAa,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACpF,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAA;AAErE,IAAI,sBAA2B,CAAA;AAE/B,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAkB,EAAE,aAAqB,EAAE,OAAO,GAAG,CAAC;IAC9F,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAA;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAA;IAExC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IACzD,MAAM,eAAe,GAAG,aAAa,GAAG,YAAY,CAAA;IAEpD,IAAI,YAAY,KAAK,aAAa,EAAE,CAAC;QACpC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,gBAAgB,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACN,GAAG,aAAa,CAAC,QAAQ,CAAC;YAC1B,UAAU,EAAE,aAAa;SACb,CAAA;IACd,CAAC;IAED,MAAM,CAAC,GAAG,MAAM,yBAAyB,EAAE,CAAA;IAC3C,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAA;IAE3C,SAAS,uBAAuB,CAAC,UAAkB;QAClD,MAAM,WAAW,GAAG,CAAC,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAA;QAC3D,MAAM,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAChE,MAAM,OAAO,GAAG,WAAW,CAAC,8BAA8B,EAAE,CAAA;QAE5D,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,UAAU,CAAC,UAAU,EAAE,CAAA;IAC1C,MAAM,qBAAqB,GAAG,CAAC,CAAC,qBAAqB,CAAC,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,CAAA;IAC7H,IAAI,UAAU,GAAG,UAAU,CAAC,KAAK,CAAA;IAEjC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,uDAAuD,UAAU,KAAK,uBAAuB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC7H,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,CAAC,kCAAkC,CAAC,qBAAqB,CAAC,CAAA;IAChF,MAAM,aAAa,GAAG,CAAC,CAAC,mCAAmC,CAAC,qBAAqB,CAAC,CAAA;IAElF,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,CAAA;IAE5B,MAAM,wBAAwB,GAAG,UAAU,CAAC,UAAU,EAAE,CAAA;IACxD,MAAM,yBAAyB,GAAG,UAAU,CAAC,UAAU,EAAE,CAAA;IAEzD,MAAM,oBAAoB,GAAG,UAAU,CAAC,iBAAiB,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;IAC3E,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IAE1G,MAAM,+BAA+B,GAAqB,EAAE,CAAA;IAE5D,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;QACxE,+BAA+B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;QACxE,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,gBAAgB,GAAG,CAAC;YACzD,MAAM,WAAW,GAAG,UAAU,GAAG,YAAY,IAAI,gBAAgB,CAAA;YAEjE,MAAM,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;YACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,GAAG,UAAU,CAAC,GAAG,gBAAgB,CAAA;YAEjG,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,MAAM,CAAA;YAEtD,MAAM,2BAA2B,GAAG,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,gBAAgB,CAAC,CAAA;YAEzH,wBAAwB,CAAC,KAAK,GAAG,gBAAgB,CAAA;YACjD,yBAAyB,CAAC,KAAK,GAAG,iBAAiB,CAAA;YAEnD,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAA;YAC1D,UAAU,GAAG,CAAC,CAAC,8BAA8B,CAAC,qBAAqB,EAAE,YAAY,EAAE,oBAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,EAAE,yBAAyB,CAAC,OAAO,CAAC,CAAA;YAEpN,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,qDAAqD,UAAU,KAAK,uBAAuB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;YAC3H,CAAC;YAED,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,KAAK,CAAA;YACvD,MAAM,mBAAmB,GAAG,yBAAyB,CAAC,KAAK,CAAA;YAE3D,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;YAEtF,+BAA+B,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;YAEzE,UAAU,IAAI,gBAAgB,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,CAAC,CAAC,wBAAwB,CAAC,qBAAqB,CAAC,CAAA;IACjD,UAAU,CAAC,OAAO,EAAE,CAAA;IAEpB,MAAM,cAAc,GAAa;QAChC,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,aAAa;KACzB,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAElG,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3F,CAAC;IAED,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB;IAC9C,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC7B,IAAI,MAAM,mBAAmB,EAAE,EAAE,CAAC;YACjC,MAAM,EAAE,OAAO,EAAE,yBAAyB,EAAE,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAA;YACpG,sBAAsB,GAAG,MAAM,yBAAyB,EAAE,CAAA;QAC3D,CAAC;aAAM,CAAC;YACP,MAAM,EAAE,OAAO,EAAE,yBAAyB,EAAE,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAC/F,sBAAsB,GAAG,MAAM,yBAAyB,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,sBAAsB,CAAA;AAC9B,CAAC"}
1
+ {"version":3,"file":"SpeexResampler.js","sourceRoot":"","sources":["../../src/dsp/SpeexResampler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,aAAa,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACpF,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAA;AAE5D,IAAI,sBAA2B,CAAA;AAE/B,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAkB,EAAE,aAAqB,EAAE,OAAO,GAAG,CAAC;IAC9F,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAA;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAA;IAExC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IACzD,MAAM,eAAe,GAAG,aAAa,GAAG,YAAY,CAAA;IAEpD,IAAI,YAAY,KAAK,aAAa,EAAE,CAAC;QACpC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,gBAAgB,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACN,GAAG,aAAa,CAAC,QAAQ,CAAC;YAC1B,UAAU,EAAE,aAAa;SACb,CAAA;IACd,CAAC;IAED,MAAM,CAAC,GAAG,MAAM,yBAAyB,EAAE,CAAA;IAC3C,MAAM,QAAQ,GAAG,wBAAwB,CAAC,CAAC,CAAC,CAAA;IAE5C,SAAS,uBAAuB,CAAC,UAAkB;QAClD,MAAM,WAAW,GAAG,CAAC,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAA;QAC3D,MAAM,UAAU,GAAG,QAAQ,CAAC,4BAA4B,CAAC,WAAW,CAAC,CAAA;QACrE,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAA;QAEhC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAA;IACxC,MAAM,qBAAqB,GAAG,CAAC,CAAC,qBAAqB,CAAC,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,CAAA;IAC7H,IAAI,UAAU,GAAG,UAAU,CAAC,KAAK,CAAA;IAEjC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,uDAAuD,UAAU,KAAK,uBAAuB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC7H,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,CAAC,kCAAkC,CAAC,qBAAqB,CAAC,CAAA;IAChF,MAAM,aAAa,GAAG,CAAC,CAAC,mCAAmC,CAAC,qBAAqB,CAAC,CAAA;IAElF,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,CAAA;IAE5B,MAAM,wBAAwB,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAA;IACtD,MAAM,yBAAyB,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAA;IAEvD,MAAM,oBAAoB,GAAG,QAAQ,CAAC,iBAAiB,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;IACzE,MAAM,qBAAqB,GAAG,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IAExG,MAAM,+BAA+B,GAAqB,EAAE,CAAA;IAE5D,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;QACxE,+BAA+B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;QACxE,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,gBAAgB,GAAG,CAAC;YACzD,MAAM,WAAW,GAAG,UAAU,GAAG,YAAY,IAAI,gBAAgB,CAAA;YAEjE,MAAM,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;YACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,GAAG,UAAU,CAAC,GAAG,gBAAgB,CAAA;YAEjG,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,YAAY,CAAA;YAE5D,MAAM,2BAA2B,GAAG,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,gBAAgB,CAAC,CAAA;YAEzH,wBAAwB,CAAC,KAAK,GAAG,gBAAgB,CAAA;YACjD,yBAAyB,CAAC,KAAK,GAAG,iBAAiB,CAAA;YAEnD,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAA;YAC1D,UAAU,GAAG,CAAC,CAAC,8BAA8B,CAAC,qBAAqB,EAAE,YAAY,EAAE,oBAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,EAAE,yBAAyB,CAAC,OAAO,CAAC,CAAA;YAEpN,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,qDAAqD,UAAU,KAAK,uBAAuB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;YAC3H,CAAC;YAED,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,KAAK,CAAA;YACvD,MAAM,mBAAmB,GAAG,yBAAyB,CAAC,KAAK,CAAA;YAE3D,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;YAEtF,+BAA+B,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;YAEzE,UAAU,IAAI,gBAAgB,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,CAAC,CAAC,wBAAwB,CAAC,qBAAqB,CAAC,CAAA;IACjD,QAAQ,CAAC,OAAO,EAAE,CAAA;IAElB,MAAM,cAAc,GAAa;QAChC,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,aAAa;KACzB,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAElG,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3F,CAAC;IAED,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB;IAC9C,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC7B,IAAI,MAAM,mBAAmB,EAAE,EAAE,CAAC;YACjC,MAAM,EAAE,OAAO,EAAE,yBAAyB,EAAE,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAA;YACpG,sBAAsB,GAAG,MAAM,yBAAyB,EAAE,CAAA;QAC3D,CAAC;aAAM,CAAC;YACP,MAAM,EAAE,OAAO,EAAE,yBAAyB,EAAE,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAC/F,sBAAsB,GAAG,MAAM,yBAAyB,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,sBAAsB,CAAA;AAC9B,CAAC"}
@@ -46,7 +46,7 @@ export async function phonemizeText(text, voice, substitutionMap) {
46
46
  .replaceAll(')', ', ')
47
47
  .replaceAll('«', ', ')
48
48
  .replaceAll('»', ', ');
49
- const segmentedText = await Segmentation.parse(text, voice);
49
+ const segmentedText = await Segmentation.parseTextAndConvertToFragmentObjects(text, voice);
50
50
  const preparedPhrases = [];
51
51
  const phraseBreakers = [];
52
52
  for (const sentence of segmentedText) {
@@ -1 +1 @@
1
- {"version":3,"file":"EspeakPhonemizer.js","sourceRoot":"","sources":["../../src/nlp/EspeakPhonemizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,2BAA2B,CAAA;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA;AAEjD,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,WAAmB,EAAE,eAAuC,EAAE,MAAM,GAAG,IAAI;IACpI,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;IAE/E,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAE5C,MAAM,OAAO,GAAiB,EAAE,CAAA;IAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;QAC7E,MAAM,YAAY,GAAG,aAAa,CAAC,WAAW,CAAC,CAAA;QAE/C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACpD,MAAM,KAAK,GAAe,EAAE,CAAA;QAE5B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAElC,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/D,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC1C,CAAC;qBAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;gBAC/E,CAAC;qBAAM,CAAC;oBACP,OAAO,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBAClD,CAAC;YACF,CAAC,CAAC,CAAA;YAEF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzB,CAAC;QACF,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,eAAuC;IACvG,IAAI,GAAG,IAAI;SACR,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAExB,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC3D,MAAM,eAAe,GAAa,EAAE,CAAA;IACpC,MAAM,cAAc,GAAa,EAAE,CAAA;IAEnC,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;YACjG,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAErF,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YAExC,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAEhE,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;gBAChC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzE,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;qBAAM,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChF,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;qBAAM,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,IAAI,QAAQ,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;oBACxC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC9B,CAAC;qBAAM,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,eAAe,EAAE,KAAK,EAAE,cAAc,EAAE,eAAe,CAAC,CAAA;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAiB,EAAE,KAAa,EAAE,cAAwB,EAAE,eAAuC;IACzI,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA,CAAC,6CAA6C;IAEvF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACxC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEnD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACjC,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAElC,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/D,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC1C,CAAC;qBAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;gBAC/E,CAAC;qBAAM,CAAC;oBACP,OAAO,CAAC,OAAO,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC,CAAC,CAAA;YAEF,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;YAC1F,CAAC;YAED,OAAO,YAAY,CAAA;QACpB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC9C,GAAG,CAAC,OAAO,CAAC,CAAA;QACZ,GAAG,CAAC,QAAQ,CAAC,CAAA;QACb,GAAG,CAAC,cAAc,CAAC,CAAA;QAEnB,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,MAAM,4CAA4C,cAAc,CAAC,MAAM,GAAG,CAAC,CAAA;IACrI,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE5C,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;IACvC,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,iBAA+B;IAC3E,IAAI,mBAAmB,GAAiB,CAAC,EAAE,CAAC,CAAA;IAE5C,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;QAClD,mBAAmB,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAA;QAE7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC9D,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAEjD,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,mBAAmB,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE3E,OAAO,mBAAmB,CAAA;AAC3B,CAAC"}
1
+ {"version":3,"file":"EspeakPhonemizer.js","sourceRoot":"","sources":["../../src/nlp/EspeakPhonemizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,2BAA2B,CAAA;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAA;AAEjD,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,WAAmB,EAAE,eAAuC,EAAE,MAAM,GAAG,IAAI;IACpI,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;IAE/E,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAE5C,MAAM,OAAO,GAAiB,EAAE,CAAA;IAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;QAC7E,MAAM,YAAY,GAAG,aAAa,CAAC,WAAW,CAAC,CAAA;QAE/C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACpD,MAAM,KAAK,GAAe,EAAE,CAAA;QAE5B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACrE,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAElC,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/D,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC1C,CAAC;qBAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;gBAC/E,CAAC;qBAAM,CAAC;oBACP,OAAO,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBAClD,CAAC;YACF,CAAC,CAAC,CAAA;YAEF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzB,CAAC;QACF,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,eAAuC;IACvG,IAAI,GAAG,IAAI;SACR,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAExB,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,oCAAoC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC1F,MAAM,eAAe,GAAa,EAAE,CAAA;IACpC,MAAM,cAAc,GAAa,EAAE,CAAA;IAEnC,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;YACjG,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAErF,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YAExC,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAEhE,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;gBAChC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzE,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;qBAAM,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChF,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;qBAAM,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,IAAI,QAAQ,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;oBACxC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC9B,CAAC;qBAAM,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzB,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,eAAe,EAAE,KAAK,EAAE,cAAc,EAAE,eAAe,CAAC,CAAA;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAiB,EAAE,KAAa,EAAE,cAAwB,EAAE,eAAuC;IACzI,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA,CAAC,6CAA6C;IAEvF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACxC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEnD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACjC,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAElC,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/D,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC1C,CAAC;qBAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;gBAC/E,CAAC;qBAAM,CAAC;oBACP,OAAO,CAAC,OAAO,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC,CAAC,CAAA;YAEF,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;YAC1F,CAAC;YAED,OAAO,YAAY,CAAA;QACpB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC9C,GAAG,CAAC,OAAO,CAAC,CAAA;QACZ,GAAG,CAAC,QAAQ,CAAC,CAAA;QACb,GAAG,CAAC,cAAc,CAAC,CAAA;QAEnB,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,MAAM,4CAA4C,cAAc,CAAC,MAAM,GAAG,CAAC,CAAA;IACrI,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE5C,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;IACvC,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,iBAA+B;IAC3E,IAAI,mBAAmB,GAAiB,CAAC,EAAE,CAAC,CAAA;IAE5C,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;QAClD,mBAAmB,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAA;QAE7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC9D,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAEjD,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3C,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,mBAAmB,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE3E,OAAO,mBAAmB,CAAA;AAC3B,CAAC"}
@@ -1,13 +1,23 @@
1
1
  import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js';
2
- export declare const wordCharacterPattern: RegExp;
3
- export declare const punctuationPattern: RegExp;
2
+ import * as TextSegmentation from '@echogarden/text-segmentation';
3
+ export declare const wordCharacterRegExp: RegExp;
4
+ export declare const emojiCharacterRegExp: RegExp;
5
+ export declare const punctuationRegExp: RegExp;
4
6
  export declare const phraseSeparators: string[];
5
7
  export declare const symbolWords: string[];
6
8
  export declare function isWordOrSymbolWord(str: string): boolean;
7
9
  export declare function isSymbolWord(str: string): boolean;
8
10
  export declare function isWord(str: string): boolean;
9
- export declare function isPunctuation(str: string): boolean;
11
+ export declare function includesPunctuation(str: string): boolean;
12
+ export declare function includesEmoji(str: string): boolean;
10
13
  export declare function isWhitespace(str: string): boolean;
14
+ export declare function splitToParagraphs(text: string, paragraphBreakType: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing): string[];
15
+ export declare function splitToLines(text: string): string[];
16
+ export declare function parseText(text: string, langCode: string): Promise<TextSegmentation.SegmentationResult>;
17
+ export declare function splitToWords(text: string, langCode: string): Promise<TextSegmentation.WordSequence>;
18
+ export declare function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing): string;
19
+ export declare function splitToFragments(text: string, maxFragmentLength: number, langCode: string, preserveSentences?: boolean, preservePhrases?: boolean): Promise<Fragment[]>;
20
+ export declare function parseTextAndConvertToFragmentObjects(text: string, langCode: string): Promise<Sentence[]>;
11
21
  export declare class Sentence {
12
22
  phrases: Phrase[];
13
23
  readonly isSentenceFinalizer = true;
@@ -39,10 +49,3 @@ export declare class Fragment {
39
49
  get isNonempty(): boolean;
40
50
  get lastSegment(): Segment | undefined;
41
51
  }
42
- export declare function splitToFragments(text: string, maxFragmentLength: number, langCode: string, preserveSentences?: boolean, preservePhrases?: boolean): Promise<Fragment[]>;
43
- export declare function parse(text: string, langCode: string): Promise<Sentence[]>;
44
- export declare function splitToSentences(text: string, langCode: string): string[];
45
- export declare function splitToWords(text: string, langCode: string): Promise<string[]>;
46
- export declare function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespaceProcessingMethod: WhitespaceProcessing): string[];
47
- export declare function applyWhitespaceProcessing(text: string, whitespaceProcessingMethod: WhitespaceProcessing): string;
48
- export declare function splitToLines(text: string): string[];
@@ -1,74 +1,109 @@
1
- import * as CldrSegmentation from 'cldr-segmentation';
2
- import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js';
3
1
  import { sumArray, logToStderr } from '../utilities/Utilities.js';
4
2
  import { getShortLanguageCode } from '../utilities/Locale.js';
3
+ import { includesAnyOf, splitAndPreserveSeparators } from '../utilities/StringUtilities.js';
4
+ import * as TextSegmentation from '@echogarden/text-segmentation';
5
+ import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js';
5
6
  import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js';
6
- import { includesAnyOf, indexOfAnyOf } from '../utilities/StringUtilities.js';
7
7
  const log = logToStderr;
8
- export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u;
9
- export const punctuationPattern = /[\p{Punctuation}]/u;
10
- export const phraseSeparators = [',', ';', ':', ',', '、'];
8
+ export const wordCharacterRegExp = /[\p{Letter}\p{Number}]/u;
9
+ export const emojiCharacterRegExp = /[\p{Emoji}]/u;
10
+ export const punctuationRegExp = /[\p{Punctuation}]/u;
11
+ export const phraseSeparators = [',', '、', ',', '،', ';', ';', ':', ':', '—'];
11
12
  export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾'];
13
+ ///////////////////////////////////////////////////////////////////////////////////////////////
14
+ // Predicates
15
+ ///////////////////////////////////////////////////////////////////////////////////////////////
12
16
  export function isWordOrSymbolWord(str) {
13
- return isWord(str) || symbolWords.includes(str);
17
+ return isWord(str) || includesEmoji(str) || symbolWords.includes(str);
14
18
  }
15
19
  export function isSymbolWord(str) {
16
20
  return symbolWords.includes(str.trim());
17
21
  }
18
22
  export function isWord(str) {
19
- return wordCharacterPattern.test(str.trim());
23
+ return wordCharacterRegExp.test(str.trim());
20
24
  }
21
- export function isPunctuation(str) {
22
- return punctuationPattern.test(str.trim());
25
+ export function includesPunctuation(str) {
26
+ return punctuationRegExp.test(str.trim());
27
+ }
28
+ export function includesEmoji(str) {
29
+ return emojiCharacterRegExp.test(str.trim());
23
30
  }
24
31
  export function isWhitespace(str) {
25
32
  return str.trim().length === 0;
26
33
  }
27
- export class Sentence {
28
- phrases = [];
29
- isSentenceFinalizer = true;
30
- get length() { return sumArray(this.phrases, (phrase) => phrase.length); }
31
- get text() { return this.phrases.reduce((result, phrase) => result + phrase.text, ''); }
34
+ ///////////////////////////////////////////////////////////////////////////////////////////////
35
+ // Paragraph, line, sentence, phrase, and word segmentation
36
+ ///////////////////////////////////////////////////////////////////////////////////////////////
37
+ export function splitToParagraphs(text, paragraphBreakType, whitespaceProcessingMethod) {
38
+ let paragraphs = [];
39
+ if (paragraphBreakType === 'single') {
40
+ paragraphs = splitAndPreserveSeparators(text, /(\r?\n)+/g);
41
+ }
42
+ else if (paragraphBreakType === 'double') {
43
+ paragraphs = splitAndPreserveSeparators(text, /(\r?\n)(\r?\n)+/g);
44
+ }
45
+ else {
46
+ throw new Error(`Invalid paragraph break type: '${paragraphBreakType}'`);
47
+ }
48
+ paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p, whitespaceProcessingMethod));
49
+ paragraphs = paragraphs.filter(p => p.length > 0);
50
+ return paragraphs;
32
51
  }
33
- export class Phrase {
34
- words = [];
35
- get length() { return sumArray(this.words, (word) => word.length); }
36
- get text() { return this.words.reduce((result, word) => result + word.text, ''); }
37
- get lastWord() {
38
- if (this.words.length == 0) {
39
- return undefined;
52
+ export function splitToLines(text) {
53
+ return splitAndPreserveSeparators(text, /\r?\n/g);
54
+ }
55
+ export async function parseText(text, langCode) {
56
+ const shortLangCode = getShortLanguageCode(langCode || '');
57
+ const wordSequence = await splitToWords(text, shortLangCode);
58
+ const parsedText = await TextSegmentation.segmentWordSequence(wordSequence);
59
+ return parsedText;
60
+ }
61
+ export async function splitToWords(text, langCode) {
62
+ const shortLangCode = getShortLanguageCode(langCode || '');
63
+ if (shortLangCode === 'zh' || shortLangCode === 'cmn' || shortLangCode === 'ja') {
64
+ let wordArray = [];
65
+ if (shortLangCode === 'zh' || shortLangCode === 'cmn') {
66
+ wordArray = await splitChineseTextToWords_Jieba(text);
40
67
  }
41
- return this.words[this.words.length - 1];
68
+ else {
69
+ wordArray = await splitJapaneseTextToWords_Kuromoji(text);
70
+ }
71
+ const wordSequence = new TextSegmentation.WordSequence();
72
+ let offset = 0;
73
+ for (const wordText of wordArray) {
74
+ const startOffset = offset;
75
+ const endOffset = startOffset + wordText.length;
76
+ const isNonPunctuation = isWordOrSymbolWord(wordText);
77
+ wordSequence.addWord(wordText, startOffset, isNonPunctuation);
78
+ offset = endOffset;
79
+ }
80
+ return wordSequence;
42
81
  }
43
- get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false; }
44
- }
45
- export class Word {
46
- text;
47
- isSentenceFinalizer;
48
- constructor(text, isSentenceFinalizer) {
49
- this.text = text;
50
- this.isSentenceFinalizer = isSentenceFinalizer;
82
+ else {
83
+ return TextSegmentation.splitToWords(text, { language: langCode });
51
84
  }
52
- get containsOnlyPunctuation() { return !wordCharacterPattern.test(this.text) && !this.isSymbolWord; }
53
- get isSymbolWord() { return symbolWords.includes(this.text); }
54
- get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators); }
55
- get length() { return this.text.length; }
56
85
  }
57
- export class Fragment {
58
- segments = [];
59
- get length() { return sumArray(this.segments, (phrase) => phrase.length); }
60
- get text() { return this.segments.reduce((result, segment) => result + segment.text, ''); }
61
- get isEmpty() { return this.length == 0; }
62
- get isNonempty() { return !this.isEmpty; }
63
- get lastSegment() {
64
- if (this.isEmpty) {
65
- return undefined;
66
- }
67
- return this.segments[this.segments.length - 1];
86
+ export function applyWhitespaceProcessing(text, whitespaceProcessingMethod) {
87
+ if (whitespaceProcessingMethod === 'removeLineBreaks') {
88
+ return text.trim().replaceAll(/(\r?\n)+/g, ' ');
89
+ }
90
+ else if (whitespaceProcessingMethod === 'collapse') {
91
+ return text.trim().replaceAll(/\s+/g, ' ');
92
+ }
93
+ else if (whitespaceProcessingMethod === 'preserve') {
94
+ return text;
95
+ }
96
+ else {
97
+ throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`);
68
98
  }
69
99
  }
100
+ ///////////////////////////////////////////////////////////////////////////////////////////////
101
+ // Fragment segmentation
102
+ //
103
+ // Used to split text to fragments, to fit particular size constraints.
104
+ ///////////////////////////////////////////////////////////////////////////////////////////////
70
105
  export async function splitToFragments(text, maxFragmentLength, langCode, preserveSentences = true, preservePhrases = true) {
71
- const parsedText = await parse(text, langCode);
106
+ const parsedText = await parseTextAndConvertToFragmentObjects(text, langCode);
72
107
  const fragments = [];
73
108
  let currentFragment = new Fragment();
74
109
  const remainingCharactersInCurrentFragment = () => maxFragmentLength - currentFragment.length;
@@ -120,79 +155,67 @@ export async function splitToFragments(text, maxFragmentLength, langCode, preser
120
155
  createNewFragmentIfNeeded();
121
156
  return fragments;
122
157
  }
123
- export async function parse(text, langCode) {
124
- const sentencesText = splitToSentences(text, langCode);
158
+ export async function parseTextAndConvertToFragmentObjects(text, langCode) {
159
+ const segmentedText = await parseText(text, langCode);
125
160
  const sentences = [];
126
- for (const sentenceText of sentencesText) {
161
+ for (const sentenceEntry of segmentedText.sentences) {
127
162
  const sentence = new Sentence();
128
- let currentPhrase = new Phrase();
129
- const wordTexts = await splitToWords(sentenceText, langCode);
130
- for (let wordIndex = 0; wordIndex < wordTexts.length; wordIndex++) {
131
- const word = new Word(wordTexts[wordIndex], wordIndex == wordTexts.length - 1);
132
- if (word.isPhraseSeperator) {
133
- const separatorIndex = indexOfAnyOf(word.text, phraseSeparators);
134
- currentPhrase.words.push(new Word(word.text.substring(0, separatorIndex + 1), word.isSentenceFinalizer));
135
- sentence.phrases.push(currentPhrase);
136
- currentPhrase = new Phrase();
137
- currentPhrase.words.push(new Word(word.text.substring(separatorIndex + 1), false));
163
+ for (const phraseEntry of sentenceEntry.phrases) {
164
+ const phrase = new Phrase();
165
+ for (const wordEntry of phraseEntry.words.entries) {
166
+ const isSentenceFinalizer = wordEntry === sentenceEntry.words.lastEntry;
167
+ const word = new Word(wordEntry.text, isSentenceFinalizer);
168
+ phrase.words.push(word);
138
169
  }
139
- else {
140
- currentPhrase.words.push(word);
170
+ if (phrase.words.length > 0) {
171
+ sentence.phrases.push(phrase);
141
172
  }
142
173
  }
143
- if (currentPhrase.words.length > 0) {
144
- sentence.phrases.push(currentPhrase);
145
- }
146
174
  sentences.push(sentence);
147
175
  }
148
176
  return sentences;
149
177
  }
150
- export function splitToSentences(text, langCode) {
151
- const shortLangCode = getShortLanguageCode(langCode || '');
152
- return CldrSegmentation.sentenceSplit(text, CldrSegmentation.suppressions[shortLangCode]);
178
+ export class Sentence {
179
+ phrases = [];
180
+ isSentenceFinalizer = true;
181
+ get length() { return sumArray(this.phrases, (phrase) => phrase.length); }
182
+ get text() { return this.phrases.reduce((result, phrase) => result + phrase.text, ''); }
153
183
  }
154
- export async function splitToWords(text, langCode) {
155
- const shortLangCode = getShortLanguageCode(langCode || '');
156
- if (shortLangCode == 'zh' || shortLangCode == 'cmn') {
157
- return splitChineseTextToWords_Jieba(text, undefined, true);
158
- }
159
- else if (shortLangCode == 'ja') {
160
- return splitJapaneseTextToWords_Kuromoji(text);
161
- }
162
- else {
163
- return CldrSegmentation.wordSplit(text, CldrSegmentation.suppressions[shortLangCode]);
184
+ export class Phrase {
185
+ words = [];
186
+ get length() { return sumArray(this.words, (word) => word.length); }
187
+ get text() { return this.words.reduce((result, word) => result + word.text, ''); }
188
+ get lastWord() {
189
+ if (this.words.length == 0) {
190
+ return undefined;
191
+ }
192
+ return this.words[this.words.length - 1];
164
193
  }
194
+ get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false; }
165
195
  }
166
- export function splitToParagraphs(text, paragraphBreaks, whitespaceProcessingMethod) {
167
- let paragraphs = [];
168
- if (paragraphBreaks === 'single') {
169
- paragraphs = text.split(/(\r?\n)+/g);
170
- }
171
- else if (paragraphBreaks === 'double') {
172
- paragraphs = text.split(/(\r?\n)(\r?\n)+/g);
173
- }
174
- else {
175
- throw new Error(`Invalid paragraph break type: '${paragraphBreaks}'`);
196
+ export class Word {
197
+ text;
198
+ isSentenceFinalizer;
199
+ constructor(text, isSentenceFinalizer) {
200
+ this.text = text;
201
+ this.isSentenceFinalizer = isSentenceFinalizer;
176
202
  }
177
- paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p.trim(), whitespaceProcessingMethod));
178
- paragraphs = paragraphs.filter(p => p.length > 0);
179
- return paragraphs;
203
+ get containsOnlyPunctuation() { return !wordCharacterRegExp.test(this.text) && !this.isSymbolWord; }
204
+ get isSymbolWord() { return symbolWords.includes(this.text); }
205
+ get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators); }
206
+ get length() { return this.text.length; }
180
207
  }
181
- export function applyWhitespaceProcessing(text, whitespaceProcessingMethod) {
182
- if (whitespaceProcessingMethod === 'removeLineBreaks') {
183
- return text.replaceAll(/(\r?\n)+/g, ' ');
184
- }
185
- else if (whitespaceProcessingMethod === 'collapse') {
186
- return text.replaceAll(/\s+/g, ' ');
187
- }
188
- else if (whitespaceProcessingMethod === 'preserve') {
189
- return text;
190
- }
191
- else {
192
- throw new Error(`Invalid whitespace processing method: '${whitespaceProcessingMethod}'`);
208
+ export class Fragment {
209
+ segments = [];
210
+ get length() { return sumArray(this.segments, (phrase) => phrase.length); }
211
+ get text() { return this.segments.reduce((result, segment) => result + segment.text, ''); }
212
+ get isEmpty() { return this.length == 0; }
213
+ get isNonempty() { return !this.isEmpty; }
214
+ get lastSegment() {
215
+ if (this.isEmpty) {
216
+ return undefined;
217
+ }
218
+ return this.segments[this.segments.length - 1];
193
219
  }
194
220
  }
195
- export function splitToLines(text) {
196
- return text.split(/\r?\n/g);
197
- }
198
221
  //# sourceMappingURL=Segmentation.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"Segmentation.js","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,gBAAgB,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AAExE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAC7D,OAAO,EAAE,iCAAiC,EAAE,MAAM,2BAA2B,CAAA;AAE7E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAE7E,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AAC7D,MAAM,CAAC,MAAM,kBAAkB,GAAG,oBAAoB,CAAA;AAEtD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AACzD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEzI,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;AAChD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IACjC,OAAO,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,OAAO,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,OAAO,QAAQ;IACpB,OAAO,GAAa,EAAE,CAAA;IAEb,mBAAmB,GAAG,IAAI,CAAA;IAEnC,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEzE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;CAC/F;AAED,MAAM,OAAO,MAAM;IAClB,KAAK,GAAW,EAAE,CAAA;IAElB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEnE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAEzF,IAAI,QAAQ;QACX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzC,CAAC;IAED,IAAI,mBAAmB,KAAK,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAA,CAAC,CAAC;CACtG;AAED,MAAM,OAAO,IAAI;IACP,IAAI,CAAQ;IACrB,mBAAmB,CAAS;IAE5B,YAAY,IAAY,EAAE,mBAA4B;QACrD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;IAC/C,CAAC;IAED,IAAI,uBAAuB,KAAK,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAA,CAAC,CAAC;IAEpG,IAAI,YAAY,KAAK,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,CAAC;IAE7D,IAAI,iBAAiB,KAAK,OAAO,IAAI,CAAC,uBAAuB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA,CAAC,CAAC;IAE7G,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAA,CAAC,CAAC;CACxC;AAID,MAAM,OAAO,QAAQ;IACpB,QAAQ,GAAc,EAAE,CAAA;IAExB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAE1E,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAElG,IAAI,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA,CAAC,CAAC;IAEzC,IAAI,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAA,CAAC,CAAC;IAEzC,IAAI,WAAW;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC/C,CAAC;CACD;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,iBAAyB,EAAE,QAAgB,EAAE,iBAAiB,GAAG,IAAI,EAAE,eAAe,GAAG,IAAI;IACjJ,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAE9C,MAAM,SAAS,GAAe,EAAE,CAAA;IAChC,IAAI,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;IAEpC,MAAM,oCAAoC,GAAG,GAAG,EAAE,CAAC,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAAA;IAC7F,MAAM,yBAAyB,GAAG,GAAG,EAAE;QACtC,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAC/B,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;QACjC,CAAC;IACF,CAAC,CAAA;IAED,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,oCAAoC,EAAE,CAAA;IAE1G,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACvC,SAAQ;QACT,CAAC;QAED,IAAI,iBAAiB,EAAE,CAAC;YACvB,yBAAyB,EAAE,CAAA;YAE3B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACvC,SAAQ;YACT,CAAC;QACF,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACrC,SAAQ;YACT,CAAC;YAGD,IAAI,eAAe,EAAE,CAAC;gBACrB,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBACrC,SAAQ;gBACT,CAAC;YACF,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,mDAAmD,iBAAiB,EAAE,CAAC,CAAA;YACnI,CAAC;QACF,CAAC;IACF,CAAC;IAED,yBAAyB,EAAE,CAAA;IAE3B,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAgB;IACzD,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IACtD,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAA;QAE/B,IAAI,aAAa,GAAG,IAAI,MAAM,EAAE,CAAA;QAChC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;QAE5D,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAE9E,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA;gBAChE,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAA;gBACxG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;gBAEpC,aAAa,GAAG,IAAI,MAAM,EAAE,CAAA;gBAC5B,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;YACnF,CAAC;iBAAM,CAAC;gBACP,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC/B,CAAC;QACF,CAAC;QAED,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QACrC,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IAC9D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,OAAO,gBAAgB,CAAC,aAAa,CAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAA;AAC1F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IAChE,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,IAAI,aAAa,IAAI,IAAI,IAAI,aAAa,IAAI,KAAK,EAAE,CAAC;QACrD,OAAO,6BAA6B,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IAC5D,CAAC;SAAM,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,iCAAiC,CAAC,IAAI,CAAC,CAAA;IAC/C,CAAC;SAAM,CAAC;QACP,OAAO,gBAAgB,CAAC,SAAS,CAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAA;IACtF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,eAAmC,EAAE,0BAAgD;IACpI,IAAI,UAAU,GAAa,EAAE,CAAA;IAE7B,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;QAClC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;SAAM,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;QACzC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAA;IAC5C,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kCAAkC,eAAe,GAAG,CAAC,CAAA;IACtE,CAAC;IAED,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC,CAAA;IACjG,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEjD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,IAAY,EAAE,0BAAgD;IACvG,IAAI,0BAA0B,KAAK,kBAAkB,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IACpC,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAA;IACZ,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,0CAA0C,0BAA0B,GAAG,CAAC,CAAA;IACzF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;AAC5B,CAAC"}
1
+ {"version":3,"file":"Segmentation.js","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAE7D,OAAO,EAAE,aAAa,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA;AAE3F,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AACjE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AACxE,OAAO,EAAE,iCAAiC,EAAE,MAAM,2BAA2B,CAAA;AAE7E,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,MAAM,mBAAmB,GAAG,yBAAyB,CAAA;AAC5D,MAAM,CAAC,MAAM,oBAAoB,GAAG,cAAc,CAAA;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG,oBAAoB,CAAA;AAErD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEzI,+FAA+F;AAC/F,aAAa;AACb,+FAA+F;AAC/F,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IACjC,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC9C,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AAC1C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,OAAO,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAA;AAC/B,CAAC;AAED,+FAA+F;AAC/F,2DAA2D;AAC3D,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,kBAAsC,EAAE,0BAAgD;IACvI,IAAI,UAAU,GAAa,EAAE,CAAA;IAE7B,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QACrC,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC3D,CAAC;SAAM,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAC5C,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kCAAkC,kBAAkB,GAAG,CAAC,CAAA;IACzE,CAAC;IAED,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,yBAAyB,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC,CAAA;IAC1F,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEjD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,OAAO,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAAgB;IAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IAE5D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE3E,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IAChE,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QACjF,IAAI,SAAS,GAAa,EAAE,CAAA;QAE5B,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;YACvD,SAAS,GAAG,MAAM,6BAA6B,CAAC,IAAI,CAAC,CAAA;QACtD,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,MAAM,iCAAiC,CAAC,IAAI,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,YAAY,EAAE,CAAA;QAExD,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAA;YAC1B,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAA;YAC/C,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAA;YAErD,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAA;YAE7D,MAAM,GAAG,SAAS,CAAA;QACnB,CAAC;QAED,OAAO,YAAY,CAAA;IACpB,CAAC;SAAM,CAAC;QACP,OAAO,gBAAgB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IACnE,CAAC;AACF,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,IAAY,EAAE,0BAAgD;IACvG,IAAI,0BAA0B,KAAK,kBAAkB,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC3C,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAA;IACZ,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,0CAA0C,0BAA0B,GAAG,CAAC,CAAA;IACzF,CAAC;AACF,CAAC;AAED,+FAA+F;AAC/F,wBAAwB;AACxB,EAAE;AACF,uEAAuE;AACvE,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,iBAAyB,EAAE,QAAgB,EAAE,iBAAiB,GAAG,IAAI,EAAE,eAAe,GAAG,IAAI;IACjJ,MAAM,UAAU,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAE7E,MAAM,SAAS,GAAe,EAAE,CAAA;IAChC,IAAI,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;IAEpC,MAAM,oCAAoC,GAAG,GAAG,EAAE,CAAC,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAAA;IAC7F,MAAM,yBAAyB,GAAG,GAAG,EAAE;QACtC,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAC/B,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;QACjC,CAAC;IACF,CAAC,CAAA;IAED,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,oCAAoC,EAAE,CAAA;IAE1G,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACvC,SAAQ;QACT,CAAC;QAED,IAAI,iBAAiB,EAAE,CAAC;YACvB,yBAAyB,EAAE,CAAA;YAE3B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACvC,SAAQ;YACT,CAAC;QACF,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACrC,SAAQ;YACT,CAAC;YAGD,IAAI,eAAe,EAAE,CAAC;gBACrB,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBACrC,SAAQ;gBACT,CAAC;YACF,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,mDAAmD,iBAAiB,EAAE,CAAC,CAAA;YACnI,CAAC;QACF,CAAC;IACF,CAAC;IAED,yBAAyB,EAAE,CAAA;IAE3B,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,IAAY,EAAE,QAAgB;IACxF,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAErD,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,aAAa,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAA;QAE/B,KAAK,MAAM,WAAW,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;YAE3B,KAAK,MAAM,SAAS,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnD,MAAM,mBAAmB,GAAG,SAAS,KAAK,aAAa,CAAC,KAAK,CAAC,SAAS,CAAA;gBAEvE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAA;gBAE1D,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACxB,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC9B,CAAC;QACF,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,QAAQ;IACpB,OAAO,GAAa,EAAE,CAAA;IAEb,mBAAmB,GAAG,IAAI,CAAA;IAEnC,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEzE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;CAC/F;AAED,MAAM,OAAO,MAAM;IAClB,KAAK,GAAW,EAAE,CAAA;IAElB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEnE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAEzF,IAAI,QAAQ;QACX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzC,CAAC;IAED,IAAI,mBAAmB,KAAK,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAA,CAAC,CAAC;CACtG;AAED,MAAM,OAAO,IAAI;IACP,IAAI,CAAQ;IACrB,mBAAmB,CAAS;IAE5B,YAAY,IAAY,EAAE,mBAA4B;QACrD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;IAC/C,CAAC;IAED,IAAI,uBAAuB,KAAK,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAA,CAAC,CAAC;IAEnG,IAAI,YAAY,KAAK,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,CAAC;IAE7D,IAAI,iBAAiB,KAAK,OAAO,IAAI,CAAC,uBAAuB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA,CAAC,CAAC;IAE7G,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAA,CAAC,CAAC;CACxC;AAID,MAAM,OAAO,QAAQ;IACpB,QAAQ,GAAc,EAAE,CAAA;IAExB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAE1E,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAElG,IAAI,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA,CAAC,CAAC;IAEzC,IAAI,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAA,CAAC,CAAC;IAEzC,IAAI,WAAW;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC/C,CAAC;CACD"}