echogarden 3.1.0 → 3.3.0

package/src/encodings/Utf8.ts

@@ -1,5 +1,8 @@
-import { EncodeIntoResult } from "./TextEncodingsCommon.js"
+import { EncodeIntoResult } from './TextEncodingsCommon.js'
 
+//////////////////////////////////////////////////////////////////////////////
+// UTF-8 Encoding
+//////////////////////////////////////////////////////////////////////////////
 export function encodeUtf8(text: string) {
 	const textEncoder = new TextEncoder()
 
@@ -14,104 +17,64 @@ export function encodeUtf8Into(text: string, outputArray: Uint8Array): EncodeInt
 	return result
 }
 
-export function decodeUtf8(encodedString: Uint8Array) {
-	const maxChunkLength = 2 ** 24
+export class ChunkedUtf8Encoder {
+	private readonly textEncoder = new TextEncoder()
 
-	const decoder = new ChunkedUtf8Decoder()
+	private pendingHighSurrogate = ''
 
-	for (let offset = 0; offset < encodedString.length; offset += maxChunkLength) {
-		const chunk = encodedString.subarray(offset, offset + maxChunkLength)
+	writeChunk(stringChunk: string): Uint8Array {
+		if (this.pendingHighSurrogate !== '') {
+			stringChunk = this.pendingHighSurrogate + stringChunk
 
-		decoder.writeChunk(chunk)
-	}
+			this.pendingHighSurrogate = ''
+		}
 
-	return decoder.toString()
-}
+		const lastCodeUnit = stringChunk.charCodeAt(stringChunk.length - 1)
 
-export class ChunkedUtf8Decoder {
-	private str = ''
-	private readonly textDecoder = new TextDecoder('utf-8')
+		if (lastCodeUnit >= 0xD800 && lastCodeUnit <= 0xDBFF) {
+			this.pendingHighSurrogate = stringChunk[stringChunk.length - 1]
 
-	writeChunk(chunk: Uint8Array) {
-		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
+			stringChunk = stringChunk.substring(0, stringChunk.length - 1)
+		}
 
-		this.str += decodedChunk
+		return this.textEncoder.encode(stringChunk)
 	}
 
-	toString() {
-		return this.str
+	finalize(): Uint8Array {
+		const result = this.textEncoder.encode(this.pendingHighSurrogate)
+
+		this.pendingHighSurrogate = ''
+
+		return result
 	}
 }
 
-//////////////////////////////////////////////////////////////////////////////////////////////
-// Pure JavaScript implementations
-//////////////////////////////////////////////////////////////////////////////////////////////
-function encodeUtf8Into_JS(str: string, outputArray: Uint8Array): EncodeIntoResult {
-	let readOffset = 0
-	let writeOffset = 0
-
-	while (readOffset < str.length) {
-		const charCode = str.codePointAt(readOffset++)!
-
-		if (charCode <= 0x7f) {
-			outputArray[writeOffset++] = charCode
-		} else if (charCode <= 0x7ff) {
-			outputArray[writeOffset++] = 0xc0 | (charCode >>> 6)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-		} else if (charCode <= 0xffff) {
-			outputArray[writeOffset++] = 0xe0 | (charCode >>> 12)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-		} else if (charCode <= 0x10ffff) {
-			outputArray[writeOffset++] = 0xf0 | (charCode >>> 18)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 12) & 63)
-			outputArray[writeOffset++] = 0x80 | ((charCode >>> 6) & 63)
-			outputArray[writeOffset++] = 0x80 | (charCode & 63)
-
-			readOffset++
-		}
+//////////////////////////////////////////////////////////////////////////////
+// UTF-8 Decoding
+//////////////////////////////////////////////////////////////////////////////
+export function decodeUtf8(utf8Bytes: Uint8Array) {
+	const maxChunkLength = 2 ** 24
+
+	const chunkedUtf8Decoder = new ChunkedUtf8Decoder()
+
+	let resultString = ''
+
+	for (let offset = 0; offset < utf8Bytes.length; offset += maxChunkLength) {
+		const utf8Chunk = utf8Bytes.subarray(offset, offset + maxChunkLength)
+		const stringChunk = chunkedUtf8Decoder.writeChunk(utf8Chunk)
+
+		resultString += stringChunk
 	}
 
-	return { read: str.length, written: writeOffset }
+	return resultString
 }
 
-function decodeUtf8_JS(utf8Bytes: Uint8Array): string {
-	let decodedString = ''
-
-	let readOffset = 0
-
-	while (readOffset < utf8Bytes.length) {
-		const leadByte = utf8Bytes[readOffset++]
-
-		let outputCodePoint: number
-
-		if (leadByte >>> 7 === 0) {
-			outputCodePoint = leadByte
-		} else if (leadByte >>> 5 === 6) {
-			outputCodePoint =
-				(leadByte & 31) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else if (leadByte >>> 4 === 14) {
-			outputCodePoint =
-				(leadByte & 15) << 12 |
-				(utf8Bytes[readOffset++] & 63) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else if (leadByte >>> 3 === 30) {
-			outputCodePoint =
-				(leadByte & 7) << 18 |
-				(utf8Bytes[readOffset++] & 63) << 12 |
-				(utf8Bytes[readOffset++] & 63) << 6 |
-				(utf8Bytes[readOffset++] & 63)
-		} else {
-			throw new Error(`Invalid UTF-8 stream: An invalid lead byte value encountered at position ${readOffset}`)
-		}
+export class ChunkedUtf8Decoder {
+	private readonly textDecoder = new TextDecoder('utf-8')
 
-		decodedString += String.fromCodePoint(outputCodePoint)
-	}
+	writeChunk(chunk: Uint8Array) {
+		const decodedChunk = this.textDecoder.decode(chunk, { stream: true })
 
-	if (readOffset > utf8Bytes.length) {
-		throw new Error(`UTF-8 decoding failed. Byte sequence is truncated.`)
+		return decodedChunk
 	}
-
-	return decodedString
 }

package/src/nlp/Segmentation.ts

@@ -22,7 +22,11 @@ const includesEmojiSequencePattern = anyOf(
 )
 const includesEmojiSequenceRegExp = buildRegExp(includesEmojiSequencePattern)
 
-const symbolWordsList = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥']
+const symbolWordsList = [
+	'$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷',
+	'/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾', '→', '≤', '≥', '∞'
+]
+
 const includesSymbolWordPattern = anyOf(...symbolWordsList)
 const includesSymbolWordRegExp = buildRegExp(includesSymbolWordPattern)
 

package/src/nlp/TextNormalizer.ts

@@ -31,7 +31,7 @@ export function getNormalizedFragmentsForSpeech(
 
 		//const isWordPrecedingAYear = wordsPrecedingAYear.includes(lowerCaseWord)
 		const isWordPrecedingAYear =
-			isAllLettersRegExp.test(lowerCaseWord) || isSpecialCharacterBeforeYear
+			isAllLettersOrApostropheRegExp.test(lowerCaseWord) || isSpecialCharacterBeforeYear
 
 		const followedByFourDigitYearPattern = fourDigitYearRegExp.test(nextNonWhitespaceWord)
 
@@ -284,3 +284,13 @@ const fourDigitYearRangeRegExp = buildRegExp(fourDigitYearRangePattern)
 
 const isAllLettersPattern = [inputStart, oneOrMore(unicodeProperty('Letter')), inputEnd]
 const isAllLettersRegExp = buildRegExp(isAllLettersPattern)
+
+const isAllLettersOrApostrophePattern = [
+	inputStart,
+
+	unicodeProperty('Letter'),
+	zeroOrMore(anyOf(unicodeProperty('Letter'), `'`, `’`)),
+	inputEnd,
+]
+
+const isAllLettersOrApostropheRegExp = buildRegExp(isAllLettersOrApostrophePattern)

package/src/synthesis/AzureCognitiveServicesTTS.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { concatUint8Arrays } from '../utilities/Utilities.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import { SynthesisCallbacks } from '../api/Synthesis.js'
 
 export async function synthesize(

package/src/synthesis/EspeakTTS.ts

@@ -9,7 +9,7 @@ import { Lexicon, tryGetFirstLexiconSubstitution } from '../nlp/Lexicon.js'
 import { phonemizeSentence } from '../nlp/EspeakPhonemizer.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import * as TextSegmentation from '@echogarden/text-segmentation'
 
 import { OperationCallbacks, SynthesisCallbacks } from '../api/API.js'
@@ -257,12 +257,29 @@ export async function synthesizeFragments(fragments: string[], espeakOptions: Es
 		}
 	}
 
-	const canInsertSeparators = !['roa/an', 'art/eo', 'trk/ky', 'zlw/pl', 'zle/uk'].includes(voice)
+	const separatorString = ` | `
+
+	const canInsertSeparators = ![
+		'roa/an',
+		'an',
+		'art/eo',
+		'eo',
+		'trk/ky',
+		'ky',
+		'zlw/pl',
+		'pl',
+		'zle/uk',
+		'uk',
+		'gmw/nl',
+		'nl'
+	].includes(voice)
 
 	let textWithMarkers: string
 
-	if (canInsertSeparators) {
-		textWithMarkers = `() | `
+	// Added `espeakOptions.insertSeparators` here because I'm not sure if adding the separator
+	// here is completely necessary, and it is causing issues with some languages.
+	if (espeakOptions.insertSeparators && canInsertSeparators) {
+		textWithMarkers = `()${separatorString}`
 	} else {
 		textWithMarkers = `() `
 	}
@@ -280,9 +297,7 @@ export async function synthesizeFragments(fragments: string[], espeakOptions: Es
 		}
 
 		if (espeakOptions.insertSeparators && canInsertSeparators) {
-			const separator = ` | `
-
-			textWithMarkers += `<mark name="s-${i}"/>${separator}${fragment}${separator}<mark name="e-${i}"/>`
+			textWithMarkers += `<mark name="s-${i}"/>${separatorString}${fragment}${separatorString}<mark name="e-${i}"/>`
 		} else {
 			if (fragment.endsWith('.')) {
 				fragment += ' ()'

package/src/synthesis/MicrosoftEdgeTTS.ts

@@ -8,7 +8,7 @@ import { concatUint8Arrays, getRandomHexString, logToStderr, sha256AsHex } from
 import { RawAudio, getEmptyRawAudio, getRawAudioDuration } from '../audio/AudioUtilities.js'
 import { Timer } from '../utilities/Timer.js'
 import { decodeUtf8, encodeUtf8 } from '../encodings/Utf8.js'
-import { escapeHtml } from '../encodings/HtmlEscape.js'
+import { escapeHtml } from 'standard-html-escaper'
 import { requestHttp } from 'easier-http-request'
 import { VoiceListRequestCallbacks, SynthesisCallbacks } from '../api/API.js'