echogarden 0.11.12 → 0.11.13

package/src/nlp/ChineseSegmentation.ts

@@ -0,0 +1,68 @@
+export async function splitChineseTextToWords_Jieba(text: string, fineGrained = false, useHMM = true) {
+	const jieba = await getWasmInstance()
+
+	if (!fineGrained) {
+		return jieba.cut(text, useHMM)
+	} else {
+		const results = jieba.tokenize(text, "search", useHMM)
+
+		const startOffsetsSet = new Set<number>()
+		const endOffsetsSet = new Set<number>()
+
+		for (const result of results) {
+			startOffsetsSet.add(result.start)
+			endOffsetsSet.add(result.end)
+		}
+
+		const startOffsets = Array.from(startOffsetsSet)
+		startOffsets.sort((a, b) => a - b)
+
+		const endOffsets = Array.from(endOffsetsSet)
+		endOffsets.sort((a, b) => a - b)
+
+		const words: string[] = []
+
+		for (let i = 0; i < startOffsets.length; i++) {
+			const wordStartOffset = startOffsets[i]
+
+			function getWordEndOffset() {
+				if (i < startOffsets.length - 1) {
+					const nextWordStartOffset = startOffsets[i + 1]
+
+					for (let j = 0; j < endOffsets.length - 1; j++) {
+						const currentEndOffset = endOffsets[j]
+						const nextEndOffset = endOffsets[j + 1]
+
+						if (currentEndOffset >= nextWordStartOffset) {
+							return nextWordStartOffset
+						} else if (
+							currentEndOffset > wordStartOffset &&
+							currentEndOffset < nextWordStartOffset &&
+							nextEndOffset > nextWordStartOffset) {
+
+							return currentEndOffset
+						}
+					}
+				}
+
+				return endOffsets[endOffsets.length - 1]
+			}
+
+			const wordEndOffset = getWordEndOffset()
+
+			words.push(text.substring(wordStartOffset, wordEndOffset))
+		}
+
+		return words
+	}
+}
+
+let JiebaWasmInstance: typeof import("jieba-wasm")
+async function getWasmInstance() {
+	if (!JiebaWasmInstance) {
+		const { default: JibeaWasm } = await import("jieba-wasm")
+		JiebaWasmInstance = JibeaWasm
+	}
+
+	return JiebaWasmInstance
+}

package/src/nlp/CompromiseNLP.ts

@@ -0,0 +1,113 @@
+import { getShortLanguageCode } from '../utilities/Locale.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { Lexicon } from './Lexicon.js'
+
+const log = logToStderr
+
+export async function parse(text: string): Promise<CompromiseParsedDocument> {
+	const { default: nlp } = await import('compromise')
+
+	const doc = nlp(text)
+
+	doc.compute('penn')
+
+	const jsonDoc: any[] = doc.json({ offset: true })
+
+	//log(jsonDoc)
+
+	const result: CompromiseParsedDocument = jsonDoc.map(sentence => {
+		const terms = sentence.terms
+		const parsedSentence: CompromiseParsedSentence = []
+
+		for (let termIndex = 0; termIndex < terms.length; termIndex++) {
+			const term = terms[termIndex]
+
+			const parsedTerm: CompromiseParsedTerm = {
+				text: term.text,
+				pos: term.penn,
+				tags: term.tags,
+				preText: term.pre,
+				postText: term.post,
+				startOffset: term.offset.start,
+				endOffset: term.offset.start + term.offset.length
+			}
+
+			if (parsedTerm.text == "") {
+				if (parsedSentence.length > 0) {
+					parsedSentence[parsedSentence.length - 1].postText += parsedTerm.preText + parsedTerm.postText
+				}
+			} else if (parsedTerm.tags.includes("Abbreviation") && parsedTerm.postText.startsWith(".")) {
+				parsedTerm.text += "."
+				parsedTerm.endOffset += 1
+				parsedSentence.push(parsedTerm)
+			} else {
+				parsedSentence.push(parsedTerm)
+			}
+		}
+
+		return parsedSentence
+	})
+
+	//log(result)
+
+	return result
+}
+
+export function tryMatchInLexicons(term: CompromiseParsedTerm, lexicons: Lexicon[], espeakVoice: string) {
+	const reversedLexicons = [...lexicons].reverse() // Give precedence to later lexicons
+
+	for (const lexicon of reversedLexicons) {
+		const match = tryMatchInLexicon(term, lexicon, espeakVoice)
+
+		if (match) {
+			return match
+		}
+	}
+}
+
+export function tryMatchInLexicon(term: CompromiseParsedTerm, lexicon: Lexicon, espeakVoice: string) {
+	const shortLanguageCode = getShortLanguageCode(espeakVoice)
+
+	const lexiconForLanguage = lexicon[shortLanguageCode]
+
+	if (!lexiconForLanguage) {
+		return undefined
+	}
+
+	const termText = term.text
+	const lowerCaseTermText = termText.toLocaleLowerCase()
+
+	const entry = lexiconForLanguage[lowerCaseTermText]
+
+	if (!entry) {
+		return undefined
+	}
+
+	for (const substitutionEntry of entry) {
+		if (!substitutionEntry.pos || substitutionEntry.pos.includes(term.pos)) {
+			const substitutionPhonemesText = substitutionEntry?.pronunciation?.espeak?.[espeakVoice]
+
+			if (substitutionPhonemesText) {
+				const substitutionPhonemes = substitutionPhonemesText.split(/ +/g)
+
+				return substitutionPhonemes
+			}
+		}
+	}
+
+	return undefined
+}
+
+export type CompromiseParsedDocument = CompromiseParsedSentence[]
+
+export type CompromiseParsedSentence = CompromiseParsedTerm[]
+
+export type CompromiseParsedTerm = {
+	text: string
+	pos: string
+	tags: string[]
+	preText: string,
+	postText: string,
+	startOffset: number,
+	endOffset: number
+}

package/src/nlp/EspeakPhonemizer.ts

@@ -0,0 +1,168 @@
+import * as EspeakTTS from "../synthesis/EspeakTTS.js"
+import { logToStderr } from "../utilities/Utilities.js"
+import * as Segmentation from "./Segmentation.js"
+
+const log = logToStderr
+
+export async function phonemizeSentence(sentence: string, espeakVoice: string, substitutionMap?: Map<string, string[]>, useIpa = true) {
+	const ipaString = await EspeakTTS.textToPhonemes(sentence, espeakVoice, useIpa)
+
+	const clauseStrings = ipaString.split(" | ")
+
+	const clauses: string[][][] = []
+
+	for (let clauseIndex = 0; clauseIndex < clauseStrings.length; clauseIndex++) {
+		const clauseString = clauseStrings[clauseIndex]
+
+		const wordStrings = clauseString.trim().split(/ +/g)
+		const words: string[][] = []
+
+		for (let wordIndex = 0; wordIndex < wordStrings.length; wordIndex++) {
+			const word = wordStrings[wordIndex]
+
+			let wordPhonemes = word.split("_")
+
+			wordPhonemes = wordPhonemes.flatMap(phoneme => {
+				if (!phoneme || phoneme.startsWith("(")) {
+					return []
+				} else if (phoneme.startsWith("ˈ") || phoneme.startsWith("ˌ")) {
+					return [phoneme[0], phoneme.substring(1)]
+				} else if (phoneme.endsWith("ˈ") || phoneme.endsWith("ˌ")) {
+					return [phoneme.substring(0, phoneme.length - 1), phoneme[phoneme.length - 1]]
+				} else {
+					return substitutionMap?.get(phoneme) || [phoneme]
+				}
+			})
+
+			if (wordPhonemes.length > 0) {
+				words.push(wordPhonemes)
+			}
+		}
+
+		if (words.length > 0) {
+			clauses.push(words)
+		}
+	}
+
+	return clauses
+}
+
+export async function phonemizeText(text: string, voice: string, substitutionMap?: Map<string, string[]>) {
+	text = text
+			.replaceAll("，", ",")
+			.replaceAll("、", ",")
+			.replaceAll("。", ".")
+			.replaceAll("(", ", ")
+			.replaceAll(")", ", ")
+			.replaceAll("«", ", ")
+			.replaceAll("»", ", ")
+
+	const segmentedText = await Segmentation.parse(text, voice)
+	const preparedClauses: string[] = []
+	const clauseBreakers: string[] = []
+
+	for (const sentence of segmentedText) {
+		for (const clause of sentence.phrases) {
+			const words = clause.words.filter(wordObject => Segmentation.isWordOrSymbolWord(wordObject.text))
+			const preparedClauseText = words.map(word => word.text.replace(/\./g, " ")).join(" ")
+
+			preparedClauses.push(preparedClauseText)
+
+			const trimmedClauseText = clause.text.trim()
+			const lastChar = trimmedClauseText[trimmedClauseText.length - 1]
+
+			if (clause.isSentenceFinalizer) {
+				if (trimmedClauseText.endsWith('?') || trimmedClauseText.endsWith('?"')) {
+					clauseBreakers.push('?')
+				} else if (trimmedClauseText.endsWith('!') || trimmedClauseText.endsWith('!"')) {
+					clauseBreakers.push('!')
+				} else {
+					clauseBreakers.push(".")
+				}
+			} else {
+				if (lastChar == ':' || lastChar == ';') {
+					clauseBreakers.push(lastChar)
+				} else {
+					clauseBreakers.push(",")
+				}
+			}
+		}
+	}
+
+	return phonemizeClauses(preparedClauses, voice, clauseBreakers, substitutionMap)
+}
+
+export async function phonemizeClauses(clauses: string[], voice: string, clauseBreakers: string[], substitutionMap?: Map<string, string[]>) {
+	if (clauses.length == 0) {
+		return []
+	}
+
+	const preparedText = clauses.join("\n\n") // filter(clause => clause.trim().length > 0)
+
+	const ipaString = await EspeakTTS.textToIPA(preparedText, voice)
+
+	const ipaLines = ipaString.split("\n")
+
+	const phonemeLines = ipaLines.map(line => {
+		line = line.replace(/_+/g, "_").replace(/ +/g, " ")
+
+		return line.split(" ").map(word => {
+			word = word.replaceAll("_", " ").trim()
+			let wordPhonemes = word.split(" ")
+
+			wordPhonemes = wordPhonemes.flatMap(phoneme => {
+				if (!phoneme || phoneme.startsWith("(")) {
+					return []
+				} else if (phoneme.startsWith("ˈ") || phoneme.startsWith("ˌ")) {
+					return [phoneme[0], phoneme.substring(1)]
+				} else if (phoneme.endsWith("ˈ") || phoneme.endsWith("ˌ")) {
+					return [phoneme.substring(0, phoneme.length - 1), phoneme[phoneme.length - 1]]
+				} else {
+					return [phoneme]
+				}
+			})
+
+			if (substitutionMap) {
+				wordPhonemes = wordPhonemes.flatMap(phoneme => substitutionMap.get(phoneme) || [phoneme])
+			}
+
+			return wordPhonemes
+		})
+	})
+
+	if (ipaLines.length != clauseBreakers.length) {
+		log(clauses)
+		log(ipaLines)
+		log(clauseBreakers)
+
+		throw new Error(`Unexpected: IPA lines count (${ipaLines.length}) is not equal to clause breakers count (${clauseBreakers.length})`)
+	}
+
+	for (let i = 0; i < phonemeLines.length; i++) {
+		const line = phonemeLines[i]
+		const lastWordInLine = line[line.length - 1]
+
+		lastWordInLine.push(clauseBreakers[i])
+	}
+
+	return phonemeLines
+}
+
+export function phonemizedClausesToSentences(phonemizedClauses: string[][][]) {
+	let phonemizedSentences: string[][][] = [[]]
+
+	for (const phonemizedClause of phonemizedClauses) {
+		phonemizedSentences[phonemizedSentences.length - 1].push(...phonemizedClause)
+
+		const lastWord = phonemizedClause[phonemizedClause.length - 1]
+		const lastPhoneme = lastWord[lastWord.length - 1]
+
+		if ([".", "?", "!"].includes(lastPhoneme)) {
+			phonemizedSentences.push([])
+		}
+	}
+
+	phonemizedSentences = phonemizedSentences.filter(entry => entry.length > 0)
+
+	return phonemizedSentences
+}

package/src/nlp/IPA.ts

@@ -0,0 +1,139 @@
+import { logToStderr } from "../utilities/Utilities.js"
+
+const log = logToStderr
+
+const consonant = {
+	plosive: ["p", "b", "t", "d", "ʈ", "ɖ", "c", "ɟ", "k", "g", "q", "ɢ", "ʔ", /* extensions */ "ɡ"],
+	nasal: ["m", "ɱ", "n", "ɳ", "ɲ", "ŋ", "ɴ", "n̩"],
+	trill: ["ʙ", "r", "ʀ"],
+	tapOrFlap: ["ⱱ", "ɾ", "ɽ"],
+	fricative: ["ɸ", "β", "f", "v", "θ", "ð", "s", "z", "ʃ", "ʒ", "ʂ", "ʐ", "ç", "ʝ", "x", "ɣ", "χ", "ʁ", "ħ", "ʕ", "h", "ɦ"],
+	lateralFricative: ["ɬ", "ɮ"],
+	affricate: ["tʃ", "ʈʃ", "dʒ"], // very incomplete, there are many others
+	approximant: ["ʋ", "ɹ", "ɻ", "j", "ɰ", /* extensions */ "w"],
+	lateralApproximant: ["l", "ɭ", "ʎ", "ʟ"]
+}
+
+const vowel = {
+	close: ["i", "yɨ", "ʉɯ", "u", "iː"],
+	closeOther: ["ɪ", "ʏ", "ʊ", "ɨ", "ᵻ"],
+	closeMid: ["e", "ø", "ɘ", "ɵ", "ɤ", "o", "ə", "oː"],
+	openMid: ["ɛ", "œ", "ɜ", "ɞ", "ʌ", "ɔ", "ɜː", "uː", "ɔː", "ɛː"],
+	open: ["æ", "a", "ɶ", "ɐ", "ɑ", "ɒ", "ɑː"],
+
+	rhotic: ["◌˞", "ɚ", "ɝ", "ɹ̩"],
+
+	diphtongs: [
+		"eɪ", "əʊ", "oʊ", "aɪ", "ɔɪ", "aʊ", "iə",
+		"ɜr", "ɑr", "ɔr", "oʊr", "oːɹ", "ir", "ɪɹ", "ɔːɹ", "ɑːɹ", "ʊɹ", "ʊr", "ɛr", "ɛɹ",
+		"əl", "aɪɚ", "aɪə"
+	],
+}
+
+let consonants: string[] = []
+
+for (const p in consonant) {
+	consonants = [...consonants, ...(consonant as any)[p]]
+}
+
+let vowels: string[] = []
+
+for (const p in vowel) {
+	vowels = [...vowels, ...(vowel as any)[p]]
+}
+
+const all = [" ", ...consonants, ...vowels]
+
+export function getPhoneSubstitutionCost1(ipa1: string, ipa2: string) {
+	if (ipa1 == ipa2) {
+		return 0
+	} else {
+		return 1
+	}
+}
+
+export function getPhoneSubstitutionCost2(ipa1: string, ipa2: string) {
+	if (!isKnownSymbol(ipa1)) {
+		throw new Error(`'${ipa1}' is not a known IPA symbol`)
+	}
+
+	if (!isKnownSymbol(ipa2)) {
+		throw new Error(`'${ipa2}' is not a known IPA symbol`)
+	}
+
+	if (ipa1 == ipa2) {
+		return 0
+	} else if (isVowel(ipa1) && isVowel(ipa2)) {
+		return 0.5
+	} else if (isConsonant(ipa1) && isConsonant(ipa2)) {
+		return 0.5
+	} else {
+		return 1
+	}
+}
+
+export function getPhoneSubstitutionCost3(ipa1: string, ipa2: string) {
+	if (!isKnownSymbol(ipa1)) {
+		throw new Error(`'${ipa1}' is not a known IPA symbol`)
+	}
+
+	if (!isKnownSymbol(ipa2)) {
+		throw new Error(`'${ipa2}' is not a known IPA symbol`)
+	}
+
+	if (ipa1 == ipa2) {
+		return 0
+	} else if (isVowel(ipa1) && isVowel(ipa2)) {
+		return 0.75
+	} else if (isConsonant(ipa1) && isConsonant(ipa2)) {
+		if ((isPlosiveOrNasal(ipa1) && isPlosiveOrNasal(ipa2)) ||
+			(isTrillTapOrFlap(ipa1) && isTrillTapOrFlap(ipa2)) ||
+			(isFricativeLike(ipa1) && isFricativeLike(ipa2)) ||
+			(isApproximantLike(ipa1) && isApproximantLike(ipa2))) {
+			return 0.5
+		} else {
+			return 0.75
+		}
+	} else {
+		return 1
+	}
+}
+
+// All symbols:
+export function isKnownSymbol(ipa: string) { return all.includes(ipa) }
+
+// All consonants:
+export function isConsonant(ipa: string) { return consonants.includes(ipa) }
+
+// All vowels:
+export function isVowel(ipa: string) { return vowels.includes(ipa) }
+
+// Higher level grouping:
+export function isPlosiveOrNasal(ipa: string) { return isPlosive(ipa) || isNasal(ipa) }
+
+export function isTrillTapOrFlap(ipa: string) { return isTrill(ipa) || isTapOrFlap(ipa) }
+
+export function isFricativeLike(ipa: string) { return isFricative(ipa) || isLateralFricative(ipa) || isAffricate(ipa) }
+
+export function isApproximantLike(ipa: string) { return isApproximant(ipa) || isLateralApproximant(ipa) }
+
+// Grouping:
+export function isPlosive(ipa: string) { return consonant.plosive.includes(ipa) }
+
+export function isNasal(ipa: string) { return consonant.nasal.includes(ipa) }
+
+export function isTrill(ipa: string) { return consonant.trill.includes(ipa) }
+
+export function isTapOrFlap(ipa: string) { return consonant.tapOrFlap.includes(ipa) }
+
+export function isFricative(ipa: string) { return consonant.fricative.includes(ipa) }
+
+export function isLateralFricative(ipa: string) { return consonant.lateralFricative.includes(ipa) }
+
+export function isAffricate(ipa: string) { return consonant.affricate.includes(ipa) }
+
+export function isApproximant(ipa: string) { return consonant.approximant.includes(ipa) }
+
+export function isLateralApproximant(ipa: string) { return consonant.lateralApproximant.includes(ipa) }
+
+

package/src/nlp/JapaneseSegmentation.ts

@@ -0,0 +1,53 @@
+import path from "path"
+import { OpenPromise } from "../utilities/OpenPromise.js"
+import { resolveModuleScriptPath } from "../utilities/Utilities.js"
+
+export async function splitJapaneseTextToWords_Kuromoji(text: string) {
+	const tokenizer = await getKuromojiTokenizer()
+
+	const results: any[] = tokenizer.tokenize(text)
+	const words = results.map(entry => entry.surface_form)
+
+	return words
+}
+
+let kuromojiTokenizer: any
+
+async function getKuromojiTokenizer() {
+	if (kuromojiTokenizer) {
+		return kuromojiTokenizer
+	}
+
+	const { default: kuromoji } = await import("kuromoji")
+
+	const resultOpenPromise = new OpenPromise<any>()
+
+	const kuromojiScriptPath = await resolveModuleScriptPath('kuromoji')
+	const dictionaryPath = path.join(path.dirname(kuromojiScriptPath), "..", '/dict')
+
+	kuromoji.builder({ dicPath: dictionaryPath }).build(function (error: any, tokenizer: any) {
+		if (error) {
+			resultOpenPromise.reject(error)
+			return
+		}
+
+		kuromojiTokenizer = tokenizer
+
+		resultOpenPromise.resolve(kuromojiTokenizer)
+	})
+
+	return resultOpenPromise.promise
+}
+
+/*
+export async function splitJapaneseTextToWords_Sudachi(text: string, mode: 0 | 1 | 2) {
+	const { TokenizeMode, tokenize } = await import("sudachi")
+
+	const resultString = tokenize(text, mode)
+
+	const parsedResult: any[] = JSON.parse(resultString)
+	const result = parsedResult.map(entry => entry.surface)
+
+	return result
+}
+*/

package/src/nlp/Lexicon.ts

@@ -0,0 +1,119 @@
+import { readAndParseJsonFile, resolveToModuleRootDir } from "../utilities/FileSystem.js"
+import { getShortLanguageCode } from "../utilities/Locale.js"
+
+export function tryGetFirstLexiconSubstitution(sentenceWords: string[], wordIndex: number, lexicons: Lexicon[], languageCode: string) {
+	const reversedLexicons = [...lexicons].reverse() // Give precedence to later lexicons
+
+	for (const lexicon of reversedLexicons) {
+		const match = tryGetLexiconSubstitution(sentenceWords, wordIndex, lexicon, languageCode)
+
+		if (match) {
+			return match
+		}
+	}
+
+	return undefined
+}
+
+export function tryGetLexiconSubstitution(sentenceWords: string[], wordIndex: number, lexicon: Lexicon, languageCode: string) {
+	let word = sentenceWords[wordIndex]
+
+	if (!word) {
+		return
+	}
+
+	const shortLanguageCode = getShortLanguageCode(languageCode)
+	const lexiconForLanguage = lexicon[shortLanguageCode]
+
+	if (!lexiconForLanguage) {
+		return
+	}
+
+	const lexiconEntry = lexiconForLanguage[word]
+
+	if (!lexiconEntry) {
+		return
+	}
+
+	for (let i = 0; i < lexiconEntry.length; i++) {
+		const substitutionEntry = lexiconEntry[i]
+
+		const substitutionPhonemesText = substitutionEntry?.pronunciation?.espeak?.[languageCode]
+
+		if (!substitutionPhonemesText) {
+			continue
+		}
+
+		const precedingWord = sentenceWords[wordIndex - 1] || ""
+		const succeedingWord = sentenceWords[wordIndex + 1] || ""
+
+		const precededBy = substitutionEntry?.precededBy || []
+		const notPrecededBy = substitutionEntry?.notPrecededBy || []
+
+		const succeededBy = substitutionEntry?.succeededBy || []
+		const notSucceededBy = substitutionEntry?.notSucceededBy || []
+
+		const hasNegativePattern = notPrecededBy.includes(precedingWord) || notSucceededBy.includes(succeedingWord)
+		const hasPositivePattern = precededBy.includes(precedingWord) || succeededBy.includes(succeedingWord)
+
+		if (i == lexiconEntry.length - 1 || (hasPositivePattern && !hasNegativePattern)) {
+			const substitutionPhonemes = substitutionPhonemesText.split(/ +/g)
+
+			return substitutionPhonemes
+		}
+	}
+}
+
+export async function loadLexiconFile(jsonFilePath: string): Promise<Lexicon> {
+	const parsedLexicon: Lexicon = await readAndParseJsonFile(jsonFilePath)
+
+	return parsedLexicon
+}
+
+export async function loadLexiconsForLanguage(language: string, customLexiconPaths?: string[]) {
+	const lexicons: Lexicon[] = []
+
+	if (getShortLanguageCode(language) == "en") {
+		const heteronymsLexicon = await loadLexiconFile(resolveToModuleRootDir("data/lexicons/heteronyms.en.json"))
+		lexicons.push(heteronymsLexicon)
+	}
+
+	if (customLexiconPaths && customLexiconPaths.length > 0) {
+		for (const customLexicon of customLexiconPaths) {
+			const customLexiconObject = await loadLexiconFile(customLexicon)
+
+			lexicons.push(customLexiconObject)
+		}
+	}
+
+	return lexicons
+}
+
+export type Lexicon = {
+	[shortLanguageCode: string]: LexiconForLanguage
+}
+
+export type LexiconForLanguage = {
+	[word: string]: LexiconEntry[]
+}
+
+export type LexiconEntry = {
+	pos?: string[]
+	case?: LexiconWordCase
+
+	pronunciation?: {
+		espeak?: LexiconPronunciationForLanguageCodes
+		sapi?: LexiconPronunciationForLanguageCodes
+	},
+
+	precededBy?: string[]
+	notPrecededBy?: string[]
+
+	succeededBy?: string[]
+	notSucceededBy?: string[]
+
+	example?: string
+}
+
+export type LexiconWordCase = "any" | "capitalized" | "uppercase" | "lowercase" | "titlecase" | "camelcase" | "pascalcase"
+export type LexiconPronunciationForLanguageCodes = { [languageCode: string]: string }