echogarden 0.11.11 → 0.11.13

package/src/nlp/Segmentation.ts

@@ -0,0 +1,237 @@
+import * as CldrSegmentation from 'cldr-segmentation'
+import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js'
+
+import { sumArray, includesAnyOf, indexOfAnyOf, logToStderr } from '../utilities/Utilities.js'
+import { getShortLanguageCode } from '../utilities/Locale.js'
+import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
+import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
+
+const log = logToStderr
+
+export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
+export const phraseSeparators = [",", ";", ":"]
+export const sentenceSeparators = [".", "?", "!"]
+export const symbolWords = ["$", "€", "¢", "£", "¥", "©", "®", "™", "%", "&", "#", "~", "@", "+", "±", "÷", "/", "*", "=", "¼", "½", "¾"]
+
+export function isWordOrSymbolWord(str: string) {
+	return isWord(str) || symbolWords.includes(str)
+}
+
+export function isWord(str: string) {
+	str = str.trim()
+	return wordCharacterPattern.test(str) || symbolWords.includes(str)
+}
+
+export class Sentence {
+	phrases: Phrase[] = []
+
+	readonly isSentenceFinalizer = true
+
+	get length() { return sumArray(this.phrases, (phrase) => phrase.length) }
+
+	get text() { return this.phrases.reduce<string>((result, phrase) => result + phrase.text, "") }
+}
+
+export class Phrase {
+	words: Word[] = []
+
+	get length() { return sumArray(this.words, (word) => word.length) }
+
+	get text() { return this.words.reduce<string>((result, word) => result + word.text, "") }
+
+	get lastWord() {
+		if (this.words.length == 0) {
+			return undefined
+		}
+
+		return this.words[this.words.length - 1]
+	}
+
+	get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false }
+}
+
+export class Word {
+	readonly text: string
+	isSentenceFinalizer: boolean
+
+	constructor(text: string, isSentenceFinalizer: boolean) {
+		this.text = text
+		this.isSentenceFinalizer = isSentenceFinalizer
+	}
+
+	get containsOnlyPunctuation() { return !wordCharacterPattern.test(this.text) && !this.isSymbolWord }
+
+	get isSymbolWord() { return symbolWords.includes(this.text) }
+
+	get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators) }
+
+	get length() { return this.text.length }
+}
+
+export type Segment = Sentence | Phrase | Word
+
+export class Fragment {
+	segments: Segment[] = []
+
+	get length() { return sumArray(this.segments, (phrase) => phrase.length) }
+
+	get text() { return this.segments.reduce<string>((result, segment) => result + segment.text, "") }
+
+	get isEmpty() { return this.length == 0 }
+
+	get isNonempty() { return !this.isEmpty }
+
+	get lastSegment() {
+		if (this.isEmpty) {
+			return undefined
+		}
+
+		return this.segments[this.segments.length - 1]
+	}
+}
+
+export async function splitToFragments(text: string, maxFragmentLength: number, langCode: string, preserveSentences = true, preservePhrases = true) {
+	const parsedText = await parse(text, langCode)
+
+	const fragments: Fragment[] = []
+	let currentFragment = new Fragment()
+
+	const remainingCharactersInCurrentFragment = () => maxFragmentLength - currentFragment.length
+	const createNewFragmentIfNeeded = () => {
+		if (currentFragment.isNonempty) {
+			fragments.push(currentFragment)
+			currentFragment = new Fragment()
+		}
+	}
+
+	const fitsCurrentFragment = (segment: Segment) => segment.length <= remainingCharactersInCurrentFragment()
+
+	for (const sentence of parsedText) {
+		if (fitsCurrentFragment(sentence)) {
+			currentFragment.segments.push(sentence)
+			continue
+		}
+
+		if (preserveSentences) {
+			createNewFragmentIfNeeded()
+
+			if (fitsCurrentFragment(sentence)) {
+				currentFragment.segments.push(sentence)
+				continue
+			}
+		}
+
+		for (const phrase of sentence.phrases) {
+			if (fitsCurrentFragment(phrase)) {
+				currentFragment.segments.push(phrase)
+				continue
+			}
+
+
+			if (preservePhrases) {
+				createNewFragmentIfNeeded()
+
+				if (fitsCurrentFragment(phrase)) {
+					currentFragment.segments.push(phrase)
+					continue
+				}
+			}
+
+			for (const word of phrase.words) {
+				if (fitsCurrentFragment(word)) {
+					currentFragment.segments.push(word)
+					continue
+				}
+
+				createNewFragmentIfNeeded()
+
+				if (fitsCurrentFragment(word)) {
+					currentFragment.segments.push(word)
+					continue
+				}
+
+				throw new Error(`Encountered a word of length ${word.length}, which excceeds the maximum fragment length of ${maxFragmentLength}`)
+			}
+		}
+	}
+
+	createNewFragmentIfNeeded()
+
+	return fragments
+}
+
+export async function parse(text: string, langCode: string) {
+	const sentencesText = splitToSentences(text, langCode)
+	const sentences: Sentence[] = []
+
+	for (const sentenceText of sentencesText) {
+		const sentence = new Sentence()
+
+		let currentPhrase = new Phrase()
+		const wordTexts = await splitToWords(sentenceText, langCode)
+
+		for (let wordIndex = 0; wordIndex < wordTexts.length; wordIndex++) {
+			const word = new Word(wordTexts[wordIndex], wordIndex == wordTexts.length - 1)
+
+			if (word.isPhraseSeperator) {
+				const separatorIndex = indexOfAnyOf(word.text, phraseSeparators)
+				currentPhrase.words.push(new Word(word.text.substring(0, separatorIndex + 1), word.isSentenceFinalizer))
+				sentence.phrases.push(currentPhrase)
+
+				currentPhrase = new Phrase()
+				currentPhrase.words.push(new Word(word.text.substring(separatorIndex + 1), false))
+			} else {
+				currentPhrase.words.push(word)
+			}
+		}
+
+		if (currentPhrase.words.length > 0) {
+			sentence.phrases.push(currentPhrase)
+		}
+
+		sentences.push(sentence)
+	}
+
+	return sentences
+}
+
+export function splitToSentences(text: string, langCode: string): string[] {
+	const shortLangCode = getShortLanguageCode(langCode || "")
+
+	return CldrSegmentation.sentenceSplit(text, CldrSegmentation.suppressions[shortLangCode])
+}
+
+export async function splitToWords(text: string, langCode: string): Promise<string[]> {
+	const shortLangCode = getShortLanguageCode(langCode || "")
+
+	if (shortLangCode == "zh" || shortLangCode == "cmn") {
+		return splitChineseTextToWords_Jieba(text, undefined, true)
+	} else if (shortLangCode == "ja") {
+		return splitJapaneseTextToWords_Kuromoji(text)
+	} else {
+		return CldrSegmentation.wordSplit(text, CldrSegmentation.suppressions[shortLangCode])
+	}
+}
+
+export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakType, whitespace: WhitespaceProcessing) {
+	let paragraphs: string[] = []
+
+	if (paragraphBreaks == 'single') {
+		paragraphs = text.split(/(\r?\n)+/g)
+	} else if (paragraphBreaks == 'double') {
+		paragraphs = text.split(/(\r?\n)(\r?\n)+/g)
+	} else {
+		throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`)
+	}
+
+	if (whitespace == "removeLineBreaks") {
+		paragraphs = paragraphs.map(p => p.replaceAll(/(\r?\n)+/g, " "))
+	} else if (whitespace == "collapse") {
+		paragraphs = paragraphs.map(p => p.replaceAll(/\s+/g, " "))
+	}
+
+	paragraphs = paragraphs.map(p => p.trim())
+	paragraphs = paragraphs.filter(p => p.length > 0)
+
+	return paragraphs
+}

package/src/nlp/TextNormalizer.ts

@@ -0,0 +1,160 @@
+import { getShortLanguageCode } from "../utilities/Locale.js"
+
+export function getNormalizedFragmentsForSpeech(words: string[], language: string) {
+	language = getShortLanguageCode(language)
+
+	if (language != "en") {
+		return { normalizedFragments: [...words], referenceFragments: [...words] }
+	}
+
+	const numberPattern = /^[0-9][0-9\,\.]*$/
+
+	const fourDigitYearPattern = /^[0-9][0-9][0-9][0-9]$/
+	const fourDigitDecadePattern = /^[0-9][0-9][0-9]0s$/
+
+	const fourDigitYearRangePattern = /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/
+
+	const wordsPrecedingAYear = [
+		"in", "the", "a", "to", "of", "since", "from", "between", "by", "until", "around", "before", "after",
+		"his", "her", "year", "years", "during", "copyright", "©", "early", "mid", "late",
+		"january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december",
+		"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"
+	]
+
+	const wordsPrecedingADecade = [
+		"the", "in", "early", "mid", "late", "a"
+	]
+
+	const symbolsPrecedingACurrency = [
+		"$", "€", "£", "¥"
+	]
+
+	const symbolsPrecedingACurrencyAsWords = [
+		"dollars", "euros", "pounds", "yen"
+	]
+
+	const wordsSucceedingACurrency = [
+		"million", "billion", "trillion"
+	]
+
+	const normalizedFragments: string[] = []
+	const referenceFragments: string[] = []
+
+	for (let wordIndex = 0; wordIndex < words.length; wordIndex++) {
+		const word = words[wordIndex]
+		const lowerCaseWord = word.toLowerCase()
+
+		const nextWords = words.slice(wordIndex + 1)
+		const nextWord = nextWords[0]
+
+		if ( // Normalize a four digit year pattern, e.g. "in 1995".
+			wordsPrecedingAYear.includes(lowerCaseWord) &&
+			fourDigitYearPattern.test(nextWord)) {
+
+			const normalizedString = normalizeFourDigitYearString(nextWord)
+
+			normalizedFragments.push(word)
+			referenceFragments.push(word)
+
+			normalizedFragments.push(normalizedString)
+			referenceFragments.push(nextWord)
+
+			wordIndex += 1
+		} else if ( // Normalize a four digit decade pattern, e.g. "the 1980s".
+			wordsPrecedingADecade.includes(lowerCaseWord) &&
+			fourDigitDecadePattern.test(nextWord)) {
+
+			const normalizedString = normalizeFourDigitDecadeString(nextWord)
+
+			normalizedFragments.push(word)
+			referenceFragments.push(word)
+
+			normalizedFragments.push(normalizedString)
+			referenceFragments.push(nextWord)
+
+			wordIndex += 1
+		} else if ( // Normalize a year range pattern, e.g. "1835-1896"
+			fourDigitYearRangePattern.test(words.slice(wordIndex, wordIndex + 3).join(""))) {
+
+			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex]))
+			referenceFragments.push(words[wordIndex])
+
+			normalizedFragments.push("to")
+			referenceFragments.push(words[wordIndex + 1])
+
+			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex + 2]))
+			referenceFragments.push(words[wordIndex + 2])
+
+			wordIndex += 2
+		} else if ( // Normalize a currency pattern, e.g. "$53.1 million", "€3.53"
+			symbolsPrecedingACurrency.includes(lowerCaseWord) &&
+			numberPattern.test(nextWord)) {
+
+			let currencyWord = symbolsPrecedingACurrencyAsWords[symbolsPrecedingACurrency.indexOf(lowerCaseWord)]
+
+			if (wordsSucceedingACurrency.includes(nextWords[1].toLowerCase())) {
+				const normalizedString = `${nextWord} ${nextWords[1]} ${currencyWord}`
+
+				normalizedFragments.push(normalizedString)
+
+				const referenceString = `${word}${nextWord} ${nextWords[1]}`
+				referenceFragments.push(referenceString)
+
+				wordIndex += 2
+			} else {
+				const normalizedString = `${nextWord} ${currencyWord}`
+
+				normalizedFragments.push(normalizedString)
+
+				const referenceString = `${word}${nextWord}`
+				referenceFragments.push(referenceString)
+
+				wordIndex += 1
+			}
+		} else {
+			normalizedFragments.push(word)
+			referenceFragments.push(word)
+		}
+	}
+
+	return { normalizedFragments, referenceFragments }
+}
+
+export function normalizeFourDigitYearString(yearString: string) {
+	const firstTwoDigitsValue = parseFloat(yearString.substring(0, 2))
+	const secondTwoDigitsValue = parseFloat(yearString.substring(2, 4))
+
+	let normalizedString: string
+
+	if (firstTwoDigitsValue >= 10 && secondTwoDigitsValue >= 10) {
+		normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}`
+	} else if (firstTwoDigitsValue >= 10 && firstTwoDigitsValue % 10 != 0 && secondTwoDigitsValue < 10) {
+		normalizedString = `${firstTwoDigitsValue} oh ${secondTwoDigitsValue}`
+	} else {
+		normalizedString = yearString
+	}
+
+	return normalizedString
+}
+
+export function normalizeFourDigitDecadeString(decadeString: string) {
+	const firstTwoDigitsValue = parseInt(decadeString.substring(0, 2))
+	const secondTwoDigitsValue = parseInt(decadeString.substring(2, 4))
+
+	let normalizedString: string
+
+	const isBeforeSecondMillenium = firstTwoDigitsValue < 10
+	const isMilleniumDecade =  firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0
+
+	if (!isBeforeSecondMillenium && !isMilleniumDecade) {
+		if (secondTwoDigitsValue != 0) {
+			normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}s`
+		} else {
+			normalizedString = `${firstTwoDigitsValue} hundreds`
+		}
+	} else {
+		normalizedString = decadeString
+	}
+
+	return normalizedString
+}

package/src/recognition/AmazonTranscribeSTT.ts

@@ -0,0 +1,112 @@
+import type { Item, StartStreamTranscriptionCommandInput } from "@aws-sdk/client-transcribe-streaming"
+import { wordCharacterPattern } from "../nlp/Segmentation.js"
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import { Logger } from "../utilities/Logger.js"
+import { Timeline } from "../utilities/Timeline.js"
+import { RawAudio } from "../audio/AudioUtilities.js"
+
+export async function recgonize(rawAudio: RawAudio, languageCode: string, region: string, accessKeyId: string, secretAccessKey: string) {
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "flac", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 })
+
+	const logger = new Logger()
+	logger.start("Initialize Amazon Transcribe streaming client module")
+
+	const streamingTranscribeSdk = await import("@aws-sdk/client-transcribe-streaming")
+
+	const streamingTranscribeClient = new streamingTranscribeSdk.TranscribeStreamingClient({
+		region,
+		credentials: {
+			accessKeyId,
+			secretAccessKey
+		}
+	})
+
+	const audioStream = async function* () {
+		const chunkSize = 2 ** 12
+		//const audioSamples = encodeToAudioBuffer(rawAudio.audioChannels, 16, SampleFormat.PCM)
+
+		for (let i = 0; i < flac16Khz16bitMonoAudio.length; i += chunkSize) {
+			const chunk = flac16Khz16bitMonoAudio.subarray(i, i + chunkSize)
+
+			yield { AudioEvent: { AudioChunk: chunk } }
+		}
+	}
+
+	const params: StartStreamTranscriptionCommandInput = {
+		LanguageCode: languageCode,
+		MediaSampleRateHertz: rawAudio.sampleRate,
+		MediaEncoding: "flac",
+		AudioStream: audioStream(),
+	}
+
+	logger.start("Request recognition from Amazon Transcribe..")
+
+	const command = new streamingTranscribeSdk.StartStreamTranscriptionCommand(params)
+
+	const response = await streamingTranscribeClient.send(command)
+
+	let transcript = ""
+	let events: Item[] = []
+
+	for await (const event of response.TranscriptResultStream!) {
+		if (!event.TranscriptEvent) {
+			continue
+		}
+
+		const transcriptEvent = event.TranscriptEvent
+
+		const results = transcriptEvent.Transcript!.Results!
+
+		if (results.length == 0) {
+			continue
+		}
+
+		const firstResult = results[0]
+		const alternatives = firstResult.Alternatives
+
+		if (!alternatives || alternatives.length == 0) {
+			continue
+		}
+
+		const firstAlternative = alternatives[0]
+		//logger.log(firstAlternative.Transcript!)
+
+		if (firstResult.IsPartial === false) {
+			events = [...events, ...firstAlternative.Items!]
+			transcript += " " + firstAlternative.Transcript!
+		}
+	}
+
+	logger.start("Process result")
+
+	transcript = transcript.replace(/ +/g, " ").trim()
+
+	const timeline: Timeline = []
+
+	for (const event of events) {
+		const text = event.Content!
+
+		if (!wordCharacterPattern.test(text)) {
+			continue
+		}
+
+		const startTime = event.StartTime!
+		const endTime = event.EndTime!
+		const confidence = event.Confidence
+
+		timeline[timeline.length - 1].endTime = startTime
+
+		timeline.push(
+			{
+				type: "word",
+				text,
+				startTime,
+				endTime,
+				confidence
+			})
+	}
+
+	logger.end()
+
+	return { transcript, timeline }
+}

package/src/recognition/AzureCognitiveServicesSTT.ts

@@ -0,0 +1,76 @@
+import SpeechSDK from 'microsoft-cognitiveservices-speech-sdk'
+
+import { RawAudio, encodeWaveBuffer } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+import { Timeline } from '../utilities/Timeline.js'
+
+export async function recognize(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
+	const logger = new Logger()
+	logger.start("Request recognition from Azure Cognitive Services")
+
+	const result = await requestRecognition(rawAudio, subscriptionKey, serviceRegion, languageCode)
+
+	logger.start("Process result")
+
+	const transcript = result.text
+
+	const resultObject = JSON.parse(result.json)
+	const bestResult = resultObject.NBest[0]
+
+	const timeline: Timeline = []
+
+	for (const wordEntry of bestResult.Words) {
+		const text = wordEntry.Word
+		const startTime = wordEntry.Offset / 10000000
+		const endTime = (wordEntry.Offset + wordEntry.Duration) / 10000000
+
+		timeline.push({
+			type: "word",
+			text,
+			startTime,
+			endTime
+		})
+	}
+
+	logger.end()
+
+	return { transcript, timeline }
+}
+
+async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
+	//const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "wav", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 });
+	const encodedAudio = encodeWaveBuffer(rawAudio)
+
+	return new Promise<SpeechSDK.SpeechRecognitionResult>((resolve, reject) => {
+		const audioFormat = SpeechSDK.AudioStreamFormat.getWaveFormat(16000, 16, 1, SpeechSDK.AudioFormatTag.PCM)
+
+		const inputStream = SpeechSDK.AudioInputStream.createPushStream(audioFormat)
+
+		inputStream.write(encodedAudio)
+		inputStream.close()
+
+		const audioConfig = SpeechSDK.AudioConfig.fromStreamInput(inputStream)
+
+		const speechConfig = SpeechSDK.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion)
+
+		speechConfig.speechRecognitionLanguage = languageCode
+
+		speechConfig.setProfanity(profanity)
+		speechConfig.requestWordLevelTimestamps()
+
+		speechConfig.outputFormat = SpeechSDK.OutputFormat.Detailed
+
+		const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, audioConfig)
+
+		recognizer.recognizeOnceAsync(
+			(result) => {
+				recognizer.close()
+				resolve(result)
+			},
+
+			(error) => {
+				recognizer.close()
+				reject(error)
+			})
+	})
+}

package/src/recognition/GoogleCloudSTT.ts

@@ -0,0 +1,92 @@
+import { request } from "gaxios"
+
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import { Logger } from "../utilities/Logger.js"
+import { Timeline } from "../utilities/Timeline.js"
+import { RawAudio } from "../audio/AudioUtilities.js"
+
+export type AudioEncoding = "LINEAR16" | "FLAC" | "MULAW" | "AMR" | "AMR" | "AMR_WB" | "OGG_OPUS" | "SPEEX_WITH_HEADER_BYTE" | "MP3" | "WEBM_OPUS"
+
+export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode = "en-US") {
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "flac", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 })
+
+	const logger = new Logger()
+	logger.start("Request recognition from Google Cloud")
+
+	const requestBody = {
+		config: {
+			encoding: "FLAC",
+			sampleRateHertz: 16000,
+			audioChannelCount: 1,
+			languageCode,
+			alternativeLanguageCodes: [],
+			maxAlternatives: 1,
+			profanityFilter: false,
+			enableWordTimeOffsets: true,
+			enableWordConfidence: true,
+			enableAutomaticPunctuation: true,
+			model: "latest_long",
+			useEnhanced: true
+		},
+
+		audio: {
+			content: flac16Khz16bitMonoAudio.toString("base64")
+		}
+	}
+
+	const response = await request<any>({
+		method: "POST",
+
+		url: `https://speech.googleapis.com/v1p1beta1/speech:recognize`,
+
+		params: {
+			"key": apiKey
+		},
+
+		headers: {
+			"User-Agent": ""
+		},
+
+		data: requestBody,
+
+
+		responseType: "json"
+	})
+
+	logger.start("Parse response body")
+
+	const result = parseResponseBody(response.data)
+
+	logger.end()
+
+	return result
+}
+
+function parseResponseBody(responseBody: any) {
+	const results = responseBody.results
+
+	let transcript = ""
+	const timeline: Timeline = []
+
+	for (const result of results) {
+		if (!result.alternatives || !result.alternatives[0] || !result.alternatives[0].transcript) {
+			continue
+		}
+
+		const firstAlternative = result.alternatives[0]
+
+		transcript += firstAlternative.transcript
+
+		for (const wordEvent of firstAlternative.words) {
+			timeline.push({
+				type: "word",
+				text: wordEvent.word,
+				startTime: parseFloat(wordEvent.startTime.replace("s","")),
+				endTime: parseFloat(wordEvent.endTime.replace("s", "")),
+				confidence: wordEvent.confidence
+			})
+		}
+	}
+
+	return { transcript, timeline }
+}