echogarden 0.12.2 → 1.0.0

package/src/nlp/Segmentation.ts

@@ -9,9 +9,10 @@ import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
 const log = logToStderr
 
 export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
-export const phraseSeparators = [",", ";", ":"]
-export const sentenceSeparators = [".", "?", "!"]
-export const symbolWords = ["$", "€", "¢", "£", "¥", "©", "®", "™", "%", "&", "#", "~", "@", "+", "±", "÷", "/", "*", "=", "¼", "½", "¾"]
+export const punctuationPattern = /[\p{Punctuation}]/u
+export const phraseSeparators = [',', ';', ':']
+export const sentenceSeparators = ['.', '?', '!']
+export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
 
 export function isWordOrSymbolWord(str: string) {
 	return isWord(str) || symbolWords.includes(str)
@@ -22,6 +23,10 @@ export function isWord(str: string) {
 	return wordCharacterPattern.test(str) || symbolWords.includes(str)
 }
 
+export function isPunctuation(str: string) {
+	return punctuationPattern.test(str)
+}
+
 export class Sentence {
 	phrases: Phrase[] = []
 
@@ -29,7 +34,7 @@ export class Sentence {
 
 	get length() { return sumArray(this.phrases, (phrase) => phrase.length) }
 
-	get text() { return this.phrases.reduce<string>((result, phrase) => result + phrase.text, "") }
+	get text() { return this.phrases.reduce<string>((result, phrase) => result + phrase.text, '') }
 }
 
 export class Phrase {
@@ -37,7 +42,7 @@ export class Phrase {
 
 	get length() { return sumArray(this.words, (word) => word.length) }
 
-	get text() { return this.words.reduce<string>((result, word) => result + word.text, "") }
+	get text() { return this.words.reduce<string>((result, word) => result + word.text, '') }
 
 	get lastWord() {
 		if (this.words.length == 0) {
@@ -75,7 +80,7 @@ export class Fragment {
 
 	get length() { return sumArray(this.segments, (phrase) => phrase.length) }
 
-	get text() { return this.segments.reduce<string>((result, segment) => result + segment.text, "") }
+	get text() { return this.segments.reduce<string>((result, segment) => result + segment.text, '') }
 
 	get isEmpty() { return this.length == 0 }
 
@@ -196,17 +201,17 @@ export async function parse(text: string, langCode: string) {
 }
 
 export function splitToSentences(text: string, langCode: string): string[] {
-	const shortLangCode = getShortLanguageCode(langCode || "")
+	const shortLangCode = getShortLanguageCode(langCode || '')
 
 	return CldrSegmentation.sentenceSplit(text, CldrSegmentation.suppressions[shortLangCode])
 }
 
 export async function splitToWords(text: string, langCode: string): Promise<string[]> {
-	const shortLangCode = getShortLanguageCode(langCode || "")
+	const shortLangCode = getShortLanguageCode(langCode || '')
 
-	if (shortLangCode == "zh" || shortLangCode == "cmn") {
+	if (shortLangCode == 'zh' || shortLangCode == 'cmn') {
 		return splitChineseTextToWords_Jieba(text, undefined, true)
-	} else if (shortLangCode == "ja") {
+	} else if (shortLangCode == 'ja') {
 		return splitJapaneseTextToWords_Kuromoji(text)
 	} else {
 		return CldrSegmentation.wordSplit(text, CldrSegmentation.suppressions[shortLangCode])
@@ -224,10 +229,10 @@ export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakT
 		throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`)
 	}
 
-	if (whitespace == "removeLineBreaks") {
-		paragraphs = paragraphs.map(p => p.replaceAll(/(\r?\n)+/g, " "))
-	} else if (whitespace == "collapse") {
-		paragraphs = paragraphs.map(p => p.replaceAll(/\s+/g, " "))
+	if (whitespace == 'removeLineBreaks') {
+		paragraphs = paragraphs.map(p => p.replaceAll(/(\r?\n)+/g, ' '))
+	} else if (whitespace == 'collapse') {
+		paragraphs = paragraphs.map(p => p.replaceAll(/\s+/g, ' '))
 	}
 
 	paragraphs = paragraphs.map(p => p.trim())
@@ -235,3 +240,7 @@ export function splitToParagraphs(text: string, paragraphBreaks: ParagraphBreakT
 
 	return paragraphs
 }
+
+export function splitToLines(text: string) {
+	return text.split(/\r?\n/g)
+}

package/src/nlp/TextNormalizer.ts

@@ -1,9 +1,9 @@
-import { getShortLanguageCode } from "../utilities/Locale.js"
+import { getShortLanguageCode } from '../utilities/Locale.js'
 
 export function getNormalizedFragmentsForSpeech(words: string[], language: string) {
 	language = getShortLanguageCode(language)
 
-	if (language != "en") {
+	if (language != 'en') {
 		return { normalizedFragments: [...words], referenceFragments: [...words] }
 	}
 
@@ -15,26 +15,26 @@ export function getNormalizedFragmentsForSpeech(words: string[], language: strin
 	const fourDigitYearRangePattern = /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/
 
 	const wordsPrecedingAYear = [
-		"in", "the", "a", "to", "of", "since", "from", "between", "by", "until", "around", "before", "after",
-		"his", "her", "year", "years", "during", "copyright", "©", "early", "mid", "late",
-		"january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december",
-		"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"
+		'in', 'the', 'a', 'to', 'of', 'since', 'from', 'between', 'by', 'until', 'around', 'before', 'after',
+		'his', 'her', 'year', 'years', 'during', 'copyright', '©', 'early', 'mid', 'late',
+		'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
+		'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
 	]
 
 	const wordsPrecedingADecade = [
-		"the", "in", "early", "mid", "late", "a"
+		'the', 'in', 'early', 'mid', 'late', 'a'
 	]
 
 	const symbolsPrecedingACurrency = [
-		"$", "€", "£", "¥"
+		'$', '€', '£', '¥'
 	]
 
 	const symbolsPrecedingACurrencyAsWords = [
-		"dollars", "euros", "pounds", "yen"
+		'dollars', 'euros', 'pounds', 'yen'
 	]
 
 	const wordsSucceedingACurrency = [
-		"million", "billion", "trillion"
+		'million', 'billion', 'trillion'
 	]
 
 	const normalizedFragments: string[] = []
@@ -47,7 +47,7 @@ export function getNormalizedFragmentsForSpeech(words: string[], language: strin
 		const nextWords = words.slice(wordIndex + 1)
 		const nextWord = nextWords[0]
 
-		if ( // Normalize a four digit year pattern, e.g. "in 1995".
+		if ( // Normalize a four digit year pattern, e.g. 'in 1995'.
 			wordsPrecedingAYear.includes(lowerCaseWord) &&
 			fourDigitYearPattern.test(nextWord)) {
 
@@ -60,7 +60,7 @@ export function getNormalizedFragmentsForSpeech(words: string[], language: strin
 			referenceFragments.push(nextWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a four digit decade pattern, e.g. "the 1980s".
+		} else if ( // Normalize a four digit decade pattern, e.g. 'the 1980s'.
 			wordsPrecedingADecade.includes(lowerCaseWord) &&
 			fourDigitDecadePattern.test(nextWord)) {
 
@@ -73,20 +73,20 @@ export function getNormalizedFragmentsForSpeech(words: string[], language: strin
 			referenceFragments.push(nextWord)
 
 			wordIndex += 1
-		} else if ( // Normalize a year range pattern, e.g. "1835-1896"
-			fourDigitYearRangePattern.test(words.slice(wordIndex, wordIndex + 3).join(""))) {
+		} else if ( // Normalize a year range pattern, e.g. '1835-1896'
+			fourDigitYearRangePattern.test(words.slice(wordIndex, wordIndex + 3).join(''))) {
 
 			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex]))
 			referenceFragments.push(words[wordIndex])
 
-			normalizedFragments.push("to")
+			normalizedFragments.push('to')
 			referenceFragments.push(words[wordIndex + 1])
 
 			normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex + 2]))
 			referenceFragments.push(words[wordIndex + 2])
 
 			wordIndex += 2
-		} else if ( // Normalize a currency pattern, e.g. "$53.1 million", "€3.53"
+		} else if ( // Normalize a currency pattern, e.g. '$53.1 million', '€3.53'
 			symbolsPrecedingACurrency.includes(lowerCaseWord) &&
 			numberPattern.test(nextWord)) {
 

package/src/recognition/AmazonTranscribeSTT.ts

@@ -1,17 +1,17 @@
-import type { Item, LanguageCode, StartStreamTranscriptionCommandInput } from "@aws-sdk/client-transcribe-streaming"
-import { wordCharacterPattern } from "../nlp/Segmentation.js"
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
-import { Logger } from "../utilities/Logger.js"
-import { Timeline } from "../utilities/Timeline.js"
-import { RawAudio } from "../audio/AudioUtilities.js"
+import type { Item, LanguageCode, StartStreamTranscriptionCommandInput } from '@aws-sdk/client-transcribe-streaming'
+import { wordCharacterPattern } from '../nlp/Segmentation.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
+import { Timeline } from '../utilities/Timeline.js'
+import { RawAudio } from '../audio/AudioUtilities.js'
 
 export async function recgonize(rawAudio: RawAudio, languageCode: string, region: string, accessKeyId: string, secretAccessKey: string) {
-	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "flac", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 })
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 })
 
 	const logger = new Logger()
-	logger.start("Initialize Amazon Transcribe streaming client module")
+	logger.start('Initialize Amazon Transcribe streaming client module')
 
-	const streamingTranscribeSdk = await import("@aws-sdk/client-transcribe-streaming")
+	const streamingTranscribeSdk = await import('@aws-sdk/client-transcribe-streaming')
 
 	const streamingTranscribeClient = new streamingTranscribeSdk.TranscribeStreamingClient({
 		region,
@@ -23,7 +23,6 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 
 	const audioStream = async function* () {
 		const chunkSize = 2 ** 12
-		//const audioSamples = encodeToAudioBuffer(rawAudio.audioChannels, 16, SampleFormat.PCM)
 
 		for (let i = 0; i < flac16Khz16bitMonoAudio.length; i += chunkSize) {
 			const chunk = flac16Khz16bitMonoAudio.subarray(i, i + chunkSize)
@@ -35,17 +34,17 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 	const params: StartStreamTranscriptionCommandInput = {
 		LanguageCode: languageCode as LanguageCode,
 		MediaSampleRateHertz: rawAudio.sampleRate,
-		MediaEncoding: "flac",
+		MediaEncoding: 'flac',
 		AudioStream: audioStream(),
 	}
 
-	logger.start("Request recognition from Amazon Transcribe..")
+	logger.start('Request recognition from Amazon Transcribe')
 
 	const command = new streamingTranscribeSdk.StartStreamTranscriptionCommand(params)
 
 	const response = await streamingTranscribeClient.send(command)
 
-	let transcript = ""
+	let transcript = ''
 	let events: Item[] = []
 
 	for await (const event of response.TranscriptResultStream!) {
@@ -73,13 +72,13 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 
 		if (firstResult.IsPartial === false) {
 			events = [...events, ...firstAlternative.Items!]
-			transcript += " " + firstAlternative.Transcript!
+			transcript += ' ' + firstAlternative.Transcript!
 		}
 	}
 
-	logger.start("Process result")
+	logger.start('Process result')
 
-	transcript = transcript.replace(/ +/g, " ").trim()
+	transcript = transcript.replace(/ +/g, ' ').trim()
 
 	const timeline: Timeline = []
 
@@ -98,7 +97,7 @@ export async function recgonize(rawAudio: RawAudio, languageCode: string, region
 
 		timeline.push(
 			{
-				type: "word",
+				type: 'word',
 				text,
 				startTime,
 				endTime,

package/src/recognition/AzureCognitiveServicesSTT.ts

@@ -1,16 +1,17 @@
 import * as SpeechSDK from 'microsoft-cognitiveservices-speech-sdk'
 
-import { RawAudio, encodeWaveBuffer } from '../audio/AudioUtilities.js'
+import { RawAudio, encodeRawAudioToWave } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 
 export async function recognize(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
 	const logger = new Logger()
-	logger.start("Request recognition from Azure Cognitive Services")
+
+	logger.start('Request recognition from Azure Cognitive Services')
 
 	const result = await requestRecognition(rawAudio, subscriptionKey, serviceRegion, languageCode)
 
-	logger.start("Process result")
+	logger.start('Process result')
 
 	const transcript = result.text
 
@@ -25,7 +26,7 @@ export async function recognize(rawAudio: RawAudio, subscriptionKey: string, ser
 		const endTime = (wordEntry.Offset + wordEntry.Duration) / 10000000
 
 		timeline.push({
-			type: "word",
+			type: 'word',
 			text,
 			startTime,
 			endTime
@@ -38,8 +39,7 @@ export async function recognize(rawAudio: RawAudio, subscriptionKey: string, ser
 }
 
 async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
-	//const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "wav", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 });
-	const encodedAudio = encodeWaveBuffer(rawAudio)
+	const encodedAudio = encodeRawAudioToWave(rawAudio)
 
 	return new Promise<SpeechSDK.SpeechRecognitionResult>((resolve, reject) => {
 		const audioFormat = SpeechSDK.AudioStreamFormat.getWaveFormat(16000, 16, 1, SpeechSDK.AudioFormatTag.PCM)
@@ -65,11 +65,13 @@ async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, s
 		recognizer.recognizeOnceAsync(
 			(result) => {
 				recognizer.close()
+
 				resolve(result)
 			},
 
 			(error) => {
 				recognizer.close()
+
 				reject(error)
 			})
 	})

package/src/recognition/GoogleCloudSTT.ts

@@ -1,21 +1,21 @@
-import { request } from "gaxios"
+import { request } from 'gaxios'
 
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
-import { Logger } from "../utilities/Logger.js"
-import { Timeline } from "../utilities/Timeline.js"
-import { RawAudio } from "../audio/AudioUtilities.js"
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
+import { Timeline } from '../utilities/Timeline.js'
+import { RawAudio } from '../audio/AudioUtilities.js'
 
-export type AudioEncoding = "LINEAR16" | "FLAC" | "MULAW" | "AMR" | "AMR" | "AMR_WB" | "OGG_OPUS" | "SPEEX_WITH_HEADER_BYTE" | "MP3" | "WEBM_OPUS"
+export type AudioEncoding = 'LINEAR16' | 'FLAC' | 'MULAW' | 'AMR' | 'AMR' | 'AMR_WB' | 'OGG_OPUS' | 'SPEEX_WITH_HEADER_BYTE' | 'MP3' | 'WEBM_OPUS'
 
-export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode = "en-US") {
-	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: "flac", sampleRate: 16000, sampleFormat: "s16", channelCount: 1 })
+export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode = 'en-US') {
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 })
 
 	const logger = new Logger()
-	logger.start("Request recognition from Google Cloud")
+	logger.start('Request recognition from Google Cloud')
 
 	const requestBody = {
 		config: {
-			encoding: "FLAC",
+			encoding: 'FLAC',
 			sampleRateHertz: 16000,
 			audioChannelCount: 1,
 			languageCode,
@@ -25,35 +25,35 @@ export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode
 			enableWordTimeOffsets: true,
 			enableWordConfidence: true,
 			enableAutomaticPunctuation: true,
-			model: "latest_long",
+			model: 'latest_long',
 			useEnhanced: true
 		},
 
 		audio: {
-			content: flac16Khz16bitMonoAudio.toString("base64")
+			content: flac16Khz16bitMonoAudio.toString('base64')
 		}
 	}
 
 	const response = await request<any>({
-		method: "POST",
+		method: 'POST',
 
 		url: `https://speech.googleapis.com/v1p1beta1/speech:recognize`,
 
 		params: {
-			"key": apiKey
+			'key': apiKey
 		},
 
 		headers: {
-			"User-Agent": ""
+			'User-Agent': ''
 		},
 
 		data: requestBody,
 
 
-		responseType: "json"
+		responseType: 'json'
 	})
 
-	logger.start("Parse response body")
+	logger.start('Parse response body')
 
 	const result = parseResponseBody(response.data)
 
@@ -65,7 +65,7 @@ export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode
 function parseResponseBody(responseBody: any) {
 	const results = responseBody.results
 
-	let transcript = ""
+	let transcript = ''
 	const timeline: Timeline = []
 
 	for (const result of results) {
@@ -79,10 +79,10 @@ function parseResponseBody(responseBody: any) {
 
 		for (const wordEvent of firstAlternative.words) {
 			timeline.push({
-				type: "word",
+				type: 'word',
 				text: wordEvent.word,
-				startTime: parseFloat(wordEvent.startTime.replace("s","")),
-				endTime: parseFloat(wordEvent.endTime.replace("s", "")),
+				startTime: parseFloat(wordEvent.startTime.replace('s','')),
+				endTime: parseFloat(wordEvent.endTime.replace('s', '')),
 				confidence: wordEvent.confidence
 			})
 		}

package/src/recognition/OpenAICloudSTT.ts

@@ -0,0 +1,142 @@
+import { RawAudio } from '../audio/AudioUtilities.js';
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js';
+import { extendDeep } from '../utilities/ObjectUtilities.js';
+import { Timeline, TimelineEntry } from '../utilities/Timeline.js';
+
+export async function recognize(rawAudio: RawAudio, languageCode: string, options: OpenAICloudSTTOptions, task: Task = 'transcribe') {
+	const logger = new Logger()
+
+	logger.start('Load OpenAI module')
+
+	options = extendDeep(defaultOpenAICloudSTTOptions, options)
+
+	const { default: OpenAI } = await import('openai')
+	const openai = new OpenAI(options)
+
+	logger.start('Encode audio to send')
+	const ffmpegOptions = FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('mp3')
+	const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, ffmpegOptions)
+	const audioAsWaveBlob = new FileLikeBlob([encodedAudio], 'audio', Date.now(), { type: 'audio/mpeg' })
+
+	logger.start('Request recognition from OpenAI Cloud API')
+
+	let response: VerboseResponse
+
+	if (task =='transcribe') {
+		response = await openai.audio.transcriptions.create({
+			file: audioAsWaveBlob,
+			model: options.model!,
+			language: languageCode,
+			prompt: options.prompt,
+			response_format: 'verbose_json',
+			temperature: options.temperature,
+			timestamp_granularities: ['word', 'segment']
+		}) as VerboseResponse
+	} else if (task == 'translate') {
+		response = await openai.audio.translations.create({
+			file: audioAsWaveBlob,
+			model: options.model!,
+			prompt: options.prompt,
+			response_format: 'verbose_json',
+			temperature: options.temperature,
+		}) as VerboseResponse
+	} else {
+		throw new Error(`Invalid task`)
+	}
+
+	const transcript = response.text
+
+	let timeline: Timeline
+
+	if (response.words) {
+		timeline = response.words.map<TimelineEntry>(entry => ({
+			type: 'word',
+			text: entry.word,
+			startTime: entry.start,
+			endTime: entry.end
+		}))
+	} else {
+		timeline = response.segments.map<TimelineEntry>(entry => ({
+			type: 'segment',
+			text: entry.text,
+			startTime: entry.start,
+			endTime: entry.end
+		}))
+	}
+
+	logger.end()
+
+	return { transcript, timeline }
+}
+
+class FileLikeBlob extends Blob {
+	constructor(
+		public readonly parts: BlobPart[],
+		public readonly name: string,
+		public readonly lastModified: number,
+		options: BlobPropertyBag,
+	) {
+		super(parts, options)
+	}
+}
+
+export interface OpenAICloudSTTOptions {
+	model?: 'whisper-1'
+
+	apiKey?: string
+	organization?: string
+	baseURL?: string
+
+	temperature?: number
+	prompt?: string
+
+	timeout?: number
+	maxRetries?: number
+}
+
+export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
+	apiKey: undefined,
+	organization: undefined,
+	baseURL: undefined,
+
+	model: 'whisper-1',
+	temperature: 0,
+	prompt: undefined,
+
+	timeout: undefined,
+	maxRetries: 10,
+}
+
+interface VerboseResponse {
+	task: string
+	language: string
+	duration: number
+
+	text: string
+
+	segments: {
+		text: string
+
+		start: number
+		end: number
+
+		id: number
+		no_speech_prob: number
+		compression_ratio: number
+		avg_logprob: number
+		seek: number
+		temperature: number
+
+		tokens: number[]
+	}[]
+
+	words: {
+		word: string
+
+		start: number
+		end: number
+	}[]
+}
+
+type Task = 'transcribe' | 'translate'