echogarden 1.4.4 → 1.6.0

package/src/speech-embeddings/WavToVec2BertFeatureEmbeddings.ts

@@ -0,0 +1,107 @@
+import type * as Onnx from 'onnxruntime-node'
+import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js';
+
+import { RawAudio } from "../audio/AudioUtilities.js";
+import { computeMelSpectogram } from "../dsp/MelSpectogram.js";
+import { Logger } from '../utilities/Logger.js';
+import { concatFloat32Arrays, splitFloat32Array } from '../utilities/Utilities.js';
+import { applyEmphasis } from '../dsp/MFCC.js';
+
+export function computeEmbeddings(audioSamples: RawAudio, modelFilePath: string, executionProviders: OnnxExecutionProvider[]) {
+	const wav2vecBert = new Wav2Vec2BertFeatureEmbeddings(
+		modelFilePath,
+		executionProviders,
+	)
+
+	const result = wav2vecBert.computeEmbeddings(audioSamples)
+
+	return result
+}
+
+export class Wav2Vec2BertFeatureEmbeddings {
+	session?: Onnx.InferenceSession
+
+	constructor(
+		public readonly modelFilePath: string,
+		public readonly executionProviders: OnnxExecutionProvider[]) {
+	}
+
+	async computeEmbeddings(rawAudio: RawAudio) {
+		const logger = new Logger()
+
+		rawAudio.audioChannels[0] = applyEmphasis(rawAudio.audioChannels[0], 0.97)
+
+		const { melSpectogram } = await computeMelSpectogram(
+			rawAudio,
+			512,
+			400,
+			160,
+			80,
+			20,
+			8000,
+			'povey')
+
+		// Ensure even length
+		if (melSpectogram.length % 2 != 0) {
+			melSpectogram.push(new Float32Array(80))
+		}
+
+		// Normalize filterbanks
+		for (let filterbankIndex = 0; filterbankIndex < 80; filterbankIndex++) {
+			let sum = 0
+			let sumOfSquares = 0
+
+			for (let i = 0; i < melSpectogram.length; i++) {
+				const value = melSpectogram[i][filterbankIndex]
+
+				sum += value
+				sumOfSquares += value ** 2
+			}
+
+			const mean = sum / melSpectogram.length
+			const normalizationFactor = 1 / (Math.sqrt(sumOfSquares / melSpectogram.length) + 1e-40)
+
+			for (let i = 0; i < melSpectogram.length; i++) {
+				melSpectogram[i][filterbankIndex] -= mean
+				melSpectogram[i][filterbankIndex] *= normalizationFactor
+			}
+		}
+
+		// Flatten
+		const flattenedMelSpectogram = concatFloat32Arrays(melSpectogram)
+
+		// Initialize session
+		await this.initializeSessionIfNeeded()
+
+		const session = this.session!
+
+		const Onnx = await import('onnxruntime-node')
+
+		const inputTensor = new Onnx.Tensor('float32', flattenedMelSpectogram, [1, melSpectogram.length / 2, 80 * 2])
+
+		const attentionMask = new Int32Array(melSpectogram.length / 2).fill(1)
+		const attentionMaskTensor = new Onnx.Tensor('int32', attentionMask, [1, attentionMask.length])
+
+		// Run inference
+		const outputs = await session.run({ 'input_features': inputTensor, 'attention_mask': attentionMaskTensor })
+
+		// Return output
+		const lastHiddenStateData = outputs['last_hidden_state'].data as Float32Array
+
+		const outputEmbeddings = splitFloat32Array(lastHiddenStateData, outputs['last_hidden_state'].dims[2])
+
+		return outputEmbeddings
+	}
+
+	private async initializeSessionIfNeeded() {
+		if (this.session) {
+			return
+		}
+
+		const Onnx = await import('onnxruntime-node')
+
+		const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.executionProviders })
+
+		this.session = await Onnx.InferenceSession.create(this.modelFilePath, onnxSessionOptions)
+	}
+}

package/src/speech-language-detection/SileroLanguageDetection.ts

@@ -3,9 +3,10 @@ import { softmax } from '../math/VectorMath.js'
 import { Logger } from '../utilities/Logger.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { readAndParseJsonFile } from '../utilities/FileSystem.js'
-import { detectSpeechLanguageByParts, type LanguageDetectionResults } from '../api/LanguageDetection.js'
+import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
 import { languageCodeToName } from '../utilities/Locale.js'
 import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
+import { LanguageDetectionResults } from '../api/LanguageDetectionCommon.js'
 
 export async function detectLanguage(
 	rawAudio: RawAudio,

package/src/subtitles/Subtitles.ts

@@ -1,4 +1,4 @@
-import { convert as convertHtmlToText } from 'html-to-text'
+import { htmlToText } from 'html-to-text'
 
 import { formatHMS, formatMS, secondsToHMS, secondsToMS, startsWithAnyOf } from '../utilities/Utilities.js'
 import { isWord, isWordOrSymbolWord } from '../nlp/Segmentation.js'
@@ -68,7 +68,7 @@ export function subtitlesToTimeline(subtitles: string, removeMarkup = true) {
 
 		plainText = plainText.replaceAll(/<[^>]*>/g, '')
 
-		plainText = convertHtmlToText(plainText, { wordwrap: false })
+		plainText = htmlToText(plainText, { wordwrap: false })
 
 		plainText = plainText.replaceAll(/\s+/g, ' ').trim()
 

package/src/synthesis/GoogleTranslateTTS.ts

@@ -6,6 +6,7 @@ import { trimAudioEnd } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { getShortLanguageCode } from '../utilities/Locale.js'
+import { getChromeOnWindowsHeaders } from '../utilities/BrowserRequestHeaders.js'
 
 const log = logToStderr
 
@@ -98,27 +99,11 @@ export async function synthesizeShortText(text: string, languageCode = 'en', tld
 		},
 
 		headers: {
-			'sec-ch-ua': `".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"`,
-			'x-same-domain': '1',
-			'dnt': '1',
-			'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
-			'sec-ch-ua-mobile': '?0',
-			'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
-			'sec-ch-ua-arch': 'x86',
-			'sec-ch-ua-full-version': '103.0.5060.114',
-			'sec-ch-ua-platform-version': '10.0.0',
-			'sec-ch-ua-full-version-list': `".Not/A)Brand";v="99.0.0.0", "Google Chrome";v="103.0.5060.114", "Chromium";v="103.0.5060.114"`,
-			'sec-ch-ua-bitness': '64',
-			'sec-ch-ua-model': '',
-			'sec-ch-ua-platform': 'Windows',
-			'accept': '*/*',
-			'origin': 'https://translate.google.com',
-			'sec-fetch-site': 'same-origin',
-			'sec-fetch-mode': 'cors',
-			'sec-fetch-dest': 'empty',
-			'referer': 'https://translate.google.com/',
-			'accept-encoding': 'gzip, deflate, br',
-			'accept-language': 'en-US,en;q=0.9',
+			...getChromeOnWindowsHeaders({
+				origin: 'https://translate.google.${tld}',
+				referrer: `https://translate.google.${tld}/`
+			}),
+			'Content-Type': 'application/x-www-form-urlencoded',
 		},
 
 		body: requestBody,
@@ -140,6 +125,7 @@ export async function synthesizeShortText(text: string, languageCode = 'en', tld
 	}
 
 	const resultMp3Stream = Buffer.concat(audioChunks)
+
 	return resultMp3Stream
 }
 

package/src/synthesis/VitsTTS.ts

@@ -252,6 +252,12 @@ export const voiceList: SynthesisVoice[] = [
 		gender: 'male',
 	},
 
+	{
+		name: 'cy_GB-gwryw_gogleddol-medium',
+		languages: ['cy-GB', 'cy'],
+		gender: 'male',
+	},
+
 	{
 		name: 'da_DK-nst_talesyntese-medium',
 		languages: ['da-DK', 'da'],
@@ -375,7 +381,11 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-GB', 'en'],
 		gender: 'female',
 	},
-
+	{
+		name: 'en_GB-cori-medium',
+		languages: ['en-GB', 'en'],
+		gender: 'female',
+	},
 
 	{
 		name: 'en_US-amy-low',
@@ -481,6 +491,21 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-US', 'en'],
 		gender: 'female',
 	},
+	{
+		name: 'en_US-norman-medium',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'en_US-john-medium',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'en_US-bryce-medium',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
 
 	{
 		name: 'es_ES-carlfm-x_low',
@@ -531,7 +556,6 @@ export const voiceList: SynthesisVoice[] = [
 		gender: 'male',
 	},
 
-
 	{
 		name: 'fi_FI-harri-low',
 		languages: ['fi-FI', 'fi'],
@@ -618,6 +642,11 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['it-IT', 'it'],
 		gender: 'male',
 	},
+	{
+		name: 'it_IT-paola-medium',
+		languages: ['it-IT', 'it'],
+		gender: 'female',
+	},
 
 	{
 		name: 'ka_GE-natia-medium',
@@ -699,7 +728,6 @@ export const voiceList: SynthesisVoice[] = [
 		gender: 'male',
 	},
 
-
 	{
 		name: 'no_NO-talesyntese-medium',
 		languages: ['no-NO', 'no'],

package/src/tests/Test.ts

@@ -1,8 +1,5 @@
-import { getTokenRepetitionScore, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
 import { makeTarballsForInstalledPackages } from '../utilities/TarballMaker.js'
-import { defaultEspeakOptions, listVoices, synthesize, testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from '../synthesis/EspeakTTS.js'
-import { getRawAudioDuration } from '../audio/AudioUtilities.js'
-import { alignText } from '../alignment/TextAlignment.js'
+import { logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
 
 const log = logToStderr
 
@@ -11,41 +8,7 @@ setupProgramTerminationListeners()
 //process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
 //process.env.http_proxy = 'http://localhost:8080'
 
-//const testText = `After a while, finding that nothing more happened, she decided on going into the garden at once; but, alas for poor Alice! when she got to the door, she found she had forgotten the little golden key, and when she went back to the table for it, she found she could not possibly reach it: she could see it quite plainly through the glass, and she tried her best to climb up one of the legs of the table, but it was too slippery; and when she had tired herself out with trying, the poor little thing sat down and cried.`
-//await testKirshenbaumPhonemization(testText)
-
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'c', 'a', 'b', 'c'])
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'd', 'a', 'b', 'a', 'd'])
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'b', 'c', 'a', 'b', 'a', 'b'])
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'a', 'a', 'a', 'b'])
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
-//getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
-
 //await makeTarballsForInstalledPackages(true)
 
-/*
-const voices = await listVoices()
-
-for (const voice of voices) {
-	const result = await synthesize('|', { ...defaultEspeakOptions, voice: voice.identifier })
-
-	if (getRawAudioDuration(result.rawAudio) > 0.25) {
-		log(voice.identifier)
-	}
-}
-*/
-/*
-await alignText(
-	`
-Una mañana de invierno, un coche tirado por tres caballos pasó por en medio de Uzquiano y sin detenerse, siguió camino de Peña Cerrada. El coche había salido de Vitoria horas antes y llevaba tres viajeros, una muchachita vestida de blanco, talle alto, gabán, esclavina, gran sombrero Pamela, de moda por los años treinta y cinco al cuarenta, una criada vieja, de aspecto de dueña, enlutada, con peluca rojiza y toca blanca, y un hombre joven, alto, elegante, vestido de negro, con pantalón estrecho, entravillado y sombrero de copa.
-`,
-	'es',
-	`
-One winter morning, a car drawn by three horses passed through the middle of Uzquiano and without stopping, continued on its way to Peña Cerrada. The car had left Vitoria hours before and was carrying three passengers, a little girl dressed in white, high-waisted, overcoat, cape, large Pamela hat, fashionable in the 1930s to 1940s, an old maid, with the appearance of a mistress, in mourning, with a reddish wig and a white cap, and a young, tall, elegant man, dressed in black, with narrow pants, a bow tie and a top hat.
-	`,
-	'en',
-)
-*/
-
 process.exit(0)
 

package/src/text-language-detection/FastTextLanguageDetection.ts

@@ -1,8 +1,8 @@
 import path from 'path'
-import type { LanguageDetectionResults } from '../api/LanguageDetection.js'
 import { languageCodeToName } from '../utilities/Locale.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { resolveModuleMainPath, roundToDigits } from '../utilities/Utilities.js'
+import { LanguageDetectionResults } from '../api/LanguageDetectionCommon.js'
 
 let fastTextLanguageDetectionModel: any
 

package/src/text-language-detection/TinyLDLanguageDetection.ts

@@ -1,6 +1,6 @@
-import type { LanguageDetectionResults } from '../api/LanguageDetection.js'
 import { detectAll } from 'tinyld'
 import { languageCodeToName } from '../utilities/Locale.js'
+import { LanguageDetectionResults } from '../api/LanguageDetectionCommon.js'
 
 export async function detectLanguage(text: string) {
 	const tinyldResults = detectAll(text)

package/src/text-translation/DeepLTextTranslation.ts

@@ -0,0 +1,88 @@
+import { request } from "gaxios"
+import { Logger } from "../utilities/Logger.js"
+import { splitToSentences } from "../nlp/Segmentation.js"
+import { TranslationPair } from "../api/TextTranslation.js"
+import { getChromeOnWindowsHeaders } from "../utilities/BrowserRequestHeaders.js"
+
+export async function translateText(text: string, sourceLanguage: string, targetLanguage: string): Promise<TranslationPair[]> {
+	const logger = new Logger()
+
+	logger.start(`Prepare request`)
+
+	const textSentences = splitToSentences(text, sourceLanguage)
+
+	const requestBodyObject = {
+		jsonrpc: '2.0',
+		method: 'LMT_handle_jobs',
+		params: {
+			jobs: [
+				{
+					kind: 'default',
+					sentences: [
+						{
+							text: textSentences[0],
+							id: 1,
+							prefix: '',
+						}
+					],
+					raw_en_context_before: [],
+					raw_en_context_after: [],
+					preferred_num_beams: 4,
+					quality: 'fast',
+				}
+			],
+
+			lang: {
+				target_lang: targetLanguage.toUpperCase(),
+
+				preference: {
+					weight: {},
+					default: 'default',
+				},
+
+				source_lang_computed: sourceLanguage.toUpperCase(),
+			},
+
+			priority: -1,
+
+			commonJobParams: {
+				mode: 'translate',
+				browserType: 1,
+				textType: 'plaintext',
+			},
+
+			timestamp: Date.now(),
+		},
+
+		id: 756456347,
+	}
+
+	const response = await request<any>({
+		method: 'POST',
+
+		url: `https://www2.deepl.com/jsonrpc`,
+
+		params: {
+			'method': 'LMT_handle_jobs',
+		},
+
+		headers: {
+			...getChromeOnWindowsHeaders({
+				origin: 'https://www.deepl.com',
+				referrer: `https://www.deepl.com/`
+			}),
+
+			'Content-Type': 'application/json',
+		},
+
+		body: JSON.stringify(requestBodyObject),
+
+		responseType: 'json'
+	})
+
+	logger.start('Parse response')
+
+	logger.end()
+
+	return []
+}