echogarden 1.6.2 → 1.7.0

package/src/text-translation/GoogleTranslateTextTranslation.ts

@@ -1,35 +1,55 @@
 import { request } from "gaxios"
 import { Logger } from "../utilities/Logger.js"
-import { Fragment, splitToFragments, splitToLines, splitToParagraphs, splitToSentences } from "../nlp/Segmentation.js"
+import { Fragment, splitToFragments, splitToLines } from "../nlp/Segmentation.js"
 import { TranslationPair } from "../api/TextTranslation.js"
 import { getChromeOnWindowsHeaders, getChromeOnAndroidHeaders } from "../utilities/BrowserRequestHeaders.js"
-import { logToStderr, splitAndPreserveSeparators } from "../utilities/Utilities.js"
+import { logToStderr } from "../utilities/Utilities.js"
 import { getShortLanguageCode } from "../utilities/Locale.js"
+import { PlainTextOptions } from "../api/Common.js"
+import { extendDeep } from "../utilities/ObjectUtilities.js"
+import { splitAndPreserveSeparators } from "../utilities/StringUtilities.js"
 
 const log = logToStderr
 
-export async function translateText(text: string, sourceLanguage: string, targetLanguage: string, tld = 'com') {
+export async function translateText(
+	text: string,
+	sourceLanguage: string,
+	targetLanguage: string,
+	plainTextOptions: PlainTextOptions,
+	options: GoogleTranslateTextTranslationOptions) {
+
 	const logger = new Logger()
 
-	const maxTextLengthPerRequest = 2000
+	if (!supportsLanguage(sourceLanguage)) {
+		throw new Error(`Language code ${sourceLanguage} is not supported by the Google Translate engine. Supported language codes are ${supportedLanguageCodes.join(', ')}`)
+	}
+
+	if (!supportsLanguage(targetLanguage)) {
+		throw new Error(`Language code ${sourceLanguage} is not supported by the Google Translate engine. Supported language codes are ${supportedLanguageCodes.join(', ')}`)
+	}
+
+	options = extendDeep(defaultGoogleTranslateTextTranslationOptions, options)
+
+	const maxCharactersPerPart = options.maxCharactersPerPart!
 
-	//const paragraphs = splitAndPreserveSeparators(text, /(\r?\n)(\r?\n)/g)
-	const paragraphs = splitToParagraphs(text, 'double', 'preserve')
+	const paragraphSeperatorPattern = plainTextOptions.paragraphBreaks === 'double' ? /(\r?\n)(\r?\n)+/g : /(\r?\n)+/g
+
+	const paragraphs = splitAndPreserveSeparators(text, paragraphSeperatorPattern)
 
 	const fragmentsForParagraph: Fragment[][] = []
 
 	for (const paragraph of paragraphs) {
-		const fragments = await splitToFragments(paragraph, maxTextLengthPerRequest, getShortLanguageCode(sourceLanguage))
+		const fragments = await splitToFragments(paragraph, maxCharactersPerPart, getShortLanguageCode(sourceLanguage))
 
 		fragmentsForParagraph.push(fragments)
 	}
 
-	const fragmentBundles: Fragment[][] = [[]]
+	const parts: Fragment[][] = [[]]
 
-	function totalLengthOfLastBundle() {
+	function totalLengthOfLastPart() {
 		let total = 0
 
-		for (const fragment of fragmentBundles[fragmentBundles.length - 1]) {
+		for (const fragment of parts[parts.length - 1]) {
 			total += fragment.length
 		}
 
@@ -42,14 +62,14 @@ export async function translateText(text: string, sourceLanguage: string, target
 		const fragments = fragmentsForParagraph[paragraphIndex]
 
 		for (const fragment of fragments) {
-			let lastBundle = fragmentBundles[fragmentBundles.length - 1]
+			let lastPart = parts[parts.length - 1]
 
-			if (totalLengthOfLastBundle() + fragment.length > maxTextLengthPerRequest) {
-				lastBundle = []
-				fragmentBundles.push(lastBundle)
+			if (totalLengthOfLastPart() + fragment.length > maxCharactersPerPart) {
+				lastPart = []
+				parts.push(lastPart)
 			}
 
-			lastBundle.push(fragment)
+			lastPart.push(fragment)
 
 			fragmentToParagraphIndex.set(fragment, paragraphIndex)
 		}
@@ -57,25 +77,25 @@ export async function translateText(text: string, sourceLanguage: string, target
 
 	const translatedFragmentsForParagraphs = paragraphs.map(_ => [] as string[])
 
-	for (let fragmentBundleIndex = 0; fragmentBundleIndex < fragmentBundles.length; fragmentBundleIndex++) {
-		const bundle = fragmentBundles[fragmentBundleIndex]
+	for (let partIndex = 0; partIndex < parts.length; partIndex++) {
+		const part = parts[partIndex]
 
-		const joinedBundle = bundle.map(x => x.text.replaceAll('|', ' ')).join(` | `)
+		const joinedFragmentsInPart = part.map(x => x.text.replaceAll('|', ' ')).join(`\n|\n`)
 
-		logger.logTitledMessage(`\nTranslate part ${fragmentBundleIndex + 1} of ${fragmentBundles.length}`, joinedBundle)
+		logger.logTitledMessage(`\nTranslate part ${partIndex + 1} of ${parts.length}`, joinedFragmentsInPart.replaceAll('\n|\n', ''))
 
 		logger.start(`Request translation from Google Translate`)
-		const fragmentTranslationPair = await translateText_MobileWeb(joinedBundle, sourceLanguage, targetLanguage)
+		const fragmentTranslationPair = await translateText_MobileWeb(joinedFragmentsInPart, sourceLanguage, targetLanguage, options)
 		logger.end()
 
-		const translatedTextForBundle = fragmentTranslationPair[0].translatedText
+		const translatedTextForPart = fragmentTranslationPair[0].translatedText
 
-		logger.logTitledMessage(`Translated part`, `"${translatedTextForBundle}"`)
+		logger.logTitledMessage(`Translated part`, `"${translatedTextForPart.replaceAll(' | ', '\n') }"`)
 
-		const splitTranslation = translatedTextForBundle.split(`|`)
+		const splitTranslation = translatedTextForPart.split(`|`)
 
-		for (let fragmentIndex = 0; fragmentIndex < bundle.length; fragmentIndex++) {
-			const fragment = bundle[fragmentIndex]
+		for (let fragmentIndex = 0; fragmentIndex < part.length; fragmentIndex++) {
+			const fragment = part[fragmentIndex]
 			const translatedFragment = splitTranslation[fragmentIndex].trim()
 
 			const paragraphIndex = fragmentToParagraphIndex.get(fragment)!
@@ -87,18 +107,46 @@ export async function translateText(text: string, sourceLanguage: string, target
 	const translationPairs: TranslationPair[] = []
 
 	for (let paragraphIndex = 0; paragraphIndex < paragraphs.length; paragraphIndex++) {
+		const sourceParagraph = paragraphs[paragraphIndex]
+		const translatedParagraph = translatedFragmentsForParagraphs[paragraphIndex].join(' ')
+
+		let translatedParagraphWithWhitespace = ''
+
+		const sourceParagraphLeadingWhitespaceMatches = sourceParagraph.match(/^\s+/)
+
+		if (sourceParagraphLeadingWhitespaceMatches) {
+			translatedParagraphWithWhitespace += sourceParagraphLeadingWhitespaceMatches[0]
+		}
+
+		translatedParagraphWithWhitespace += translatedParagraph
+
+		const sourceParagraphTrailingWhitespaceMatches = sourceParagraph.match(/\s+$/)
+
+		if (sourceParagraphTrailingWhitespaceMatches) {
+			translatedParagraphWithWhitespace += sourceParagraphTrailingWhitespaceMatches[0]
+		}
+
 		translationPairs.push({
 			sourceText: paragraphs[paragraphIndex],
-			translatedText: translatedFragmentsForParagraphs[paragraphIndex].join(' ')
+			translatedText: translatedParagraphWithWhitespace,
 		})
 	}
 
-	const translatedText = translationPairs.map(pair => pair.translatedText).join('\n\n')
+	const translatedParagraphs = translationPairs.map(pair => pair.translatedText)
+
+	const translatedText = translatedParagraphs.join('')
 
 	return { translationPairs, translatedText }
 }
 
-export async function translateText_MobileWeb(text: string, sourceLanguage: string, targetLanguage: string, tld = 'com') {
+export async function translateText_MobileWeb(
+	text: string,
+	sourceLanguage: string,
+	targetLanguage: string,
+	options: GoogleTranslateTextTranslationOptions) {
+
+	const tld = options.tld
+
 	const logger = new Logger()
 
 	logger.start(`Request translation from Google Translate`)
@@ -419,11 +467,11 @@ function normalizeLanguageCodeForGoogleTranslate(languageCode: string) {
 }
 
 export function supportsLanguage(langCode: string) {
-	return supportedLanguages.includes(normalizeLanguageCodeForGoogleTranslate(langCode))
+	return supportedLanguageCodes.includes(normalizeLanguageCodeForGoogleTranslate(langCode))
 }
 
 // 243 Languages supported
-export const supportedLanguages = [
+export const supportedLanguageCodes = [
 	'ab', // Abkhaz
 	'ace', // Acehnese
 	'ach', // Acholi
@@ -464,6 +512,7 @@ export const supportedLanguages = [
 	'ch', // Chamorro
 	'ce', // Chechen
 	'ny', // Chichewa
+	'zh', // Chinese (same as zh-CN)
 	'zh-CN', // Chinese (Simplified)
 	'zh-TW', // Chinese (Traditional)
 	'chk', // Chuukese
@@ -505,6 +554,7 @@ export const supportedLanguages = [
 	'cnh', // Hakha Chin
 	'ha', // Hausa
 	'haw', // Hawaiian
+	'he', // Hebrew
 	'iw', // Hebrew
 	'hil', // Hiligaynon
 	'hi', // Hindi
@@ -520,6 +570,7 @@ export const supportedLanguages = [
 	'it', // Italian
 	'jam', // Jamaican Patois
 	'ja', // Japanese
+	'jv', // Javanese
 	'jw', // Javanese
 	'kac', // Jingpo
 	'kl', // Kalaallisut
@@ -668,3 +719,13 @@ export const supportedLanguages = [
 	'zap', // Zapotec
 	'zu', // Zulu
 ]
+
+export interface GoogleTranslateTextTranslationOptions {
+	tld?: string
+	maxCharactersPerPart?: number
+}
+
+export const defaultGoogleTranslateTextTranslationOptions: GoogleTranslateTextTranslationOptions = {
+	tld: 'com',
+	maxCharactersPerPart: 2000,
+}

package/src/utilities/OnnxUtilities.ts

@@ -1,19 +1,11 @@
 import type * as Onnx from 'onnxruntime-node'
 
 export function getOnnxSessionOptions(options: OnnxSessionOptions) {
-
 	const onnxOptions: Onnx.InferenceSession.SessionOptions = {
 		executionProviders: ['cpu'],
 		logSeverityLevel: 3,
 	}
 
-	function dmlProviderAvailable() {
-		const platform = process.platform
-		const arch = process.arch
-
-		return platform === 'win32' && arch === 'x64'
-	}
-
 	if (options) {
 		if (options.executionProviders != null) {
 			let executionProviders = options.executionProviders.filter(provider => {
@@ -54,6 +46,13 @@ export function makeOnnxLikeFloat32Tensor(onnxTensor: Onnx.Tensor): OnnxLikeFloa
 	}
 }
 
+export function dmlProviderAvailable() {
+	const platform = process.platform
+	const arch = process.arch
+
+	return platform === 'win32' && arch === 'x64'
+}
+
 export interface OnnxLikeFloat32Tensor {
 	readonly data: Float32Array
 	readonly dims: number[]

package/src/utilities/PackageManager.ts

@@ -157,6 +157,9 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
 	'mdxnet-UVR_MDXNET_2_9682': '20240330',
 	'mdxnet-UVR_MDXNET_3_9662': '20240330',
 	'mdxnet-UVR_MDXNET_KARA': '20240330',
+	'mdxnet-UVR_MDXNET_Main': '20241008',
+	'mdxnet-Kim_Vocal_1': '20241008',
+	'mdxnet-Kim_Vocal_2': '20241008',
 
 	// whisper.cpp models
 	'whisper.cpp-tiny': '20240405',

package/src/utilities/StringUtilities.ts

@@ -0,0 +1,203 @@
+export function includesAnyOf(str: string, substrings: string[]) {
+	return indexOfAnyOf(str, substrings) >= 0
+}
+
+export function indexOfAnyOf(str: string, substrings: string[]) {
+	for (const substring of substrings) {
+		const index = str.indexOf(substring)
+
+		if (index >= 0) {
+			return index
+		}
+	}
+
+	return -1
+}
+
+export function startsWithAnyOf(str: string, prefixes: string[]) {
+	for (const prefix of prefixes) {
+		if (str.startsWith(prefix)) {
+			return true
+		}
+	}
+
+	return false
+}
+
+export function formatHMS(timeHMS: { hours: number, minutes: number, seconds: number, milliseconds: number }, decimalSeparator = '.') {
+	return `${formatIntegerWithLeadingZeros(timeHMS.hours, 2)}:${formatIntegerWithLeadingZeros(timeHMS.minutes, 2)}:${formatIntegerWithLeadingZeros(timeHMS.seconds, 2)}${decimalSeparator}${formatIntegerWithLeadingZeros(timeHMS.milliseconds, 3)}`
+}
+
+export function formatMS(timeMS: { minutes: number, seconds: number, milliseconds: number }, decimalSeparator = '.') {
+	return `${formatIntegerWithLeadingZeros(timeMS.minutes, 2)}:${formatIntegerWithLeadingZeros(timeMS.seconds, 2)}${decimalSeparator}${formatIntegerWithLeadingZeros(timeMS.milliseconds, 3)}`
+}
+
+export function formatIntegerWithLeadingZeros(num: number, minDigitCount: number) {
+	num = Math.floor(num)
+
+	let numAsString = `${num}`
+
+	while (numAsString.length < minDigitCount) {
+		numAsString = `0${numAsString}`
+	}
+
+	return numAsString
+}
+
+export function formatListWithQuotedElements(strings: string[], quoteSymbol = `'`) {
+	return strings.map(str => `${quoteSymbol}${str}${quoteSymbol}`).join(', ')
+}
+
+export function getUTF32Chars(str: string) {
+	const utf32chars: string[] = []
+	const mapping: number[] = []
+
+	let utf32Index = 0
+
+	for (const utf32char of str) {
+		utf32chars.push(utf32char)
+
+		for (let i = 0; i < utf32char.length; i++) {
+			mapping.push(utf32Index)
+		}
+
+		utf32Index += 1
+	}
+
+	mapping.push(utf32Index)
+
+	return { utf32chars, mapping }
+}
+
+export function containsInvalidCodepoint(str: string) {
+	for (const char of str) {
+		if (char.codePointAt(0) === 65533) {
+			return true
+		}
+	}
+
+	return false
+}
+
+
+export function splitAndPreserveSeparators(text: string, separatorRegex: RegExp): string[] {
+	if (!separatorRegex.flags.includes('g')) {
+		throw new Error('RegExp must be global')
+	}
+
+	// Use the match method to find all matches for the separators
+	const matches = text.match(separatorRegex)
+
+	// If no matches are found, return the original text as a single element array
+	if (!matches) {
+		return [text]
+	}
+
+	// Initialize the result array
+	const result: string[] = []
+
+	// Initialize the start position
+	let lastIndex = 0
+
+	// Iterate through the matches
+	matches.forEach(match => {
+		// Get the index of the current match
+		const matchIndex = text.indexOf(match, lastIndex)
+
+		// Add the substring before the match to the result, joined with the match itself
+		result.push(text.substring(lastIndex, matchIndex) + match)
+
+		// Update the last index to the end of the current match
+		lastIndex = matchIndex + match.length
+	})
+
+	// Add the remaining substring after the last match to the result
+	{
+		const remainingText = text.substring(lastIndex)
+
+		if (remainingText.length > 0) {
+			result.push(remainingText)
+		}
+	}
+
+	return result
+}
+
+export function getTokenRepetitionScore(tokens: string[] | number[]) {
+	const maxCycleLength = Math.floor(tokens.length / 2)
+
+	const matchLengthForCycleLength: number[] = [0]
+
+	for (let cycleLength = 1; cycleLength <= maxCycleLength; cycleLength++) {
+		let matchCount = 0
+
+		for (let leftIndex = cycleLength; leftIndex < tokens.length; leftIndex++) {
+			const referenceIndex = leftIndex - cycleLength
+
+			if (tokens[leftIndex] !== tokens[referenceIndex]) {
+				break
+			}
+
+			matchCount += 1
+		}
+
+		const score = matchCount
+
+		matchLengthForCycleLength.push(score)
+	}
+
+	let longestMatch = -Infinity
+	let longestCycleRepetition = -Infinity
+
+	for (let i = 1; i <= matchLengthForCycleLength.length; i++) {
+		const matchLength = matchLengthForCycleLength[i]
+
+		if (matchLength > longestMatch) {
+			longestMatch = matchLength
+		}
+
+		const cycleCount = (matchLength / i) + 1
+
+		if (cycleCount > longestCycleRepetition) {
+			longestCycleRepetition = cycleCount
+		}
+	}
+
+	return { longestMatch, longestCycleRepetition }
+}
+
+export async function convertHtmlToText(html: string) {
+	const { htmlToText } = await import('html-to-text')
+
+	const text = htmlToText(html, {
+		wordwrap: false,
+
+		selectors: [
+			{ selector: 'a', options: { ignoreHref: true } },
+			{ selector: 'img', format: 'skip' },
+			{ selector: 'h1', options: { uppercase: false } },
+			{ selector: 'h2', options: { uppercase: false } },
+			{ selector: 'h3', options: { uppercase: false } },
+			{ selector: 'h4', options: { uppercase: false } },
+			{ selector: 'table', options: { uppercaseHeaderCells: false } }
+		]
+	})
+
+	return text || ''
+}
+
+export function substituteCharactersUsingLookup(text: string, substitutionLookup: Record<string, string>) {
+	let resultText = ''
+
+	for (const char of text) {
+		const substitution = substitutionLookup[char]
+
+		if (substitution !== undefined) {
+			resultText += substitution
+		} else {
+			resultText += char
+		}
+	}
+
+	return resultText
+}

package/src/utilities/TarballMaker.ts

@@ -1,28 +1,11 @@
 import path from 'path'
 import { createTarball } from './Compression.js'
-import { ensureDir, existsSync, move, readdir, stat } from './FileSystem.js'
-import { ensureAndGetPackagesDir } from './PackageManager.js'
+import { ensureDir, move, readdir, stat } from './FileSystem.js'
 import { appName } from '../api/Common.js'
 import { getRandomHexString } from './Utilities.js'
 import { getAppTempDir } from './PathUtilities.js'
 
-const tarballDir = '../resources/tarballs'
-
-export async function makeTarballsForInstalledPackages(skipIfExists = false) {
-	const packagesDir = await ensureAndGetPackagesDir()
-	const packageList = await readdir(packagesDir)
-
-	for (const packageName of packageList) {
-		if (skipIfExists && existsSync(path.join(tarballDir, `${packageName}.tar.gz`))) {
-			continue
-		}
-
-		const packagePath = path.join(packagesDir, packageName)
-		await createNamedTarball(packagePath, packageName)
-	}
-}
-
-export async function createNamedTarball(inputPath: string, name: string) {
+export async function createNamedTarball(inputPath: string, name: string, tarballDir: string) {
 	const tempDir = getAppTempDir(appName)
 	await ensureDir(tempDir)
 	const tempFilename = path.join(tempDir, getRandomHexString(16))
@@ -34,7 +17,7 @@ export async function createNamedTarball(inputPath: string, name: string) {
 	await move(tempFilename, targetFilname)
 }
 
-export async function createTarballForEachDirIn(baseDir: string, namePrefix: String) {
+export async function createTarballForEachDirIn(baseDir: string, namePrefix: string, tarballDir: string) {
 	for (const dirName of await readdir(baseDir)) {
 		const dirPath = path.join(baseDir, dirName)
 
@@ -46,11 +29,11 @@ export async function createTarballForEachDirIn(baseDir: string, namePrefix: Str
 
 		const archiveName = `${namePrefix}-${dirName}`
 
-		await createNamedTarball(dirPath, archiveName)
+		await createNamedTarball(dirPath, archiveName, tarballDir)
 	}
 }
 
-export async function createTarballForEachFileIn(baseDir: string, namePrefix: String) {
+export async function createTarballForEachFileIn(baseDir: string, namePrefix: string, tarballDir: string) {
 	for (const filename of await readdir(baseDir)) {
 		const filenameWithoutExtension = path.parse(filename).name
 
@@ -64,6 +47,6 @@ export async function createTarballForEachFileIn(baseDir: string, namePrefix: St
 
 		const archiveName = `${namePrefix}-${filenameWithoutExtension}`
 
-		await createNamedTarball(filePath, archiveName)
+		await createNamedTarball(filePath, archiveName, tarballDir)
 	}
 }

package/src/utilities/Timeline.ts

@@ -1,7 +1,8 @@
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
 import { applyWhitespaceProcessing, isWord, isWordOrSymbolWord, splitToParagraphs, splitToSentences, splitToWords } from '../nlp/Segmentation.js'
 import { deepClone } from './ObjectUtilities.js'
-import { getUTF32Chars, roundToDigits, splitAndPreserveSeparators } from './Utilities.js'
+import { getUTF32Chars, splitAndPreserveSeparators } from './StringUtilities.js'
+import { roundToDigits } from './Utilities.js'
 
 export function addTimeOffsetToTimeline(targetTimeline: Timeline, timeOffset: number) {
 	if (!targetTimeline) {