echogarden 1.8.7 → 2.0.1

package/src/audio/AudioPlayer.ts

@@ -2,37 +2,36 @@ import { parentPort } from 'node:worker_threads'
 
 import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 
-import { RawAudio, encodeRawAudioToWave, fadeAudioInOut, getRawAudioDuration, sliceRawAudioByTime } from './AudioUtilities.js'
-import * as AudioBufferConversion from './AudioBufferConversion.js'
+import { RawAudio, encodeRawAudioToWave, fadeAudioInOut, getRawAudioDuration, sliceAudioChannels, sliceRawAudioByTime } from './AudioUtilities.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
 import { Timer } from '../utilities/Timer.js'
-import { getRandomHexString, waitTimeout, writeToStderr } from '../utilities/Utilities.js'
-import { encodeToAudioBuffer } from './AudioBufferConversion.js'
+import { clip, getRandomHexString, waitTimeout, writeToStderr } from '../utilities/Utilities.js'
+import { encodeToAudioBuffer, float32ToInt16Pcm, interleaveChannels } from './AudioBufferConversion.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { Timeline, addWordTextOffsetsToTimeline } from '../utilities/Timeline.js'
-import { readAndParseJsonFile, readFile, remove, writeFile } from '../utilities/FileSystem.js'
+import { readAndParseJsonFile, readFileAsUtf8, remove, writeFile } from '../utilities/FileSystem.js'
 import { tryResolvingSoxPath } from './SoxPath.js'
 import { SignalChannel } from '../utilities/SignalChannel.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
 import { appName } from '../api/Common.js'
-import path from 'node:path'
-import { getAppTempDir } from '../utilities/PathUtilities.js'
+import { getAppTempDir, joinPath } from '../utilities/PathUtilities.js'
+import { type AudioOutput } from '@echogarden/audio-io'
 
-export async function playAudioFileWithTimelineFile(audioFilename: string, timelineFileName: string, transcriptFileName?: string) {
+export async function playAudioFileWithTimelineFile(audioFilename: string, timelineFileName: string, transcriptFileName?: string, player?: AudioPlayerID) {
 	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioFilename, 48000, 1)
 
 	const timeline = await readAndParseJsonFile(timelineFileName)
 
 	let transcript: string | undefined
 	if (transcriptFileName) {
-		transcript = await readFile(transcriptFileName, 'utf8')
+		transcript = await readFileAsUtf8(transcriptFileName)
 	}
 
-	await playAudioWithWordTimeline(rawAudio, timeline, transcript)
+	await playAudioWithWordTimeline(rawAudio, timeline, transcript, player)
 }
 
-export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline: Timeline, transcript?: string) {
+export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline: Timeline, transcript?: string, player?: AudioPlayerID) {
 	if (!transcript) {
 		transcript = wordTimeline.map(entry => entry.text).join(' ')
 	}
@@ -82,77 +81,178 @@ export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline
 	const keypressListenerStartTimestamp = Date.now()
 
 	function keypressHandler(message: any) {
-		if (message.name == 'keypress' &&
-			message.key.name == 'return' &&
-			message.timestamp >= keypressListenerStartTimestamp) {
+		if (message.name === 'keypress') {
+			//console.log(`Keypress: ${message.key.name}`)
 
-			signalChannel.send('abort')
+			if (message.timestamp < keypressListenerStartTimestamp) {
+				return
+			}
+
+			const keyName = message.key.name
+
+			if (keyName === 'return') {
+				signalChannel.send('abort')
+			} else if (keyName === 'left') {
+				signalChannel.send('skip', -1)
+			} else if (keyName === 'right') {
+				signalChannel.send('skip', 1)
+
+				if (false) {
+					const nextTimelineEntryIndex = timelineEntryIndex
+
+					if (nextTimelineEntryIndex < wordTimeline.length) {
+						//signalChannel.send('skip', 5)
+						const targetTime = wordTimeline[nextTimelineEntryIndex].startTime
+
+						signalChannel.send('seek', targetTime)
+					}
+				}
+			} else if (keyName === 'space') {
+				signalChannel.send('togglePause')
+			}
 		}
 	}
 
 	parentPort?.on('message', keypressHandler)
 
-	await playAudioSamples(rawAudio, onTimePosition, signalChannel)
+	await playAudioSamples(rawAudio, onTimePosition, signalChannel, player)
 
 	parentPort?.off('message', keypressHandler)
 
 	writeToStderr('\n')
 }
 
-export async function playAudioWithTimelinePhones(rawAudio: RawAudio, timeline: Timeline) {
-	let wordIndex = 0
-	let phoneIndex = 0
+export async function playAudioSamplesWithKeyboardControls(rawAudio: RawAudio, player?: AudioPlayerID) {
+	const signalChannel = new SignalChannel()
 
-	function onTimePosition(timePosition: number) {
-		for (; wordIndex < timeline.length; wordIndex++) {
-			const wordEntry = timeline[wordIndex]
-			const phoneTimeline = wordEntry.timeline!
+	const keypressListenerStartTimestamp = Date.now()
 
-			if (phoneTimeline.every(phoneEntry => phoneEntry.text == '')) {
-				continue
-			}
+	function keypressHandler(message: any) {
+		if (message.name === 'keypress') {
+			//console.log(`Keypress: ${message.key.name}`)
 
-			for (; phoneIndex < phoneTimeline.length; phoneIndex++) {
-				const entry = phoneTimeline[phoneIndex]
+			if (message.timestamp < keypressListenerStartTimestamp) {
+				return
+			}
 
-				if (entry.startTime > timePosition) {
-					return
-				}
+			const keyName = message.key.name
 
-				writeToStderr(`${entry.text} `)
+			if (keyName === 'return') {
+				signalChannel.send('abort')
+			} else if (keyName === 'left') {
+				signalChannel.send('skip', -1)
+			} else if (keyName === 'right') {
+				signalChannel.send('skip', 1)
+			} else if (keyName === 'space') {
+				signalChannel.send('togglePause')
 			}
-
-			writeToStderr(`| `)
-			phoneIndex = 0
 		}
 	}
 
-	writeToStderr('\n')
+	parentPort?.on('message', keypressHandler)
 
-	await playAudioSamples(rawAudio, onTimePosition)
+	await playAudioSamples(rawAudio, undefined, signalChannel, player)
 
-	writeToStderr('\n')
+	parentPort?.off('message', keypressHandler)
+}
+
+export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel, player?: AudioPlayerID) {
+	if (!player) {
+		player = 'audio-io'
+	}
+
+	if (player === 'audio-io') {
+		return playAudioSamples_AudioIO(rawAudio, onTimePosition, signalChannel)
+	} else if (player === 'sox') {
+		return playAudioSamples_Sox(rawAudio, onTimePosition, signalChannel)
+	} else {
+		throw new Error(`Unsupported audio player ID: ${player}`)
+	}
 }
 
+export async function playAudioSamples_AudioIO(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel) {
+	const openPromise = new OpenPromise()
+
+	const sampleRate = rawAudio.sampleRate
+	const channelCount = rawAudio.audioChannels.length
+	const bufferDuration = 100.0
+	const audioFrameCount = rawAudio.audioChannels[0].length
+	const audioDuration = getRawAudioDuration(rawAudio)
+
+	const { createAudioOutput } = await import('@echogarden/audio-io')
+
+	let frameOffset = 0
+	let audioOutput: AudioOutput
+	let abortRequested = false
+	let ended = false
+
+	let isPaused = false
+
+	// Define an audio output handler function
+	async function audioOutputHandler(outputBuffer: Int16Array) {
+		if (ended) {
+			return
+		}
+
+		if (onTimePosition) {
+			const audioTime = Math.min(frameOffset / sampleRate, audioDuration)
+
+			onTimePosition(audioTime)
+		}
+
+		if (isPaused) {
+			return
+		}
+
+		const chunkFrameCount = outputBuffer.length / channelCount
+
+		const floatAudioChunk = sliceAudioChannels(rawAudio.audioChannels, frameOffset, frameOffset + chunkFrameCount)
+		const interleavedFloatAudioChunk = interleaveChannels(floatAudioChunk)
+		const int16AudioChunk = float32ToInt16Pcm(interleavedFloatAudioChunk)
 
-export async function playAudioPairWithTimelineInterleaved(rawAudio1: RawAudio, rawAudio2: RawAudio, timeline1: Timeline, timeline2: Timeline) {
-	if (timeline1.length != timeline2.length) {
-		throw new Error('Timelines have different lengths')
+		outputBuffer.set(int16AudioChunk)
+
+		frameOffset += chunkFrameCount
+
+		if (abortRequested || int16AudioChunk.length < outputBuffer.length) {
+			ended = true
+
+			await audioOutput.dispose()
+			openPromise.resolve()
+		}
 	}
 
-	for (let i = 0; i < timeline1.length; i++) {
-		writeToStderr(`${timeline1[i].text} `)
+	if (signalChannel) {
+		signalChannel.on('abort', () => {
+			abortRequested = true
+		})
+
+		signalChannel.on('skip', (durationToSkip: number) => {
+			frameOffset += durationToSkip * sampleRate
+			frameOffset = Math.floor(frameOffset)
+			frameOffset = clip(frameOffset, 0, audioFrameCount)
+		})
 
-		await playAudioSamples(sliceRawAudioByTime(rawAudio1, timeline1[i].startTime, timeline1[i].endTime))
-		await waitTimeout(200)
-		await playAudioSamples(sliceRawAudioByTime(rawAudio2, timeline2[i].startTime, timeline2[i].endTime))
-		await waitTimeout(500)
+		signalChannel.on('seek', (timeToSeekTo: number) => {
+			frameOffset = Math.floor(timeToSeekTo * sampleRate)
+			frameOffset = clip(frameOffset, 0, audioFrameCount)
+		})
+
+		signalChannel.on('togglePause', () => {
+			isPaused = !isPaused
+		})
 	}
 
-	writeToStderr('\n')
+	audioOutput = await createAudioOutput({
+		sampleRate, // Sample rate in Hz, should be an integer like 44100, 22050, 8000
+		channelCount, // Channel count, likely 1 (mono), or 2 (stereo)
+		bufferDuration, // Target buffer duration, in milliseconds. Defaults to 100.0
+	}, audioOutputHandler)
+
+	return openPromise.promise
 }
 
-export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel, microFadeInOut = true) {
+export function playAudioSamples_Sox(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel, microFadeInOut = true) {
 	return new Promise<void>(async (resolve, reject) => {
 		if (microFadeInOut) {
 			rawAudio = fadeAudioInOut(rawAudio, 0.0025)
@@ -180,7 +280,7 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 		}
 
 		let tempFilePath: string | undefined
-		let audioBuffer: Buffer | undefined
+		let audioBuffer: Uint8Array | undefined
 
 		async function cleanup() {
 			if (tempFilePath) {
@@ -199,7 +299,7 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 				{}
 			)
 		} else {
-			tempFilePath = path.join(getAppTempDir(appName), `${getRandomHexString(16)}.wav`)
+			tempFilePath = joinPath(getAppTempDir(appName), `${getRandomHexString(16)}.wav`)
 			const waveFileBuffer = encodeRawAudioToWave(rawAudio)
 			await writeFile(tempFilePath, waveFileBuffer)
 
@@ -282,7 +382,7 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 		}
 
 		const channelCount = rawAudio.audioChannels.length
-		let audioData = AudioBufferConversion.encodeToAudioBuffer(rawAudio.audioChannels)
+		let audioData = encodeToAudioBuffer(rawAudio.audioChannels)
 
 		const { default: Speaker } = await import('speaker')
 
@@ -315,7 +415,7 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 			mpg123AudioBufferSize = bytesPerSecond * mpg123AudioBufferDuration
 		}
 
-		audioData = concatBuffers([audioData, Buffer.alloc(mpg123AudioBufferSize)])
+		audioData = concatUint8Arrays([audioData, new Uint8Array(mpg123AudioBufferSize)])
 
 		const maxChunkSize = mpg123AudioBufferSize
 
@@ -363,3 +463,5 @@ export const charactersToWriteAhead = [
 	')', ']', '}', `"`, `'`, '”', '’',
 	'-', '—', '»', '،', '؟'
 ]
+
+export type AudioPlayerID = 'audio-io' | 'sox'

package/src/audio/AudioUtilities.ts

@@ -11,7 +11,7 @@ export function encodeRawAudioToWave(rawAudio: RawAudio, bitDepth: BitDepth = 16
 	return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask)
 }
 
-export function decodeWaveToRawAudio(waveFileBuffer: Buffer, ignoreTruncatedChunks = true, ignoreOverflowingDataChunks = true) {
+export function decodeWaveToRawAudio(waveFileBuffer: Uint8Array, ignoreTruncatedChunks = true, ignoreOverflowingDataChunks = true) {
 	return decodeWave(waveFileBuffer, ignoreTruncatedChunks, ignoreOverflowingDataChunks)
 }
 
@@ -401,33 +401,25 @@ export function getRawAudioDuration(rawAudio: RawAudio) {
 export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: number, outChannelCount?: number) {
 	let inputAsRawAudio: RawAudio = input as RawAudio
 
-	if (inputAsRawAudio.audioChannels?.length > 0 && inputAsRawAudio.sampleRate) {
+	if (isRawAudio(input)) {
 		const inputAudioChannelCount = inputAsRawAudio.audioChannels.length
 
 		if (outChannelCount == 1 && inputAudioChannelCount > 1) {
 			inputAsRawAudio = downmixToMono(inputAsRawAudio)
-		}
-
-		if (outChannelCount == 2 && inputAudioChannelCount == 1) {
+		} else if (outChannelCount == 2 && inputAudioChannelCount == 1) {
 			inputAsRawAudio = cloneRawAudio(inputAsRawAudio)
 			inputAsRawAudio.audioChannels.push(inputAsRawAudio.audioChannels[0].slice())
-		}
-
-		if (outChannelCount != null && outChannelCount > 2 && outChannelCount != inputAudioChannelCount) {
+		} else if (outChannelCount != null && outChannelCount > 2 && outChannelCount != inputAudioChannelCount) {
 			throw new Error(`Can't convert ${inputAudioChannelCount} channels to ${outChannelCount} channels. Channel conversion of raw audio currently only supports mono and stereo inputs.`)
 		}
 
-		if (outSampleRate && inputAsRawAudio.sampleRate != outSampleRate) {
+		if (outSampleRate && inputAsRawAudio.sampleRate !== outSampleRate) {
 			inputAsRawAudio = await resampleAudioSpeex(inputAsRawAudio, outSampleRate)
 		}
 	} else if (typeof input == 'string' || input instanceof Uint8Array) {
-		if (input instanceof Uint8Array && !Buffer.isBuffer(input)) {
-			input = Buffer.from(input)
-		}
-
-		const inputAsStringOrBuffer = input as string | Buffer
+		const inputAsStringOrUint8Array = input as string | Uint8Array
 
-		inputAsRawAudio = await FFMpegTranscoder.decodeToChannels(inputAsStringOrBuffer, outSampleRate, outChannelCount)
+		inputAsRawAudio = await FFMpegTranscoder.decodeToChannels(inputAsStringOrUint8Array, outSampleRate, outChannelCount)
 	} else {
 		throw new Error('Received an invalid input audio data type.')
 	}
@@ -436,21 +428,45 @@ export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: nu
 }
 
 export function subtractAudio(audio1: RawAudio, audio2: RawAudio) {
-	const sampleCount = audio1.audioChannels[0].length
+	if (audio1.sampleRate !== audio2.sampleRate) {
+		throw new Error(`Audio sequences have different sample rates`)
+	}
+
+	if (audio1.audioChannels.length !== audio2.audioChannels.length) {
+		throw new Error(`Audio sequences have different channel counts`)
+	}
+
+	const sampleRate = audio1.sampleRate
+	const channelCount = audio1.audioChannels.length
+	const sampleCount = Math.min(audio1.audioChannels[0].length, audio2.audioChannels[0].length)
+
+	const subtractedAudioChannels: Float32Array[] = []
+
+	for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) {
+		const subtractedSamples = new Float32Array(sampleCount)
 
-	const subtractedAudioChannels = [new Float32Array(sampleCount), new Float32Array(sampleCount)]
+		const samples1 = audio1.audioChannels[channelIndex]
+		const samples2 = audio2.audioChannels[channelIndex]
 
-	for (let i = 0; i < sampleCount; i++) {
-		for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
-			subtractedAudioChannels[channelIndex][i] = audio1.audioChannels[channelIndex][i] - audio2.audioChannels[channelIndex][i]
+		for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
+			subtractedSamples[sampleIndex] = samples1[sampleIndex] - samples2[sampleIndex]
 		}
+
+		subtractedAudioChannels.push(subtractedSamples)
 	}
 
-	const subtractedRawAudio: RawAudio = { audioChannels: subtractedAudioChannels, sampleRate: audio1.sampleRate }
+	const subtractedRawAudio: RawAudio = {
+		audioChannels: subtractedAudioChannels,
+		sampleRate: sampleRate
+	}
 
 	return subtractedRawAudio
 }
 
+export function isRawAudio(obj: any): obj is RawAudio {
+	return typeof obj === 'object' && typeof obj.sampleRate === 'number' && Array.isArray(obj.audioChannels)
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////
 // Unit conversion
 ////////////////////////////////////////////////////////////////////////////////////////////////
@@ -492,4 +508,4 @@ export type AudioEncoding = {
 	bitrate?: number
 }
 
-export type AudioSourceParam = string | Buffer | Uint8Array | RawAudio
+export type AudioSourceParam = string | Uint8Array | RawAudio

package/src/audio/SoxPath.ts

@@ -1,7 +1,7 @@
-import path from 'path'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { commandExists } from '../utilities/Utilities.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
+import { joinPath } from '../utilities/PathUtilities.js'
 
 export async function tryResolvingSoxPath() {
 	if (getGlobalOption('soxPath')) {
@@ -14,7 +14,7 @@ export async function tryResolvingSoxPath() {
 	if (platform === 'win32') {
 		const soxPackagePath = await loadPackage('sox-14.4.1a-win32')
 
-		return path.join(soxPackagePath, 'sox.exe')
+		return joinPath(soxPackagePath, 'sox.exe')
 	}
 
 	if (await commandExists('sox')) {
@@ -24,7 +24,7 @@ export async function tryResolvingSoxPath() {
 	if (platform === 'linux' && arch === 'x64') {
 		const soxPackagePath = await loadPackage('sox-14.4.2-linux-minimal')
 
-		return path.join(soxPackagePath, 'sox')
+		return joinPath(soxPackagePath, 'sox')
 	}
 
 	return undefined

package/src/build-tools/MakeTarballsForInstalledPackages.ts

@@ -1,6 +1,6 @@
-import path from 'path'
 import { existsSync, readdir } from '../utilities/FileSystem.js'
 import { ensureAndGetPackagesDir } from '../utilities/PackageManager.js'
+import { joinPath } from '../utilities/PathUtilities.js'
 import { createNamedTarball } from '../utilities/TarballMaker.js'
 import { setupProgramTerminationListeners } from '../utilities/Utilities.js'
 
@@ -9,11 +9,11 @@ async function makeTarballsForInstalledPackages(tarballDir: string, skipIfExists
 	const packageList = await readdir(packagesDir)
 
 	for (const packageName of packageList) {
-		if (skipIfExists && existsSync(path.join(tarballDir, `${packageName}.tar.gz`))) {
+		if (skipIfExists && existsSync(joinPath(tarballDir, `${packageName}.tar.gz`))) {
 			continue
 		}
 
-		const packagePath = path.join(packagesDir, packageName)
+		const packagePath = joinPath(packagesDir, packageName)
 		await createNamedTarball(packagePath, packageName, tarballDir)
 	}
 }