echogarden 0.12.2 → 1.0.0

package/src/audio/AudioPlayer.ts

@@ -2,11 +2,11 @@ import { parentPort } from 'node:worker_threads'
 
 import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 
-import { RawAudio, encodeWaveBuffer, fadeAudioInOut, getRawAudioDuration, sliceRawAudioByTime } from "./AudioUtilities.js"
+import { RawAudio, encodeRawAudioToWave, fadeAudioInOut, getRawAudioDuration, sliceRawAudioByTime } from './AudioUtilities.js'
 import * as AudioBufferConversion from './AudioBufferConversion.js'
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
-import { Timer } from "../utilities/Timer.js"
+import { Timer } from '../utilities/Timer.js'
 import { getRandomHexString, waitTimeout, writeToStderr } from '../utilities/Utilities.js'
 import { encodeToAudioBuffer } from './AudioBufferConversion.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
@@ -26,7 +26,7 @@ export async function playAudioFileWithTimelineFile(audioFilename: string, timel
 
 	let transcript: string | undefined
 	if (transcriptFileName) {
-		transcript = await readFile(transcriptFileName, "utf8")
+		transcript = await readFile(transcriptFileName, 'utf8')
 	}
 
 	await playAudioWithWordTimeline(rawAudio, timeline, transcript)
@@ -58,7 +58,7 @@ export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline
 			let wordEndOffset = entry.endOffsetUtf16
 
 			if (wordStartOffset == null || wordEndOffset == null) {
-				//writeToStderr(` [No offset availble for '${entry.text}'] `)
+				//writeToStderr(` [No offset available for '${entry.text}'] `)
 
 				continue
 			}
@@ -75,18 +75,18 @@ export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline
 		}
 	}
 
-	writeToStderr("\n")
+	writeToStderr('\n')
 
 	const signalChannel = new SignalChannel()
 
 	const keypressListenerStartTimestamp = Date.now()
 
 	function keypressHandler(message: any) {
-		if (message.name == "keypress" &&
+		if (message.name == 'keypress' &&
 			message.key.name == 'return' &&
 			message.timestamp >= keypressListenerStartTimestamp) {
 
-			signalChannel.send("abort")
+			signalChannel.send('abort')
 		}
 	}
 
@@ -96,7 +96,7 @@ export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline
 
 	parentPort?.off('message', keypressHandler)
 
-	writeToStderr("\n")
+	writeToStderr('\n')
 }
 
 export async function playAudioWithTimelinePhones(rawAudio: RawAudio, timeline: Timeline) {
@@ -108,7 +108,7 @@ export async function playAudioWithTimelinePhones(rawAudio: RawAudio, timeline:
 			const wordEntry = timeline[wordIndex]
 			const phoneTimeline = wordEntry.timeline!
 
-			if (phoneTimeline.every(phoneEntry => phoneEntry.text == "")) {
+			if (phoneTimeline.every(phoneEntry => phoneEntry.text == '')) {
 				continue
 			}
 
@@ -127,17 +127,17 @@ export async function playAudioWithTimelinePhones(rawAudio: RawAudio, timeline:
 		}
 	}
 
-	writeToStderr("\n")
+	writeToStderr('\n')
 
 	await playAudioSamples(rawAudio, onTimePosition)
 
-	writeToStderr("\n")
+	writeToStderr('\n')
 }
 
 
 export async function playAudioPairWithTimelineInterleaved(rawAudio1: RawAudio, rawAudio2: RawAudio, timeline1: Timeline, timeline2: Timeline) {
 	if (timeline1.length != timeline2.length) {
-		throw new Error("Timelines have different lengths")
+		throw new Error('Timelines have different lengths')
 	}
 
 	for (let i = 0; i < timeline1.length; i++) {
@@ -149,7 +149,7 @@ export async function playAudioPairWithTimelineInterleaved(rawAudio1: RawAudio,
 		await waitTimeout(500)
 	}
 
-	writeToStderr("\n")
+	writeToStderr('\n')
 }
 
 export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel, microFadeInOut = true) {
@@ -167,15 +167,15 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 
 		const soxPath = await tryResolvingSoxPath()
 
-		let aborted = false
-
 		if (!soxPath) {
 			throw new Error(`Couldn't find or install the SoX utility. Please install the SoX utility on your system path to enable audio playback.`)
 		}
 
+		let aborted = false
+
 		let streamToStdin = true
 
-		if (process.platform == "darwin") {
+		if (process.platform == 'darwin') {
 			streamToStdin = false
 		}
 
@@ -200,7 +200,7 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 			)
 		} else {
 			tempFilePath = path.join(getAppTempDir(appName), `${getRandomHexString(16)}.wav`)
-			const waveFileBuffer = encodeWaveBuffer(rawAudio)
+			const waveFileBuffer = encodeRawAudioToWave(rawAudio)
 			await outputFile(tempFilePath, waveFileBuffer)
 
 			playerProcess = spawn(
@@ -211,32 +211,32 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 		}
 
 		if (signalChannel) {
-			signalChannel.on("abort", () => {
+			signalChannel.on('abort', () => {
 				aborted = true
 				playerProcess.kill('SIGKILL')
 			})
 		}
 
 		// Required to work around SoX bug:
-		playerProcess.stderr.on("data", (data) => {
+		playerProcess.stderr.on('data', (data) => {
 			//writeToStderr(data.toString('utf-8'))
 		})
 
-		playerProcess.stdout.on("data", (data) => {
+		playerProcess.stdout.on('data', (data) => {
 			//writeToStderr(data.toString('utf-8'))
 		})
 
-		playerProcess.once("spawn", () => {
+		playerProcess.once('spawn', () => {
 			if (audioBuffer != undefined) {
 				playerProcess.stdin!.write(audioBuffer)
 				playerProcess.stdin!.end()
-				playerProcess.stdin!.on("error", () => { })
+				playerProcess.stdin!.on('error', () => { })
 			}
 
 			playerSpawnedOpenPromise.resolve(null)
 		})
 
-		playerProcess.once("error", async (e) => {
+		playerProcess.once('error', async (e) => {
 			await cleanup()
 			playerProcessClosed = true
 
@@ -291,7 +291,7 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 			sampleRate: rawAudio.sampleRate,
 		})
 
-		speaker.on("error", (e: any) => {
+		speaker.on('error', (e: any) => {
 			reject(e)
 		})
 
@@ -306,7 +306,7 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 		let mpg123AudioBufferSize: number
 		let mpg123AudioBufferDuration: number
 
-		if (process.platform == "win32") {
+		if (process.platform == 'win32') {
 			mpg123AudioBufferSize = 65536
 			mpg123AudioBufferDuration = byteCountToDuration(mpg123AudioBufferSize)
 		} else {
@@ -356,4 +356,4 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 }
 
 export const charactersToWriteAhead =
-	[",", ".", "，", "、", "：", "；", "。", ":", ";", "?", "!", ")", "]", "}", "\"", "'", "”", "’", "-", "—", "»"]
+	[',', '.', '，', '、', '：', '；', '。', ':', ';', '?', '!', ')', ']', '}', `"`, `'`, '”', '’', '-', '—', '»']

package/src/audio/AudioRecorder.ts

@@ -1,9 +1,9 @@
 import { spawn } from 'child_process'
 
-import { RawAudio, concatAudioSegments } from "./AudioUtilities.js"
+import { RawAudio, concatAudioSegments } from './AudioUtilities.js'
 import * as AudioBufferConversion from './AudioBufferConversion.js'
 
-import { Timer } from "../utilities/Timer.js"
+import { Timer } from '../utilities/Timer.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { SampleFormat } from '../codecs/WaveCodec.js'
@@ -37,7 +37,7 @@ export function captureAudioInput(channelCount = 1, sampleRate = 48000, maxTime
 		const soxPath = await tryResolvingSoxPath()
 
 		if (!soxPath) {
-			throw new Error("Could not resolve a SoX executable")
+			throw new Error('Could not resolve a SoX executable')
 		}
 
 		let args: string[]
@@ -51,7 +51,7 @@ export function captureAudioInput(channelCount = 1, sampleRate = 48000, maxTime
 				'-c', `${channelCount}`, '-r', `${sampleRate}`, '-e', 'signed',
 				'-t', 'raw', '-']
 		} else {
-			throw new Error("")
+			throw new Error('')
 		}
 
 		const recorder = spawn(
@@ -60,7 +60,7 @@ export function captureAudioInput(channelCount = 1, sampleRate = 48000, maxTime
 			{}
 		)
 
-		recorder.once("spawn", () => {
+		recorder.once('spawn', () => {
 			recorderSpawnedOpenPromise.resolve(null)
 			timer.restart()
 		})

package/src/audio/AudioUtilities.ts

@@ -1,16 +1,17 @@
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
-import { SampleFormat, encodeWave, decodeWave, BitDepth } from "../codecs/WaveCodec.js"
-import { resampleAudioSpeex } from "../dsp/SpeexResampler.js"
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { SampleFormat, encodeWave, decodeWave, BitDepth } from '../codecs/WaveCodec.js'
+import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
+import { Timeline } from '../utilities/Timeline.js'
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
 // Wave encoding and decoding
 ////////////////////////////////////////////////////////////////////////////////////////////////
-export function encodeWaveBuffer(rawAudio: RawAudio, bitDepth: BitDepth = 16, sampleFormat: SampleFormat = SampleFormat.PCM, speakerPositionMask = 0) {
+export function encodeRawAudioToWave(rawAudio: RawAudio, bitDepth: BitDepth = 16, sampleFormat: SampleFormat = SampleFormat.PCM, speakerPositionMask = 0) {
 	return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask)
 }
 
-export function decodeWaveBuffer(waveFileBuffer: Buffer, ignoreTruncatedChunks = false) {
+export function decodeWaveToRawAudio(waveFileBuffer: Buffer, ignoreTruncatedChunks = false) {
 	return decodeWave(waveFileBuffer, ignoreTruncatedChunks)
 }
 
@@ -31,6 +32,10 @@ export function trimAudioStart(audioSamples: Float32Array, targetStartSilentSamp
 }
 
 export function trimAudioEnd(audioSamples: Float32Array, targetEndSilentSampleCount = 0, amplitudeThresholdDecibels = defaultSilenceThresholdDecibels) {
+	if (audioSamples.length === 0) {
+		return new Float32Array(0)
+	}
+
 	const silentSampleCount = getEndingSilentSampleCount(audioSamples, amplitudeThresholdDecibels)
 
 	const trimmedAudio = audioSamples.subarray(0, audioSamples.length - silentSampleCount)
@@ -76,21 +81,25 @@ export function getEndingSilentSampleCount(audioSamples: Float32Array, amplitude
 ////////////////////////////////////////////////////////////////////////////////////////////////
 // Gain, normalization, mixing, and channel downmixing
 ////////////////////////////////////////////////////////////////////////////////////////////////
-export function downmixToMonoAndNormalize(rawAudio: RawAudio, targetPeakDb = -3) {
-	return normalizeAudioLevel(downmixToMono(rawAudio), targetPeakDb)
+export function downmixToMonoAndNormalize(rawAudio: RawAudio, targetPeakDecibels = -3) {
+	return normalizeAudioLevel(downmixToMono(rawAudio), targetPeakDecibels)
 }
 
-export function normalizeAudioLevel(rawAudio: RawAudio, targetPeakDb = -3, maxIncreaseDb = 30): RawAudio {
+export function attenuateIfClipping(rawAudio: RawAudio) {
+	return normalizeAudioLevel(rawAudio, -0.1, 0)
+}
+
+export function normalizeAudioLevel(rawAudio: RawAudio, targetPeakDecibels = -3, maxGainIncreaseDecibels = 30): RawAudio {
 	//rawAudio = correctDCBias(rawAudio)
 
-	const targetPeakAmplitude = decibelsToGainFactor(targetPeakDb)
-	const maxGainFactor = decibelsToGainFactor(maxIncreaseDb)
+	const targetPeakAmplitude = decibelsToGainFactor(targetPeakDecibels)
+	const maxGainFactor = decibelsToGainFactor(maxGainIncreaseDecibels)
 
-	const peakAmplitude = getAudioPeakAmplitude(rawAudio.audioChannels)
+	const peakAmplitude = getSamplePeakAmplitude(rawAudio.audioChannels)
 
 	const gainFactor = Math.min(targetPeakAmplitude / peakAmplitude, maxGainFactor)
 
-	return applyGain(rawAudio, gainFactor)
+	return applyGainFactor(rawAudio, gainFactor)
 }
 
 export function correctDCBias(rawAudio: RawAudio): RawAudio {
@@ -119,11 +128,11 @@ export function correctDCBias(rawAudio: RawAudio): RawAudio {
 	return { audioChannels: outputAudioChannels, sampleRate: rawAudio.sampleRate } as RawAudio
 }
 
-export function applyGainDecibels(rawAudio: RawAudio, decibelGain: number): RawAudio {
-	return applyGain(rawAudio, decibelsToGainFactor(decibelGain))
+export function applyGainDecibels(rawAudio: RawAudio, gainDecibels: number): RawAudio {
+	return applyGainFactor(rawAudio, decibelsToGainFactor(gainDecibels))
 }
 
-export function applyGain(rawAudio: RawAudio, gainFactor: number): RawAudio {
+export function applyGainFactor(rawAudio: RawAudio, gainFactor: number): RawAudio {
 	const outputAudioChannels: Float32Array[] = []
 
 	for (const channelSamples of rawAudio.audioChannels) {
@@ -142,8 +151,13 @@ export function applyGain(rawAudio: RawAudio, gainFactor: number): RawAudio {
 }
 
 export function downmixToMono(rawAudio: RawAudio): RawAudio {
+	const channelCount = rawAudio.audioChannels.length
 	const sampleCount = rawAudio.audioChannels[0].length
 
+	if (channelCount === 1) {
+		return cloneRawAudio(rawAudio)
+	}
+
 	const downmixedAudio = new Float32Array(sampleCount)
 
 	for (const channelSamples of rawAudio.audioChannels) {
@@ -152,14 +166,20 @@ export function downmixToMono(rawAudio: RawAudio): RawAudio {
 		}
 	}
 
+	if (channelCount > 1) {
+		for (let i = 0; i < sampleCount; i++) {
+			downmixedAudio[i] /= channelCount
+		}
+	}
+
 	return { audioChannels: [downmixedAudio], sampleRate: rawAudio.sampleRate } as RawAudio
 }
 
-export function getAudioPeakDecibels(audioChannels: Float32Array[]) {
-	return gainFactorToDecibels(getAudioPeakAmplitude(audioChannels))
+export function getSamplePeakDecibels(audioChannels: Float32Array[]) {
+	return gainFactorToDecibels(getSamplePeakAmplitude(audioChannels))
 }
 
-export function getAudioPeakAmplitude(audioChannels: Float32Array[]) {
+export function getSamplePeakAmplitude(audioChannels: Float32Array[]) {
 	let maxAmplitude = 0.00001
 
 	for (const channelSamples of audioChannels) {
@@ -173,11 +193,11 @@ export function getAudioPeakAmplitude(audioChannels: Float32Array[]) {
 
 export function mixAudio(rawAudio1: RawAudio, rawAudio2: RawAudio) {
 	if (rawAudio1.audioChannels.length != rawAudio2.audioChannels.length) {
-		throw new Error("Can't mix audio of unequal channel counts")
+		throw new Error(`Can't mix audio of unequal channel counts`)
 	}
 
 	if (rawAudio1.sampleRate != rawAudio2.sampleRate) {
-		throw new Error("Can't mix audio of different sample rates")
+		throw new Error(`Can't mix audio of different sample rates`)
 	}
 
 	const mixedAudioChannels: Float32Array[] = []
@@ -217,6 +237,14 @@ export function sliceRawAudio(rawAudio: RawAudio, startSampleIndex: number, endS
 }
 
 export function sliceAudioChannels(audioChannels: Float32Array[], startSampleIndex: number, endSampleIndex: number) {
+	if (audioChannels.length === 0) {
+		throw new Error('audioChannels array is empty')
+	}
+
+	if (startSampleIndex > endSampleIndex) {
+		throw new Error('startSampleIndex must be less or equal to endSampleIndex')
+	}
+
 	const channelCount = audioChannels.length
 
 	const outAudioChannels: Float32Array[] = []
@@ -246,6 +274,40 @@ export function concatAudioSegments(audioSegments: Float32Array[][]) {
 	return outAudioChannels
 }
 
+export function cropToTimeline(rawAudio: RawAudio, timeline: Timeline) {
+	const sampleRate = rawAudio.sampleRate
+	const channelCount = rawAudio.audioChannels.length
+	const sampleCount = rawAudio.audioChannels[0].length
+
+	const audioSegments: Float32Array[][] = []
+
+	for (let i = 0; i < timeline.length; i++) {
+		const entry = timeline[i]
+		const startTime = entry.startTime
+		const endTime = entry.endTime
+
+		const startSampleOffset = Math.max(Math.floor(startTime * sampleRate), 0)
+		const endSampleOffset = Math.min(Math.floor(endTime * sampleRate), sampleCount)
+
+		const segment: Float32Array[] = []
+
+		for (let c = 0; c < channelCount; c++) {
+			segment.push(rawAudio.audioChannels[c].subarray(startSampleOffset, endSampleOffset))
+		}
+
+		audioSegments.push(segment)
+	}
+
+	if (audioSegments.length > 0) {
+		const croppedAudioChannels =  concatAudioSegments(audioSegments)
+		const croppedRawAudio: RawAudio = { audioChannels: croppedAudioChannels, sampleRate: rawAudio.sampleRate }
+
+		return croppedRawAudio
+	} else {
+		return getEmptyRawAudio(channelCount, sampleRate)
+	}
+}
+
 export function fadeAudioInOut(rawAudio: RawAudio, fadeTime: number): RawAudio {
 	const fadeSampleCount = Math.floor(rawAudio.sampleRate * fadeTime)
 	const gainReductionPerFrameDecibels = -60 / fadeSampleCount
@@ -326,14 +388,19 @@ export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: nu
 			inputAsRawAudio = downmixToMono(inputAsRawAudio)
 		}
 
-		if (outChannelCount != null && outChannelCount >= 2 && outChannelCount != inputAudioChannelCount) {
-			throw new Error(`Can't convert ${inputAudioChannelCount} channels to ${outChannelCount} channels. Channel conversion of raw audio currently only supports downmixing to mono.`)
+		if (outChannelCount == 2 && inputAudioChannelCount == 1) {
+			inputAsRawAudio = cloneRawAudio(inputAsRawAudio)
+			inputAsRawAudio.audioChannels.push(inputAsRawAudio.audioChannels[0].slice())
+		}
+
+		if (outChannelCount != null && outChannelCount > 2 && outChannelCount != inputAudioChannelCount) {
+			throw new Error(`Can't convert ${inputAudioChannelCount} channels to ${outChannelCount} channels. Channel conversion of raw audio currently only supports mono and stereo inputs.`)
 		}
 
 		if (outSampleRate && inputAsRawAudio.sampleRate != outSampleRate) {
 			inputAsRawAudio = await resampleAudioSpeex(inputAsRawAudio, outSampleRate)
 		}
-	} else if (typeof input == "string" || input instanceof Uint8Array) {
+	} else if (typeof input == 'string' || input instanceof Uint8Array) {
 		if (input instanceof Uint8Array && !Buffer.isBuffer(input)) {
 			input = Buffer.from(input)
 		}
@@ -342,12 +409,28 @@ export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: nu
 
 		inputAsRawAudio = await FFMpegTranscoder.decodeToChannels(inputAsStringOrBuffer, outSampleRate, outChannelCount)
 	} else {
-		throw new Error("Received an invalid input audio data type.")
+		throw new Error('Received an invalid input audio data type.')
 	}
 
 	return inputAsRawAudio
 }
 
+export function subtractAudio(audio1: RawAudio, audio2: RawAudio) {
+	const sampleCount = audio1.audioChannels[0].length
+
+	const subtractedAudioChannels = [new Float32Array(sampleCount), new Float32Array(sampleCount)]
+
+	for (let i = 0; i < sampleCount; i++) {
+		for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
+			subtractedAudioChannels[channelIndex][i] = audio1.audioChannels[channelIndex][i] - audio2.audioChannels[channelIndex][i]
+		}
+	}
+
+	const subtractedRawAudio: RawAudio = { audioChannels: subtractedAudioChannels, sampleRate: audio1.sampleRate }
+
+	return subtractedRawAudio
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////
 // Unit conversion
 ////////////////////////////////////////////////////////////////////////////////////////////////