@omnimedia/omnitool 1.1.0-106 → 1.1.0-108

package/README.md

@@ -533,6 +533,15 @@ Notes:
 - Call `player.resize(width, height)` to resize the preview canvas.
 - `playbackRate` supports slower, faster, and reverse visual playback. Audio currently plays only at `1`.
 
+Subscribe to the mixed, post-gain level of selected audio items - for clip, group, or master meters:
+
+```ts
+const off = player.audio.levels.on(
+	() => itemIds,
+	({time, peak, rms}) => {}
+)
+```
+
 ## 📤 Export
 
 ```ts

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-106",
+	"version": "1.1.0-108",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",

package/s/timeline/renderers/parts/audio-level.ts

@@ -0,0 +1,16 @@
+
+export function measureAudioLevel(samples: Float32Array<ArrayBufferLike>) {
+	let peak = 0
+	let sumSquares = 0
+
+	for (const value of samples) {
+		peak = Math.max(peak, Math.abs(value))
+		sumSquares += value ** 2
+	}
+
+	return {
+		peak,
+		rms: samples.length > 0 ? Math.sqrt(sumSquares / samples.length) : 0
+	}
+}
+

package/s/timeline/renderers/parts/samplers/audio/parts/init.ts

@@ -17,14 +17,17 @@ export async function initStreams(
 				return
 			if (item.enabled === false)
 				return
+			if (localTime >= item.duration)
+				return
 
 			const sink = await pool.getSink(item.mediaHash)
 			if (!sink)
 				return
 
 			const mediaTime = item.start + localTime
+			const mediaEnd = item.start + item.duration
 			const offset = seconds((from - mediaTime) / 1000)
-			const iter = sink.samples(mediaTime / 1000)
+			const iter = sink.samples(mediaTime / 1000, mediaEnd / 1000)
 
 			const first = await iter.next()
 			if (first.done)
@@ -39,6 +42,7 @@ export async function initStreams(
 				get currentSample() {return currentSample},
 				timelineTime: () => seconds(offset + currentSample.timestamp),
 				output: () => ({
+					itemId: item.id,
 					sample: currentSample,
 					timestamp: offset + currentSample.timestamp,
 					gain: item.gain ?? 1

package/s/timeline/renderers/parts/samplers/audio/parts/types.ts

@@ -1,5 +1,6 @@
 
 import {AudioSample} from "mediabunny"
+import {Id} from "../../../../../parts/basics.js"
 import {Seconds} from "../../../../../../units/seconds.js"
 
 export type ActiveStream = {
@@ -8,6 +9,7 @@ export type ActiveStream = {
 	currentSample: AudioSample
 	timelineTime: () => Seconds
 	output: () => {
+		itemId: Id
 		sample: AudioSample
 		timestamp: number
 		gain: number

package/s/timeline/renderers/player/parts/audio-levels.ts

@@ -0,0 +1,120 @@
+
+import {Ms} from '../../../../units/ms.js'
+import {Id} from '../../../parts/basics.js'
+import {measureAudioLevel} from '../../parts/audio-level.js'
+import {AudioItemSelector, AudioLevelListener, AudioLevelObserver, Meter} from './types.js'
+
+export class AudioLevels implements AudioLevelObserver {
+	#meters = new Set<Meter>()
+	#items = new Map<GainNode, Id>()
+	#frame: number | null = null
+	#running = false
+
+	constructor(
+		private context: AudioContext,
+		private time: () => Ms,
+		private isPlaying: () => boolean
+	) {}
+
+	on(items: AudioItemSelector, listener: AudioLevelListener) {
+		const silent = this.context.createGain()
+		const analyser = this.context.createAnalyser()
+
+		silent.gain.value = 0
+		analyser.smoothingTimeConstant = 0
+
+		analyser.connect(silent)
+		silent.connect(this.context.destination)
+
+		const meter: Meter = {
+			items,
+			silent,
+			listener,
+			analyser,
+			nodes: new Set(),
+			samples: new Float32Array(analyser.fftSize) as Float32Array<ArrayBuffer>
+		}
+
+		this.#meters.add(meter)
+
+		if (this.isPlaying())
+			this.start()
+
+		return () => {
+			if (!this.#meters.delete(meter))
+				return
+
+			for (const node of meter.nodes)
+				node.disconnect(analyser)
+
+			analyser.disconnect()
+			silent.disconnect()
+
+			if (!this.#meters.size)
+				this.stop()
+		}
+	}
+
+	attach(itemId: Id, node: GainNode) {
+		this.#items.set(node, itemId)
+	}
+
+	detach(node: GainNode) {
+		this.#items.delete(node)
+
+		for (const meter of this.#meters) {
+			if (meter.nodes.delete(node))
+				node.disconnect(meter.analyser)
+		}
+	}
+
+	start() {
+		if (this.#running || !this.#meters.size)
+			return
+
+		this.#running = true
+		this.#frame = requestAnimationFrame(() => this.#tick())
+	}
+
+	stop() {
+		this.#running = false
+
+		if (this.#frame !== null)
+			cancelAnimationFrame(this.#frame)
+
+		this.#frame = null
+	}
+
+	#tick() {
+		if (!this.#running)
+			return
+
+		for (const meter of this.#meters) {
+			this.#sync(meter)
+			meter.analyser.getFloatTimeDomainData(meter.samples)
+			meter.listener({time: this.time(), ...measureAudioLevel(meter.samples)})
+		}
+
+		if (this.#running)
+			this.#frame = requestAnimationFrame(() => this.#tick())
+	}
+
+	#sync(meter: Meter) {
+		const selected = new Set(meter.items())
+
+		for (const [node, itemId] of this.#items) {
+			const connected = meter.nodes.has(node)
+			const shouldConnect = selected.has(itemId)
+
+			if (shouldConnect && !connected) {
+				node.connect(meter.analyser)
+				meter.nodes.add(node)
+			}
+			else if (!shouldConnect && connected) {
+				node.disconnect(meter.analyser)
+				meter.nodes.delete(node)
+			}
+		}
+	}
+}
+

package/s/timeline/renderers/player/parts/playback.ts

@@ -1,4 +1,5 @@
 
+import {AudioLevels} from "./audio-levels.js"
 import {Fps} from '../../../../units/fps.js'
 import {ms, Ms} from '../../../../units/ms.js'
 import {Driver} from '../../../../driver/driver.js'
@@ -12,6 +13,7 @@ import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
 import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
 
 export class Playback {
+	audioLevels
 	audioSampler
 	seekVisualSampler
 	playVisualSampler: CursorVisualSampler | null = null
@@ -26,7 +28,7 @@ export class Playback {
 
 	audioContext = new AudioContext({sampleRate: 48000})
 	audioGain = this.audioContext.createGain()
-	audioNodes = new Set<AudioBufferSourceNode>()
+	audioNodes = new Map<AudioBufferSourceNode, GainNode>()
 	#audioAbort: AbortController | null = null
 
 	constructor(
@@ -36,6 +38,11 @@ export class Playback {
 	) {
 		this.audioGain.connect(this.audioContext.destination)
 		this.audioGain.gain.value = 0.7 ** 2
+		this.audioLevels = new AudioLevels(
+			this.audioContext,
+			() => this.currentTime,
+			() => this.#controller.isPlaying()
+		)
 		this.seekVisualSampler = createVisualSampler(this.resolveMedia)
 		this.audioSampler = createAudioSampler(this.resolveMedia)
 		this.#samples()
@@ -88,12 +95,14 @@ export class Playback {
 		this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
 
 		this.#controller.play()
+		this.audioLevels.start()
 		this.#startAudio()
 	}
 
 	pause() {
 		this.#playbackStart = this.currentTime
 		this.#controller.pause()
+		this.audioLevels.stop()
 		this.#stopAudio()
 
 		if (this.playVisualSampler) {
@@ -165,8 +174,10 @@ export class Playback {
 		this.#audioAbort?.abort()
 		this.#audioAbort = null
 
-		for (const node of this.audioNodes)
+		for (const [node, gain] of this.audioNodes) {
 			node.stop()
+			this.audioLevels.detach(gain)
+		}
 
 		this.audioNodes.clear()
 	}
@@ -184,7 +195,7 @@ export class Playback {
 		if (this.#audioStartSec === null)
 			return
 
-		for await (const {sample, timestamp, gain} of this.audioSampler.sampleAudio(
+		for await (const {itemId, sample, timestamp, gain} of this.audioSampler.sampleAudio(
 			this.timeline, ms(from * 1000)
 		)) {
 
@@ -200,8 +211,14 @@ export class Playback {
 			itemGain.gain.value = gain
 			node.connect(itemGain)
 			itemGain.connect(this.audioGain)
-			node.onended = () => this.audioNodes.delete(node)
-			this.audioNodes.add(node)
+
+			node.onended = () => {
+				this.audioNodes.delete(node)
+				this.audioLevels.detach(itemGain)
+			}
+
+			this.audioNodes.set(node, itemGain)
+			this.audioLevels.attach(itemId, itemGain)
 
 			const startAt = this.#audioStartSec + timestamp - from
 

package/s/timeline/renderers/player/parts/types.ts

@@ -0,0 +1,26 @@
+
+import {Ms} from "../../../../units/ms.js"
+import {Id} from "../../../parts/basics.js"
+
+export type AudioLevel = {
+	time: Ms
+	peak: number
+	rms: number
+}
+
+export type AudioItemSelector = () => Iterable<Id>
+export type AudioLevelListener = (level: AudioLevel) => void
+
+export type AudioLevelObserver = {
+	on(items: AudioItemSelector, listener: AudioLevelListener): () => void
+}
+
+export type Meter = {
+	items: AudioItemSelector
+	listener: AudioLevelListener
+	analyser: AnalyserNode
+	silent: GainNode
+	samples: Float32Array<ArrayBuffer>
+	nodes: Set<GainNode>
+}
+

package/s/timeline/renderers/player/player.ts

@@ -4,6 +4,7 @@ import {fps} from "../../../units/fps.js"
 import {Playback} from "./parts/playback.js"
 import {Driver} from "../../../driver/driver.js"
 import {TimelineFile} from "../../parts/basics.js"
+import {AudioLevelObserver} from "./parts/types.js"
 import {DecoderSource} from "../../../driver/fns/schematic.js"
 
 type ResolveMedia = (hash: string) => DecoderSource
@@ -11,6 +12,7 @@ type ResolveMedia = (hash: string) => DecoderSource
 export class VideoPlayer {
 	canvas: HTMLCanvasElement
 	playback: Playback
+	readonly audio: {levels: AudioLevelObserver}
 
 	#pendingSeek: number | null = null
 	#flushTask: Promise<void> | null = null
@@ -21,6 +23,7 @@ export class VideoPlayer {
 		timeline: TimelineFile
 	) {
 		this.playback = new Playback(driver, timeline, resolveMedia)
+		this.audio = {levels: this.playback.audioLevels}
 		this.canvas = driver.compositor.pixi.renderer.canvas
 	}