@omnimedia/omnitool 1.1.0-9 → 1.1.0-90

package/s/timeline/parts/waveform/parts/collect.ts

@@ -0,0 +1,72 @@
+
+import {Driver} from "../../../../driver/driver.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export const PEAK_LEVELS = [2048, 1024, 512, 256, 128, 64, 32] as const
+
+export async function collectPeakLevels(driver: Driver, source: DecoderSource) {
+	const duration = (await driver.getAudioDuration(source)) ?? 0
+	const readable = driver.decodeAudio({source}).readable
+	const finestSamplesPerPeak = PEAK_LEVELS[PEAK_LEVELS.length - 1]
+	const finestPeaks: number[] = []
+
+	let currentMax = 0
+	let sampleCount = 0
+	let sampleRate = 0
+
+	for await (const audioData of readable) {
+		sampleRate ||= audioData.sampleRate
+
+		const frames = audioData.numberOfFrames
+		const plane = new Float32Array(frames)
+		audioData.copyTo(plane, {planeIndex: 0})
+
+		for (let i = 0; i < plane.length; i++) {
+			const amplitude = Math.abs(plane[i]!)
+			if (amplitude > currentMax) currentMax = amplitude
+
+			sampleCount++
+			if (sampleCount >= finestSamplesPerPeak) {
+				finestPeaks.push(currentMax)
+				currentMax = 0
+				sampleCount = 0
+			}
+		}
+
+		audioData.close()
+	}
+
+	if (sampleCount > 0) finestPeaks.push(currentMax)
+
+	const base = new Float32Array(finestPeaks)
+	const levels = PEAK_LEVELS.map(samplesPerPeak => {
+		const factor = Math.max(1, Math.round(samplesPerPeak / finestSamplesPerPeak))
+		const peaks = factor === 1 ? base : downsampleMax(base, factor)
+		return {
+			samplesPerPeak,
+			peaks,
+			peaksPerSecond: sampleRate > 0 ? sampleRate / samplesPerPeak : 0,
+		}
+	})
+
+	return {duration, levels}
+}
+
+function downsampleMax(peaks: Float32Array, factor: number) {
+	const downsampled = new Float32Array(Math.ceil(peaks.length / factor))
+
+	for (let i = 0; i < downsampled.length; i++) {
+		let maxPeak = 0
+		const start = i * factor
+		const end = Math.min(start + factor, peaks.length)
+
+		for (let j = start; j < end; j++) {
+			if (peaks[j]! > maxPeak) maxPeak = peaks[j]!
+		}
+
+		downsampled[i] = maxPeak
+	}
+
+	return downsampled
+}
+

package/s/timeline/parts/waveform/parts/render.ts

@@ -0,0 +1,45 @@
+
+export function renderTile(
+	peaks: Float32Array,
+	opts: {
+		width: number
+		height: number
+		color: string
+	}
+) {
+	const dpr = typeof window !== "undefined" ? window.devicePixelRatio || 1 : 1
+	const canvas = document.createElement("canvas")
+	canvas.width = opts.width * dpr
+	canvas.height = opts.height * dpr
+	canvas.style.width = `${opts.width}px`
+	canvas.style.height = `${opts.height}px`
+
+	const ctx = canvas.getContext("2d")
+	if (!ctx) return canvas
+
+	ctx.scale(dpr, dpr)
+	ctx.fillStyle = opts.color
+
+	const centerY = opts.height / 2
+	const columns = Math.max(1, opts.width)
+	const peaksPerPixel = peaks.length / columns
+
+	for (let px = 0; px < columns; px++) {
+		const startIndex = Math.floor(px * peaksPerPixel)
+		const endIndex = Math.max(
+			startIndex + 1,
+			Math.floor((px + 1) * peaksPerPixel)
+		)
+
+		let maxPeak = 0
+		for (let i = startIndex; i < endIndex && i < peaks.length; i++) {
+			if (peaks[i]! > maxPeak) maxPeak = peaks[i]!
+		}
+
+		const barHeight = maxPeak * opts.height
+		ctx.fillRect(px, centerY - barHeight / 2, 1, barHeight)
+	}
+
+	return canvas
+}
+

package/s/timeline/parts/waveform/parts/types.ts

@@ -0,0 +1,24 @@
+export interface WaveformTileData {
+	startTime: number
+	endTime: number
+	peaks: Float32Array
+	canvas: HTMLCanvasElement
+}
+
+export interface WaveformOptions {
+	tileSize?: number
+	zoom?: number
+	tileWidth?: number
+	tileHeight?: number
+	preloadMargin?: number
+	color?: string
+	onChange?: (tiles: WaveformTileData[]) => void
+}
+
+export type WaveformTimeRange = [start: number, end: number]
+
+export type WaveformPeakLevel = {
+	samplesPerPeak: number
+	peaks: Float32Array
+	peaksPerSecond: number
+}

package/s/timeline/parts/waveform/waveform.ts

@@ -0,0 +1,161 @@
+
+import {renderTile} from "./parts/render.js"
+import {Driver} from "../../../driver/driver.js"
+import {collectPeakLevels} from "./parts/collect.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {WaveformOptions, WaveformPeakLevel, WaveformTileData, WaveformTimeRange} from "./parts/types.js"
+
+const MAX_TILE_WIDTH = 4096
+
+export class Waveform {
+	#tiles = new Map<number, WaveformTileData>()
+	#activeRange: WaveformTimeRange = [0, 0]
+	#visibleRange: WaveformTimeRange = [0, 0]
+
+	#zoom
+	#levels
+	#onChange
+	#updateQueued = false
+
+	readonly color
+	readonly duration
+	readonly tileSize
+	readonly tileHeight
+	readonly preloadMargin
+
+	private constructor(levels: WaveformPeakLevel[], duration: number, options: WaveformOptions) {
+		this.#levels = levels
+		this.duration = duration
+		this.tileSize = options.tileSize ?? 1
+		this.#zoom = options.zoom ?? ((options.tileWidth ?? 256) / this.tileSize)
+		this.tileHeight = options.tileHeight ?? 96
+		this.preloadMargin = options.preloadMargin ?? 2
+		this.color = options.color ?? "rgb(3, 148, 129)"
+		this.#onChange = options.onChange
+	}
+
+	static async init(driver: Driver, source: DecoderSource, options: WaveformOptions = {}) {
+		const {duration, levels} = await collectPeakLevels(driver, source)
+		return new Waveform(levels, duration, options)
+	}
+
+	/** Waveform render density in pixels per second. */
+	set zoom(pixelsPerSecond: number) {
+		const next = Math.max(1, pixelsPerSecond)
+		if (next === this.#zoom)
+			return
+
+		this.#zoom = next
+		this.#tiles.clear()
+		this.#queueUpdate()
+	}
+
+	get zoom() {
+		return this.#zoom
+	}
+
+	get range() {
+		return this.#visibleRange
+	}
+
+	#computeActiveRange([start, end]: WaveformTimeRange, margin = 1): WaveformTimeRange {
+		const visibleSize = end - start
+		return [
+			Math.max(0, start - visibleSize * margin),
+			Math.min(this.duration, end + visibleSize * margin),
+		]
+	}
+
+	set range(visibleRange: WaveformTimeRange) {
+		this.#visibleRange = visibleRange
+
+		const [visibleStart, visibleEnd] = visibleRange
+		const visibleSize = visibleEnd - visibleStart
+		const [activeStart, activeEnd] = this.#activeRange
+
+		const leftBuffered = activeStart > 0
+		const rightBuffered = activeEnd < this.duration
+		const leftSettled = !leftBuffered || visibleStart >= activeStart + visibleSize
+		const rightSettled = !rightBuffered || visibleEnd <= activeEnd - visibleSize
+
+		if (leftSettled && rightSettled) return
+
+		this.#activeRange = this.#computeActiveRange(visibleRange, this.preloadMargin)
+		this.#queueUpdate()
+	}
+
+	#queueUpdate() {
+		if (this.#updateQueued) return
+		this.#updateQueued = true
+
+		queueMicrotask(() => {
+			this.#updateQueued = false
+			this.#generateTiles()
+		})
+	}
+
+	#generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededStarts = new Set<number>()
+		const level = this.#levelForZoom()
+
+		const firstStart = Math.max(0, Math.floor(rangeStart / this.tileSize) * this.tileSize)
+		const lastStart = Math.min(this.duration, rangeEnd)
+
+		for (let startTime = firstStart; startTime <= lastStart; startTime += this.tileSize) {
+			neededStarts.add(startTime)
+		}
+
+		for (const startTime of neededStarts) {
+			if (!this.#tiles.has(startTime)) {
+				const endTime = Math.min(startTime + this.tileSize, this.duration)
+				this.#tiles.set(startTime, this.#buildTileData(startTime, endTime, level))
+			}
+		}
+
+		for (const startTime of this.#tiles.keys()) {
+			if (!neededStarts.has(startTime)) this.#tiles.delete(startTime)
+		}
+
+		this.#emit()
+	}
+
+	#buildTileData(startTime: number, endTime: number, level: WaveformPeakLevel): WaveformTileData {
+		const peaks = this.#slicePeaks(level, startTime, endTime)
+		return {
+			startTime,
+			endTime,
+			peaks,
+			canvas: renderTile(peaks, {
+				width: this.#tilePixelWidth(startTime, endTime),
+				height: this.tileHeight,
+				color: this.color,
+			}),
+		}
+	}
+
+	#levelForZoom() {
+		return this.#levels.find(level => level.peaksPerSecond >= this.#zoom)
+			?? this.#levels[this.#levels.length - 1]!
+	}
+
+	#slicePeaks(level: WaveformPeakLevel, startTime: number, endTime: number) {
+		if (!level.peaksPerSecond) return new Float32Array()
+		const from = Math.max(0, Math.floor(startTime * level.peaksPerSecond))
+		const to = Math.max(from + 1, Math.min(level.peaks.length, Math.ceil(endTime * level.peaksPerSecond)))
+		return level.peaks.slice(from, to)
+	}
+
+	#tilePixelWidth(startTime: number, endTime: number) {
+		return Math.min(MAX_TILE_WIDTH, Math.max(1, Math.ceil((endTime - startTime) * this.#zoom)))
+	}
+
+	#emit() {
+		if (!this.#onChange) return
+		this.#onChange([...this.#tiles.values()].sort((a, b) => a.startTime - b.startTime))
+	}
+
+	getTiles() {
+		return this.#tiles
+	}
+}

package/s/timeline/renderers/export/parts/audio-gain.ts

@@ -0,0 +1,17 @@
+// this is mutating fn
+export const applyGainToPlanar = (
+	planes: Float32Array[],
+	gain: number
+) => {
+	if (gain === 1)
+		return
+	// planes
+
+	for (const plane of planes) {
+		for (let i = 0; i < plane.length; i++) {
+			plane[i] *= gain
+		}
+	}
+
+	// return planes
+}

package/s/timeline/renderers/export/parts/audio-mix.ts

@@ -0,0 +1,133 @@
+
+export type AudioPlanesInput = {
+	planes: Float32Array[]
+	sampleRate: number
+	timestamp: number
+}
+
+export type MixedChunk = {
+	planar: Float32Array
+	sampleRate: number
+	channels: number
+	frames: number
+	startFrame: number
+}
+
+type ActiveSample = {
+	startFrame: number
+	endFrame: number
+	data: Float32Array[]
+}
+
+export type AudioMixOptions = {
+	chunkFrames?: number
+	clamp?: boolean
+}
+
+export class AudioMix {
+	readonly #chunkFrames: number
+	readonly #clamp: boolean
+
+	constructor(options: AudioMixOptions = {}) {
+		this.#chunkFrames = options.chunkFrames ?? 1024
+		this.#clamp = options.clamp ?? true
+	}
+
+	async *mix(samples: AsyncIterable<AudioPlanesInput>): AsyncGenerator<MixedChunk> {
+		const chunkFrames = this.#chunkFrames
+		let sampleRate: number | null = null
+		let channels: number | null = null
+		const active: ActiveSample[] = []
+		let nextFrame = 0
+		let maxEnd = 0
+
+		for await (const input of samples) {
+			if (channels === null) {
+				channels = input.planes.length
+				sampleRate = input.sampleRate
+			} else {
+				if (input.planes.length !== channels) throw new Error(`Channel count changed`)
+				if (input.sampleRate !== sampleRate) throw new Error(`Sample rate changed`)
+			}
+
+			const inputStart = Math.round(input.timestamp * sampleRate)
+			const frames = input.planes[0]?.length ?? 0
+
+			while (nextFrame + chunkFrames <= inputStart) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+
+			active.push({
+				startFrame: inputStart,
+				endFrame: inputStart + frames,
+				data: input.planes
+			})
+
+			maxEnd = Math.max(maxEnd, inputStart + frames)
+		}
+
+		if (channels !== null && sampleRate !== null) {
+			while (nextFrame < maxEnd) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+		}
+	}
+
+	#processChunk(
+		active: ActiveSample[],
+		currentStartFrame: number,
+		channels: number,
+		sampleRate: number
+	): MixedChunk {
+		const chunkFrames = this.#chunkFrames
+		const outputBuffer = new Float32Array(channels * chunkFrames)
+		const chunkEnd = currentStartFrame + chunkFrames
+
+		for (let ch = 0; ch < channels; ch++) {
+			const channelOffset = ch * chunkFrames
+			const outChannelView = outputBuffer.subarray(channelOffset, channelOffset + chunkFrames)
+
+			for (const sample of active) {
+				const data = sample.data[ch]
+				if (!data) continue
+
+				const start = Math.max(currentStartFrame, sample.startFrame)
+				const end = Math.min(chunkEnd, sample.endFrame)
+
+				if (start >= end) continue
+
+				const dstIdx = start - currentStartFrame
+				const srcIdx = start - sample.startFrame
+				const len = end - start
+
+				for (let i = 0; i < len; i++) {
+					outChannelView[dstIdx + i] += data[srcIdx + i]
+				}
+			}
+
+			if (this.#clamp) {
+				for (let i = 0; i < chunkFrames; i++) {
+					const v = outChannelView[i]
+					outChannelView[i] = v < -1.0 ? -1.0 : (v > 1.0 ? 1.0 : v)
+				}
+			}
+		}
+
+		for (let i = active.length - 1; i >= 0; i--) {
+			if (active[i].endFrame <= chunkEnd) {
+				active.splice(i, 1)
+			}
+		}
+
+		return {
+			planar: outputBuffer,
+			sampleRate,
+			channels,
+			frames: chunkFrames,
+			startFrame: currentStartFrame
+		}
+	}
+}
+