@omnimedia/omnitool 1.1.0-53 → 1.1.0-56

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-53",
+	"version": "1.1.0-56",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",

package/s/demo/demo.bundle.ts

@@ -26,8 +26,11 @@ let exportState: {timeline: TimelineFile; omni: Omni} | null = null
 }
 
 const setProgress = (card: HTMLElement, state: "idle" | "running" | "done") => {
-	const progress = card.querySelector(".progress") as HTMLProgressElement
-	const status = card.querySelector(".status") as HTMLSpanElement
+	const progress = card.querySelector(".progress") as HTMLProgressElement | null
+	const status = card.querySelector(".status") as HTMLSpanElement | null
+
+	if (!progress || !status)
+		return
 
 	if (state === "running") {
 		progress.removeAttribute("value")
@@ -124,4 +127,3 @@ exportButton.addEventListener("click", async () => {
 	await exportState.omni.render(exportState.timeline)
 	setProgress(exportCard, "done")
 })
-

package/s/demo/routines/transcode-test.ts

@@ -38,8 +38,8 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 		const audio = driver.decodeAudio({source})
 
 		const {readable, done} = driver.encode({
-			video,
-			audio,
+			video: video.readable,
+			audio: audio.readable,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transitions-test.ts

@@ -30,8 +30,8 @@ export async function setupTransitionsTest(driver: Driver, source: DecoderSource
 			}
 		})
 
-		await driver.encode({
-			video,
+		driver.encode({
+			video: video.readable,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/waveform-test.ts

@@ -1,14 +1,39 @@
+
 import {Driver} from "../../driver/driver.js"
-import {Waveform} from "../../timeline/parts/waveform.js"
+import {Waveform} from "../../timeline/parts/waveform/waveform.js"
 
 export async function waveformTest(driver: Driver, source: File, root: HTMLElement) {
-	const container = root.querySelector(".waveform-canvas") as HTMLElement
+	const container = root.querySelector(".waveform-canvas") as HTMLDivElement
 	const widthSlider = root.querySelector(".width") as HTMLInputElement
+
 	container.replaceChildren()
-	const waveform = await Waveform.init(driver, source, container)
+	container.style.position = "relative"
+	container.style.height = "96px"
+	container.style.overflow = "hidden"
+
+	const waveform = await Waveform.init(driver, source, {
+		tileHeight: 96,
+		onChange: () => renderTiles()
+	})
 
-	widthSlider.oninput = () => {
+	const renderTiles = () => {
 		const width = +widthSlider.value
-		waveform.width = width
+		const zoom = pixelsPerSecond(width, waveform.duration)
+		waveform.zoom = zoom
+		container.style.width = `${width}px`
+		container.replaceChildren(...[...waveform.getTiles().values()].map(tile => {
+			tile.canvas.style.position = "absolute"
+			tile.canvas.style.top = "0"
+			tile.canvas.style.left = `${tile.startTime * pixelsPerSecond(width, waveform.duration)}px`
+			tile.canvas.style.height = "100%"
+			return tile.canvas
+		}))
 	}
+
+	widthSlider.oninput = renderTiles
+	waveform.range = [0, waveform.duration]
+}
+
+function pixelsPerSecond(width: number, duration: number) {
+	return duration > 0 ? width / duration : 0
 }

package/s/driver/driver.ts

@@ -58,33 +58,54 @@ export class Driver {
 
 	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
+		const {port1, port2} = new MessageChannel()
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
 				const frame = await input.onFrame?.(chunk) ?? chunk
-				// below code is to prevent mem leaks and hardware accelerated decoder stall
 				lastFrame?.close()
 				controller.enqueue(frame)
 				lastFrame = frame
 			}
 		})
-		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			video: videoTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return videoTransform.readable
+		return {
+			readable: videoTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
 	decodeAudio(input: DecoderInput) {
 		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+		const {port1, port2} = new MessageChannel()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			audio: audioTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return audioTransform.readable
+		return {
+			readable: audioTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
 	encode({audio, video, config}: EncoderInput) {

package/s/driver/fns/schematic.ts

@@ -14,6 +14,7 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeAudio(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			audio: WritableStream<AudioData>
 			start?: number
 			end?: number
@@ -21,6 +22,7 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeVideo(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			video: WritableStream<VideoFrame>
 			start?: number
 			end?: number

package/s/driver/fns/work.ts

@@ -20,7 +20,7 @@ export const setupDriverWork = (
 			await shell.host.world()
 		},
 
-		async decodeAudio({source, audio, start, end}) {
+		async decodeAudio({source, audio, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -30,19 +30,29 @@ export const setupDriverWork = (
 			const audioDecodable = await audioTrack?.canDecode()
 			const audioWriter = audio.getWriter()
 
-			if (audioDecodable && audioTrack) {
-				const sink = new AudioSampleSink(audioTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toAudioData()
-					await audioWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await audioWriter.close()
+			if(!audioDecodable || !audioTrack)
+				return
+
+			const sink = new AudioSampleSink(audioTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
 			}
+
+			for await (const sample of samples) {
+				const frame = sample.toAudioData()
+				sample.close()
+				await audioWriter.write(frame)
+				frame.close()
+			}
+
+			await audioWriter.close()
 		},
 
-		async decodeVideo({source, video, start, end}) {
+		async decodeVideo({source, video, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -52,16 +62,26 @@ export const setupDriverWork = (
 			const videoDecodable = await videoTrack?.canDecode()
 			const videoWriter = video.getWriter()
 
-			if (videoDecodable && videoTrack) {
-				const sink = new VideoSampleSink(videoTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toVideoFrame()
-					await videoWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await videoWriter.close()
+			if(!videoDecodable || !videoTrack)
+				return
+
+			const sink = new VideoSampleSink(videoTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
 			}
+
+			for await (const sample of samples) {
+				const frame = sample.toVideoFrame()
+				sample.close()
+				await videoWriter.write(frame)
+				frame.close()
+			}
+
+			await videoWriter.close()
 		},
 
 		async encode({video, audio, config, writable}) {
@@ -78,6 +98,7 @@ export const setupDriverWork = (
 					const sample = new VideoSample(frame)
 					await videoSource.add(sample)
 					sample.close()
+					frame.close()
 				}
 			}
 

package/s/index.html.ts

@@ -85,7 +85,7 @@ export default ssg.page(import.meta.url, async orb => ({
 					</div>
 					<div class="waveform-controls">
 						<label>width</label>
-						<input class="width" type="range" min="100" max="1000000" value="1000" />
+						<input class="width" type="range" min="100" max="1000000" value="1000" step="100" />
 					</div>
 					<div class="waveform-canvas"></div>
 				</article>
@@ -134,4 +134,3 @@ export default ssg.page(import.meta.url, async orb => ({
 		</section>
 	`,
 }))
-

package/s/timeline/parts/waveform/parts/collect.ts

@@ -0,0 +1,72 @@
+
+import {Driver} from "../../../../driver/driver.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export const PEAK_LEVELS = [2048, 1024, 512, 256, 128, 64, 32] as const
+
+export async function collectPeakLevels(driver: Driver, source: DecoderSource) {
+	const duration = (await driver.getAudioDuration(source)) ?? 0
+	const readable = driver.decodeAudio({source}).readable
+	const finestSamplesPerPeak = PEAK_LEVELS[PEAK_LEVELS.length - 1]
+	const finestPeaks: number[] = []
+
+	let currentMax = 0
+	let sampleCount = 0
+	let sampleRate = 0
+
+	for await (const audioData of readable) {
+		sampleRate ||= audioData.sampleRate
+
+		const frames = audioData.numberOfFrames
+		const plane = new Float32Array(frames)
+		audioData.copyTo(plane, {planeIndex: 0})
+
+		for (let i = 0; i < plane.length; i++) {
+			const amplitude = Math.abs(plane[i]!)
+			if (amplitude > currentMax) currentMax = amplitude
+
+			sampleCount++
+			if (sampleCount >= finestSamplesPerPeak) {
+				finestPeaks.push(currentMax)
+				currentMax = 0
+				sampleCount = 0
+			}
+		}
+
+		audioData.close()
+	}
+
+	if (sampleCount > 0) finestPeaks.push(currentMax)
+
+	const base = new Float32Array(finestPeaks)
+	const levels = PEAK_LEVELS.map(samplesPerPeak => {
+		const factor = Math.max(1, Math.round(samplesPerPeak / finestSamplesPerPeak))
+		const peaks = factor === 1 ? base : downsampleMax(base, factor)
+		return {
+			samplesPerPeak,
+			peaks,
+			peaksPerSecond: sampleRate > 0 ? sampleRate / samplesPerPeak : 0,
+		}
+	})
+
+	return {duration, levels}
+}
+
+function downsampleMax(peaks: Float32Array, factor: number) {
+	const downsampled = new Float32Array(Math.ceil(peaks.length / factor))
+
+	for (let i = 0; i < downsampled.length; i++) {
+		let maxPeak = 0
+		const start = i * factor
+		const end = Math.min(start + factor, peaks.length)
+
+		for (let j = start; j < end; j++) {
+			if (peaks[j]! > maxPeak) maxPeak = peaks[j]!
+		}
+
+		downsampled[i] = maxPeak
+	}
+
+	return downsampled
+}
+

package/s/timeline/parts/waveform/parts/render.ts

@@ -0,0 +1,45 @@
+
+export function renderTile(
+	peaks: Float32Array,
+	opts: {
+		width: number
+		height: number
+		color: string
+	}
+) {
+	const dpr = typeof window !== "undefined" ? window.devicePixelRatio || 1 : 1
+	const canvas = document.createElement("canvas")
+	canvas.width = opts.width * dpr
+	canvas.height = opts.height * dpr
+	canvas.style.width = `${opts.width}px`
+	canvas.style.height = `${opts.height}px`
+
+	const ctx = canvas.getContext("2d")
+	if (!ctx) return canvas
+
+	ctx.scale(dpr, dpr)
+	ctx.fillStyle = opts.color
+
+	const centerY = opts.height / 2
+	const columns = Math.max(1, opts.width)
+	const peaksPerPixel = peaks.length / columns
+
+	for (let px = 0; px < columns; px++) {
+		const startIndex = Math.floor(px * peaksPerPixel)
+		const endIndex = Math.max(
+			startIndex + 1,
+			Math.floor((px + 1) * peaksPerPixel)
+		)
+
+		let maxPeak = 0
+		for (let i = startIndex; i < endIndex && i < peaks.length; i++) {
+			if (peaks[i]! > maxPeak) maxPeak = peaks[i]!
+		}
+
+		const barHeight = maxPeak * opts.height
+		ctx.fillRect(px, centerY - barHeight / 2, 1, barHeight)
+	}
+
+	return canvas
+}
+

package/s/timeline/parts/waveform/parts/types.ts

@@ -0,0 +1,24 @@
+export interface WaveformTileData {
+	startTime: number
+	endTime: number
+	peaks: Float32Array
+	canvas: HTMLCanvasElement
+}
+
+export interface WaveformOptions {
+	tileSize?: number
+	zoom?: number
+	tileWidth?: number
+	tileHeight?: number
+	preloadMargin?: number
+	color?: string
+	onChange?: (tiles: WaveformTileData[]) => void
+}
+
+export type WaveformTimeRange = [start: number, end: number]
+
+export type WaveformPeakLevel = {
+	samplesPerPeak: number
+	peaks: Float32Array
+	peaksPerSecond: number
+}

package/s/timeline/parts/waveform/waveform.ts

@@ -0,0 +1,152 @@
+
+import {renderTile} from "./parts/render.js"
+import {Driver} from "../../../driver/driver.js"
+import {collectPeakLevels} from "./parts/collect.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {WaveformOptions, WaveformPeakLevel, WaveformTileData, WaveformTimeRange} from "./parts/types.js"
+
+const MAX_TILE_WIDTH = 4096
+
+export class Waveform {
+	#tiles = new Map<number, WaveformTileData>()
+	#activeRange: WaveformTimeRange = [0, 0]
+
+	#zoom
+	#levels
+	#onChange
+	#updateQueued = false
+
+	readonly color
+	readonly duration
+	readonly tileSize
+	readonly tileHeight
+	readonly preloadMargin
+
+	private constructor(levels: WaveformPeakLevel[], duration: number, options: WaveformOptions) {
+		this.#levels = levels
+		this.duration = duration
+		this.tileSize = options.tileSize ?? 1
+		this.#zoom = options.zoom ?? ((options.tileWidth ?? 256) / this.tileSize)
+		this.tileHeight = options.tileHeight ?? 96
+		this.preloadMargin = options.preloadMargin ?? 2
+		this.color = options.color ?? "rgb(3, 148, 129)"
+		this.#onChange = options.onChange
+	}
+
+	static async init(driver: Driver, source: DecoderSource, options: WaveformOptions = {}) {
+		const {duration, levels} = await collectPeakLevels(driver, source)
+		return new Waveform(levels, duration, options)
+	}
+
+	set zoom(value: number) {
+		const next = Math.max(1, value)
+		if (next === this.#zoom)
+			return
+
+		this.#zoom = next
+		this.#tiles.clear()
+		this.#queueUpdate()
+	}
+
+	get zoom() {
+		return this.#zoom
+	}
+
+	#computeActiveRange([start, end]: WaveformTimeRange, margin = 1): WaveformTimeRange {
+		const visibleSize = end - start
+		return [
+			Math.max(0, start - visibleSize * margin),
+			Math.min(this.duration, end + visibleSize * margin),
+		]
+	}
+
+	set range(visibleRange: WaveformTimeRange) {
+		const [visibleStart, visibleEnd] = visibleRange
+		const visibleSize = visibleEnd - visibleStart
+		const [activeStart, activeEnd] = this.#activeRange
+
+		const leftTrigger = activeStart + visibleSize
+		const rightTrigger = activeEnd - visibleSize
+
+		if (visibleStart >= leftTrigger && visibleEnd <= rightTrigger) return
+
+		this.#activeRange = this.#computeActiveRange(visibleRange, this.preloadMargin)
+		this.#queueUpdate()
+	}
+
+	#queueUpdate() {
+		if (this.#updateQueued) return
+		this.#updateQueued = true
+
+		queueMicrotask(() => {
+			this.#updateQueued = false
+			this.#generateTiles()
+		})
+	}
+
+	#generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededStarts = new Set<number>()
+		const level = this.#levelForZoom()
+
+		const firstStart = Math.max(0, Math.floor(rangeStart / this.tileSize) * this.tileSize)
+		const lastStart = Math.min(this.duration, rangeEnd)
+
+		for (let startTime = firstStart; startTime <= lastStart; startTime += this.tileSize) {
+			neededStarts.add(startTime)
+		}
+
+		for (const startTime of neededStarts) {
+			if (!this.#tiles.has(startTime)) {
+				const endTime = Math.min(startTime + this.tileSize, this.duration)
+				this.#tiles.set(startTime, this.#buildTileData(startTime, endTime, level))
+			}
+		}
+
+		for (const startTime of this.#tiles.keys()) {
+			if (!neededStarts.has(startTime)) this.#tiles.delete(startTime)
+		}
+
+		this.#emit()
+	}
+
+	#buildTileData(startTime: number, endTime: number, level: WaveformPeakLevel): WaveformTileData {
+		const peaks = this.#slicePeaks(level, startTime, endTime)
+		return {
+			startTime,
+			endTime,
+			peaks,
+			canvas: renderTile(peaks, {
+				width: this.#tilePixelWidth(startTime, endTime),
+				height: this.tileHeight,
+				color: this.color,
+			}),
+		}
+	}
+
+	#levelForZoom() {
+		return this.#levels.find(level => level.peaksPerSecond >= this.#zoom)
+			?? this.#levels[this.#levels.length - 1]!
+	}
+
+	#slicePeaks(level: WaveformPeakLevel, startTime: number, endTime: number) {
+		if (!level.peaksPerSecond) return new Float32Array()
+		const from = Math.max(0, Math.floor(startTime * level.peaksPerSecond))
+		const to = Math.max(from + 1, Math.min(level.peaks.length, Math.ceil(endTime * level.peaksPerSecond)))
+		return level.peaks.slice(from, to)
+	}
+
+	#tilePixelWidth(startTime: number, endTime: number) {
+		return Math.min(MAX_TILE_WIDTH, Math.max(1, Math.ceil((endTime - startTime) * this.#zoom)))
+	}
+
+	#emit() {
+		if (!this.#onChange) return
+		this.#onChange([...this.#tiles.values()].sort((a, b) => a.startTime - b.startTime))
+	}
+
+	getTiles() {
+		return this.#tiles
+	}
+}
+

package/s/timeline/parts/waveform.ts

@@ -1,61 +0,0 @@
-import WaveSurfer from "wavesurfer.js"
-
-import {Driver} from "../../driver/driver.js"
-import {DecoderSource} from "../../driver/fns/schematic.js"
-
-export class Waveform {
-	wavesurfer: WaveSurfer
-
-	constructor(peaks: number[], container: HTMLElement, duration: number) {
-		this.wavesurfer = WaveSurfer.create({
-  		container,
-  		waveColor: 'rgb(200, 0, 200)',
-  		progressColor: 'rgb(100, 0, 100)',
-  		barWidth: 10,
-  		barRadius: 10,
-  		barGap: 2,
-  		peaks: [peaks],
-  		duration
-		})
-	}
-
-	static async init(driver: Driver, source: DecoderSource, container: HTMLElement) {
-		const reader = driver.decodeAudio({source}).getReader()
-
-		const peaks: number[] = []
-		let buffer: number[] = []
-		const samplesPerPeak = 1024
-		const duration = await driver.getAudioDuration(source)
-
-		while (true) {
-			const {done, value: audioData} = await reader.read()
-			if (done) break
-
-			const frames = audioData.numberOfFrames
-			const plane = new Float32Array(frames)
-			audioData.copyTo(plane, {planeIndex: 0}) // Use left channel only
-
-			for (let i = 0; i < plane.length; i++) {
-				buffer.push(plane[i])
-				if (buffer.length >= samplesPerPeak) {
-					const chunk = buffer.splice(0, samplesPerPeak)
-					const min = Math.min(...chunk)
-					const max = Math.max(...chunk)
-					peaks.push(min, max)
-				}
-			}
-
-			audioData.close()
-		}
-
-		return new Waveform(peaks, container, duration ?? 0)
-	}
-
-	// set zoom(value: number) {
-	// 	this.wavesurfer.zoom(value)
-	// }
-
-	set width(value: number) {
-		this.wavesurfer.setOptions({width: value})
-	}
-}