@omnimedia/omnitool 1.1.0-7 → 1.1.0-71

package/s/timeline/parts/resource-pool.ts

@@ -1,27 +1,30 @@
 
-import {MapG} from "@e280/stz"
+import {GMap} from "@e280/stz"
 import {Hash} from "./basics.js"
 import {Media} from "./media.js"
 import {Resource} from "./resource.js"
 import {Datafile} from "../utils/datafile.js"
 
 export class ResourcePool {
-	#map = new MapG<Hash, Resource.Any>
+	#map = new GMap<Hash, Resource.Any>
 
 	/** store a media file (avoids duplicates via hash) */
 	async store(datafile: Datafile) {
 		const media = await Media.analyze(datafile)
 		const {hash} = media.datafile.checksum
-		const {filename, bytes} = media.datafile
+		const {filename, bytes, url, blob} = media.datafile
 
 		if (this.#map.has(hash)) {
 			const alreadyExists = this.#map.require(hash)
 			alreadyExists.filename = filename
 		}
 		else
-			this.#map.set(hash, {kind: "media", filename, bytes})
+			this.#map.set(hash, {kind: "media", filename, bytes, url, blob, duration: media.duration})
 
 		return media
 	}
-}
 
+	require(hash: Hash) {
+		return this.#map.require(hash)
+	}
+}

package/s/timeline/parts/resource.ts

@@ -4,6 +4,9 @@ export namespace Resource {
 		kind: "media"
 		filename: string
 		bytes: Uint8Array
+		blob: Blob
+		url: string
+		duration: number
 	}
 
 	export type Any = Media

package/s/timeline/parts/waveform/parts/collect.ts

@@ -0,0 +1,72 @@
+
+import {Driver} from "../../../../driver/driver.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export const PEAK_LEVELS = [2048, 1024, 512, 256, 128, 64, 32] as const
+
+export async function collectPeakLevels(driver: Driver, source: DecoderSource) {
+	const duration = (await driver.getAudioDuration(source)) ?? 0
+	const readable = driver.decodeAudio({source}).readable
+	const finestSamplesPerPeak = PEAK_LEVELS[PEAK_LEVELS.length - 1]
+	const finestPeaks: number[] = []
+
+	let currentMax = 0
+	let sampleCount = 0
+	let sampleRate = 0
+
+	for await (const audioData of readable) {
+		sampleRate ||= audioData.sampleRate
+
+		const frames = audioData.numberOfFrames
+		const plane = new Float32Array(frames)
+		audioData.copyTo(plane, {planeIndex: 0})
+
+		for (let i = 0; i < plane.length; i++) {
+			const amplitude = Math.abs(plane[i]!)
+			if (amplitude > currentMax) currentMax = amplitude
+
+			sampleCount++
+			if (sampleCount >= finestSamplesPerPeak) {
+				finestPeaks.push(currentMax)
+				currentMax = 0
+				sampleCount = 0
+			}
+		}
+
+		audioData.close()
+	}
+
+	if (sampleCount > 0) finestPeaks.push(currentMax)
+
+	const base = new Float32Array(finestPeaks)
+	const levels = PEAK_LEVELS.map(samplesPerPeak => {
+		const factor = Math.max(1, Math.round(samplesPerPeak / finestSamplesPerPeak))
+		const peaks = factor === 1 ? base : downsampleMax(base, factor)
+		return {
+			samplesPerPeak,
+			peaks,
+			peaksPerSecond: sampleRate > 0 ? sampleRate / samplesPerPeak : 0,
+		}
+	})
+
+	return {duration, levels}
+}
+
+function downsampleMax(peaks: Float32Array, factor: number) {
+	const downsampled = new Float32Array(Math.ceil(peaks.length / factor))
+
+	for (let i = 0; i < downsampled.length; i++) {
+		let maxPeak = 0
+		const start = i * factor
+		const end = Math.min(start + factor, peaks.length)
+
+		for (let j = start; j < end; j++) {
+			if (peaks[j]! > maxPeak) maxPeak = peaks[j]!
+		}
+
+		downsampled[i] = maxPeak
+	}
+
+	return downsampled
+}
+

package/s/timeline/parts/waveform/parts/render.ts

@@ -0,0 +1,45 @@
+
+export function renderTile(
+	peaks: Float32Array,
+	opts: {
+		width: number
+		height: number
+		color: string
+	}
+) {
+	const dpr = typeof window !== "undefined" ? window.devicePixelRatio || 1 : 1
+	const canvas = document.createElement("canvas")
+	canvas.width = opts.width * dpr
+	canvas.height = opts.height * dpr
+	canvas.style.width = `${opts.width}px`
+	canvas.style.height = `${opts.height}px`
+
+	const ctx = canvas.getContext("2d")
+	if (!ctx) return canvas
+
+	ctx.scale(dpr, dpr)
+	ctx.fillStyle = opts.color
+
+	const centerY = opts.height / 2
+	const columns = Math.max(1, opts.width)
+	const peaksPerPixel = peaks.length / columns
+
+	for (let px = 0; px < columns; px++) {
+		const startIndex = Math.floor(px * peaksPerPixel)
+		const endIndex = Math.max(
+			startIndex + 1,
+			Math.floor((px + 1) * peaksPerPixel)
+		)
+
+		let maxPeak = 0
+		for (let i = startIndex; i < endIndex && i < peaks.length; i++) {
+			if (peaks[i]! > maxPeak) maxPeak = peaks[i]!
+		}
+
+		const barHeight = maxPeak * opts.height
+		ctx.fillRect(px, centerY - barHeight / 2, 1, barHeight)
+	}
+
+	return canvas
+}
+

package/s/timeline/parts/waveform/parts/types.ts

@@ -0,0 +1,24 @@
+export interface WaveformTileData {
+	startTime: number
+	endTime: number
+	peaks: Float32Array
+	canvas: HTMLCanvasElement
+}
+
+export interface WaveformOptions {
+	tileSize?: number
+	zoom?: number
+	tileWidth?: number
+	tileHeight?: number
+	preloadMargin?: number
+	color?: string
+	onChange?: (tiles: WaveformTileData[]) => void
+}
+
+export type WaveformTimeRange = [start: number, end: number]
+
+export type WaveformPeakLevel = {
+	samplesPerPeak: number
+	peaks: Float32Array
+	peaksPerSecond: number
+}

package/s/timeline/parts/waveform/waveform.ts

@@ -0,0 +1,161 @@
+
+import {renderTile} from "./parts/render.js"
+import {Driver} from "../../../driver/driver.js"
+import {collectPeakLevels} from "./parts/collect.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {WaveformOptions, WaveformPeakLevel, WaveformTileData, WaveformTimeRange} from "./parts/types.js"
+
+const MAX_TILE_WIDTH = 4096
+
+export class Waveform {
+	#tiles = new Map<number, WaveformTileData>()
+	#activeRange: WaveformTimeRange = [0, 0]
+	#visibleRange: WaveformTimeRange = [0, 0]
+
+	#zoom
+	#levels
+	#onChange
+	#updateQueued = false
+
+	readonly color
+	readonly duration
+	readonly tileSize
+	readonly tileHeight
+	readonly preloadMargin
+
+	private constructor(levels: WaveformPeakLevel[], duration: number, options: WaveformOptions) {
+		this.#levels = levels
+		this.duration = duration
+		this.tileSize = options.tileSize ?? 1
+		this.#zoom = options.zoom ?? ((options.tileWidth ?? 256) / this.tileSize)
+		this.tileHeight = options.tileHeight ?? 96
+		this.preloadMargin = options.preloadMargin ?? 2
+		this.color = options.color ?? "rgb(3, 148, 129)"
+		this.#onChange = options.onChange
+	}
+
+	static async init(driver: Driver, source: DecoderSource, options: WaveformOptions = {}) {
+		const {duration, levels} = await collectPeakLevels(driver, source)
+		return new Waveform(levels, duration, options)
+	}
+
+	/** Waveform render density in pixels per second. */
+	set zoom(pixelsPerSecond: number) {
+		const next = Math.max(1, pixelsPerSecond)
+		if (next === this.#zoom)
+			return
+
+		this.#zoom = next
+		this.#tiles.clear()
+		this.#queueUpdate()
+	}
+
+	get zoom() {
+		return this.#zoom
+	}
+
+	get range() {
+		return this.#visibleRange
+	}
+
+	#computeActiveRange([start, end]: WaveformTimeRange, margin = 1): WaveformTimeRange {
+		const visibleSize = end - start
+		return [
+			Math.max(0, start - visibleSize * margin),
+			Math.min(this.duration, end + visibleSize * margin),
+		]
+	}
+
+	set range(visibleRange: WaveformTimeRange) {
+		this.#visibleRange = visibleRange
+
+		const [visibleStart, visibleEnd] = visibleRange
+		const visibleSize = visibleEnd - visibleStart
+		const [activeStart, activeEnd] = this.#activeRange
+
+		const leftBuffered = activeStart > 0
+		const rightBuffered = activeEnd < this.duration
+		const leftSettled = !leftBuffered || visibleStart >= activeStart + visibleSize
+		const rightSettled = !rightBuffered || visibleEnd <= activeEnd - visibleSize
+
+		if (leftSettled && rightSettled) return
+
+		this.#activeRange = this.#computeActiveRange(visibleRange, this.preloadMargin)
+		this.#queueUpdate()
+	}
+
+	#queueUpdate() {
+		if (this.#updateQueued) return
+		this.#updateQueued = true
+
+		queueMicrotask(() => {
+			this.#updateQueued = false
+			this.#generateTiles()
+		})
+	}
+
+	#generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededStarts = new Set<number>()
+		const level = this.#levelForZoom()
+
+		const firstStart = Math.max(0, Math.floor(rangeStart / this.tileSize) * this.tileSize)
+		const lastStart = Math.min(this.duration, rangeEnd)
+
+		for (let startTime = firstStart; startTime <= lastStart; startTime += this.tileSize) {
+			neededStarts.add(startTime)
+		}
+
+		for (const startTime of neededStarts) {
+			if (!this.#tiles.has(startTime)) {
+				const endTime = Math.min(startTime + this.tileSize, this.duration)
+				this.#tiles.set(startTime, this.#buildTileData(startTime, endTime, level))
+			}
+		}
+
+		for (const startTime of this.#tiles.keys()) {
+			if (!neededStarts.has(startTime)) this.#tiles.delete(startTime)
+		}
+
+		this.#emit()
+	}
+
+	#buildTileData(startTime: number, endTime: number, level: WaveformPeakLevel): WaveformTileData {
+		const peaks = this.#slicePeaks(level, startTime, endTime)
+		return {
+			startTime,
+			endTime,
+			peaks,
+			canvas: renderTile(peaks, {
+				width: this.#tilePixelWidth(startTime, endTime),
+				height: this.tileHeight,
+				color: this.color,
+			}),
+		}
+	}
+
+	#levelForZoom() {
+		return this.#levels.find(level => level.peaksPerSecond >= this.#zoom)
+			?? this.#levels[this.#levels.length - 1]!
+	}
+
+	#slicePeaks(level: WaveformPeakLevel, startTime: number, endTime: number) {
+		if (!level.peaksPerSecond) return new Float32Array()
+		const from = Math.max(0, Math.floor(startTime * level.peaksPerSecond))
+		const to = Math.max(from + 1, Math.min(level.peaks.length, Math.ceil(endTime * level.peaksPerSecond)))
+		return level.peaks.slice(from, to)
+	}
+
+	#tilePixelWidth(startTime: number, endTime: number) {
+		return Math.min(MAX_TILE_WIDTH, Math.max(1, Math.ceil((endTime - startTime) * this.#zoom)))
+	}
+
+	#emit() {
+		if (!this.#onChange) return
+		this.#onChange([...this.#tiles.values()].sort((a, b) => a.startTime - b.startTime))
+	}
+
+	getTiles() {
+		return this.#tiles
+	}
+}

package/s/timeline/renderers/export/parts/audio-gain.ts

@@ -0,0 +1,17 @@
+// this is mutating fn
+export const applyGainToPlanar = (
+	planes: Float32Array[],
+	gain: number
+) => {
+	if (gain === 1)
+		return
+	// planes
+
+	for (const plane of planes) {
+		for (let i = 0; i < plane.length; i++) {
+			plane[i] *= gain
+		}
+	}
+
+	// return planes
+}

package/s/timeline/renderers/export/parts/audio-mix.ts

@@ -0,0 +1,133 @@
+
+export type AudioPlanesInput = {
+	planes: Float32Array[]
+	sampleRate: number
+	timestamp: number
+}
+
+export type MixedChunk = {
+	planar: Float32Array
+	sampleRate: number
+	channels: number
+	frames: number
+	startFrame: number
+}
+
+type ActiveSample = {
+	startFrame: number
+	endFrame: number
+	data: Float32Array[]
+}
+
+export type AudioMixOptions = {
+	chunkFrames?: number
+	clamp?: boolean
+}
+
+export class AudioMix {
+	readonly #chunkFrames: number
+	readonly #clamp: boolean
+
+	constructor(options: AudioMixOptions = {}) {
+		this.#chunkFrames = options.chunkFrames ?? 1024
+		this.#clamp = options.clamp ?? true
+	}
+
+	async *mix(samples: AsyncIterable<AudioPlanesInput>): AsyncGenerator<MixedChunk> {
+		const chunkFrames = this.#chunkFrames
+		let sampleRate: number | null = null
+		let channels: number | null = null
+		const active: ActiveSample[] = []
+		let nextFrame = 0
+		let maxEnd = 0
+
+		for await (const input of samples) {
+			if (channels === null) {
+				channels = input.planes.length
+				sampleRate = input.sampleRate
+			} else {
+				if (input.planes.length !== channels) throw new Error(`Channel count changed`)
+				if (input.sampleRate !== sampleRate) throw new Error(`Sample rate changed`)
+			}
+
+			const inputStart = Math.round(input.timestamp * sampleRate)
+			const frames = input.planes[0]?.length ?? 0
+
+			while (nextFrame + chunkFrames <= inputStart) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+
+			active.push({
+				startFrame: inputStart,
+				endFrame: inputStart + frames,
+				data: input.planes
+			})
+
+			maxEnd = Math.max(maxEnd, inputStart + frames)
+		}
+
+		if (channels !== null && sampleRate !== null) {
+			while (nextFrame < maxEnd) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+		}
+	}
+
+	#processChunk(
+		active: ActiveSample[],
+		currentStartFrame: number,
+		channels: number,
+		sampleRate: number
+	): MixedChunk {
+		const chunkFrames = this.#chunkFrames
+		const outputBuffer = new Float32Array(channels * chunkFrames)
+		const chunkEnd = currentStartFrame + chunkFrames
+
+		for (let ch = 0; ch < channels; ch++) {
+			const channelOffset = ch * chunkFrames
+			const outChannelView = outputBuffer.subarray(channelOffset, channelOffset + chunkFrames)
+
+			for (const sample of active) {
+				const data = sample.data[ch]
+				if (!data) continue
+
+				const start = Math.max(currentStartFrame, sample.startFrame)
+				const end = Math.min(chunkEnd, sample.endFrame)
+
+				if (start >= end) continue
+
+				const dstIdx = start - currentStartFrame
+				const srcIdx = start - sample.startFrame
+				const len = end - start
+
+				for (let i = 0; i < len; i++) {
+					outChannelView[dstIdx + i] += data[srcIdx + i]
+				}
+			}
+
+			if (this.#clamp) {
+				for (let i = 0; i < chunkFrames; i++) {
+					const v = outChannelView[i]
+					outChannelView[i] = v < -1.0 ? -1.0 : (v > 1.0 ? 1.0 : v)
+				}
+			}
+		}
+
+		for (let i = active.length - 1; i >= 0; i--) {
+			if (active[i].endFrame <= chunkEnd) {
+				active.splice(i, 1)
+			}
+		}
+
+		return {
+			planar: outputBuffer,
+			sampleRate,
+			channels,
+			frames: chunkFrames,
+			startFrame: currentStartFrame
+		}
+	}
+}
+

package/s/timeline/renderers/export/parts/cursor.ts

@@ -0,0 +1,129 @@
+
+import {ms, Ms} from "../../../../units/ms.js"
+import {Driver} from "../../../../driver/driver.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createVisualSampler} from "../../parts/samplers/visual/sampler.js"
+
+/**
+ * forward-only frame cursor optimized for export purposes.
+ * it uses mediabunny internally so the support for non-clients
+ * should be done from mediabunny custom decoder/encoder
+ */
+
+export class CursorVisualSampler {
+	#lastTimecode = -Infinity
+	#videoCursors = new Map<number, VideoFrameCursor>()
+	#sampler
+
+	constructor(
+		private driver: Driver,
+		private resolveMedia: (hash: string) => DecoderSource,
+		private timeline: TimelineFile
+	) {
+		this.#sampler = createVisualSampler(this.resolveMedia, (item, time) => {
+			const targetUs = toUs(time)
+			let cursor = this.#videoCursors.get(item.id)
+
+			if (!cursor) {
+				const source = this.resolveMedia(item.mediaHash)
+				const endUs = toUs(ms(item.start + item.duration))
+				cursor = this.#createVideoCursor(source, targetUs, endUs)
+				this.#videoCursors.set(item.id, cursor)
+			}
+
+			return cursor.next(targetUs)
+		})
+	}
+
+	next(timecode: Ms) {
+		if (timecode < this.#lastTimecode)
+			throw new Error(`Forward-only cursor regression: ${timecode}ms < ${this.#lastTimecode}ms`)
+
+		this.#lastTimecode = timecode
+		return this.#sampler.sample(this.timeline, timecode)
+	}
+
+	async cancel() {
+		await Promise.all([...this.#videoCursors.values()].map(c => c.cancel()))
+		this.#videoCursors.clear()
+	}
+
+	#createVideoCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
+		const video = this.driver.decodeVideo({source, start: startUs / 1_000_000, end: endUs / 1_000_000})
+		const reader = video.readable.getReader()
+
+		let current: VideoFrame | null = null
+		let nextPromise: Promise<VideoFrame | null> | null = null
+		let ended = false
+
+		const readNext = async () => {
+			if (ended) return null
+			const {done, value} = await reader.read()
+			if (done) return (ended = true, null)
+
+			const frame = new VideoFrame(value)
+			value.close()
+			return frame
+		}
+
+		return {
+			async next(targetUs: number): Promise<VideoFrame | undefined> {
+				current ??= await readNext()
+				if (!current) return undefined
+
+				while (true) {
+					nextPromise ??= readNext()
+					const nextFrame = await nextPromise
+
+					if (!nextFrame) return new VideoFrame(current)
+
+					const currentUs = current.timestamp ?? -Infinity
+					const nextUs = nextFrame.timestamp ?? currentUs
+
+					if (nextUs < targetUs) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
+					}
+
+					const useNext = Math.abs(nextUs - targetUs) < Math.abs(currentUs - targetUs)
+
+					if (useNext) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
+					}
+
+					return new VideoFrame(current)
+				}
+			},
+
+			async cancel() {
+				const pending = nextPromise
+				nextPromise = null
+				ended = true
+
+				const buffered = await pending?.catch(() => null)
+				buffered?.close()
+
+				current?.close()
+				current = null
+
+				video.cancel()
+			}
+		}
+	}
+}
+
+const toUs = (ms: Ms) => Math.round(ms * 1_000)
+
+type StreamCursor<T> = {
+	next(target: number): Promise<T | undefined>
+	cancel(): Promise<void>
+}
+
+type VideoFrameCursor = StreamCursor<VideoFrame>
+

package/s/timeline/renderers/export/parts/produce-audio.ts

@@ -0,0 +1,64 @@
+
+import {AudioMix} from "./audio-mix.js"
+import {ms} from "../../../../units/ms.js"
+import {resampleToPlanar} from "./resamplers.js"
+import {applyGainToPlanar} from "./audio-gain.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createAudioSampler} from "../../parts/samplers/audio/sampler.js"
+
+export function produceAudio({
+	timeline,
+	resolveMedia
+}: {
+	timeline: TimelineFile,
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+	const mixer = new AudioMix()
+	const audio = streamAudio(timeline, resolveMedia)
+	const stream = new TransformStream<AudioData, AudioData>()
+	const writer = stream.writable.getWriter()
+
+	async function produce() {
+		for await (const chunk of mixer.mix(audio)) {
+			const data = new AudioData({
+				format: 'f32-planar',
+				sampleRate: chunk.sampleRate,
+				numberOfFrames: chunk.frames,
+				numberOfChannels: chunk.channels,
+				timestamp: Math.round(
+					(chunk.startFrame / chunk.sampleRate) * 1_000_000
+				),
+				data: new Float32Array(chunk.planar)
+			})
+
+			await writer.write(data)
+		}
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+
+async function *streamAudio(timeline: TimelineFile, resolveMedia: (hash: string) => DecoderSource) {
+	const audioSampler = createAudioSampler(resolveMedia)
+
+	for await (const {sample, timestamp, gain}
+		of audioSampler.sampleAudio(timeline, ms(0))) {
+
+		const {data} = resampleToPlanar(sample, 48000)
+		applyGainToPlanar(data, gain)
+
+		yield {
+			planes: data,
+			sampleRate: 48000,
+			timestamp
+		}
+
+		sample.close()
+	}
+}
+