@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/timeline/renderers/export/parts/audio-mix.ts

@@ -0,0 +1,133 @@
+
+export type AudioPlanesInput = {
+	planes: Float32Array[]
+	sampleRate: number
+	timestamp: number
+}
+
+export type MixedChunk = {
+	planar: Float32Array
+	sampleRate: number
+	channels: number
+	frames: number
+	startFrame: number
+}
+
+type ActiveSample = {
+	startFrame: number
+	endFrame: number
+	data: Float32Array[]
+}
+
+export type AudioMixOptions = {
+	chunkFrames?: number
+	clamp?: boolean
+}
+
+export class AudioMix {
+	readonly #chunkFrames: number
+	readonly #clamp: boolean
+
+	constructor(options: AudioMixOptions = {}) {
+		this.#chunkFrames = options.chunkFrames ?? 1024
+		this.#clamp = options.clamp ?? true
+	}
+
+	async *mix(samples: AsyncIterable<AudioPlanesInput>): AsyncGenerator<MixedChunk> {
+		const chunkFrames = this.#chunkFrames
+		let sampleRate: number | null = null
+		let channels: number | null = null
+		const active: ActiveSample[] = []
+		let nextFrame = 0
+		let maxEnd = 0
+
+		for await (const input of samples) {
+			if (channels === null) {
+				channels = input.planes.length
+				sampleRate = input.sampleRate
+			} else {
+				if (input.planes.length !== channels) throw new Error(`Channel count changed`)
+				if (input.sampleRate !== sampleRate) throw new Error(`Sample rate changed`)
+			}
+
+			const inputStart = Math.round(input.timestamp * sampleRate)
+			const frames = input.planes[0]?.length ?? 0
+
+			while (nextFrame + chunkFrames <= inputStart) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+
+			active.push({
+				startFrame: inputStart,
+				endFrame: inputStart + frames,
+				data: input.planes
+			})
+
+			maxEnd = Math.max(maxEnd, inputStart + frames)
+		}
+
+		if (channels !== null && sampleRate !== null) {
+			while (nextFrame < maxEnd) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+		}
+	}
+
+	#processChunk(
+		active: ActiveSample[],
+		currentStartFrame: number,
+		channels: number,
+		sampleRate: number
+	): MixedChunk {
+		const chunkFrames = this.#chunkFrames
+		const outputBuffer = new Float32Array(channels * chunkFrames)
+		const chunkEnd = currentStartFrame + chunkFrames
+
+		for (let ch = 0; ch < channels; ch++) {
+			const channelOffset = ch * chunkFrames
+			const outChannelView = outputBuffer.subarray(channelOffset, channelOffset + chunkFrames)
+
+			for (const sample of active) {
+				const data = sample.data[ch]
+				if (!data) continue
+
+				const start = Math.max(currentStartFrame, sample.startFrame)
+				const end = Math.min(chunkEnd, sample.endFrame)
+
+				if (start >= end) continue
+
+				const dstIdx = start - currentStartFrame
+				const srcIdx = start - sample.startFrame
+				const len = end - start
+
+				for (let i = 0; i < len; i++) {
+					outChannelView[dstIdx + i] += data[srcIdx + i]
+				}
+			}
+
+			if (this.#clamp) {
+				for (let i = 0; i < chunkFrames; i++) {
+					const v = outChannelView[i]
+					outChannelView[i] = v < -1.0 ? -1.0 : (v > 1.0 ? 1.0 : v)
+				}
+			}
+		}
+
+		for (let i = active.length - 1; i >= 0; i--) {
+			if (active[i].endFrame <= chunkEnd) {
+				active.splice(i, 1)
+			}
+		}
+
+		return {
+			planar: outputBuffer,
+			sampleRate,
+			channels,
+			frames: chunkFrames,
+			startFrame: currentStartFrame
+		}
+	}
+}
+

package/s/timeline/renderers/export/parts/cursor.ts

@@ -0,0 +1,291 @@
+
+import {ALL_FORMATS, Input, VideoSampleSink} from "mediabunny"
+
+import {ms, Ms} from "../../../../units/ms.js"
+import {Driver} from "../../../../driver/driver.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../driver/utils/load-decoder-source.js"
+import {createVisualSampler} from "../../parts/samplers/visual/sampler.js"
+
+type StreamCursor<T> = {
+	next(target: number): Promise<T | undefined>
+	cancel(): Promise<void>
+}
+
+type VideoFrameCursor = StreamCursor<VideoFrame>
+
+abstract class BaseVisualSampler {
+	readonly #videoCursors = new Map<number, VideoFrameCursor>()
+	readonly #sampler
+
+	constructor(
+		protected driver: Driver,
+		protected resolveMedia: (hash: string) => DecoderSource,
+		protected timeline: TimelineFile
+	) {
+		this.#sampler = createVisualSampler(this.resolveMedia, (item, time) => {
+			const targetUs = toUs(ms(item.start + time))
+			let cursor = this.#videoCursors.get(item.id)
+
+			if (!cursor) {
+				const source = this.resolveMedia(item.mediaHash)
+				const endUs = toUs(ms(item.start + item.duration))
+				cursor = this.createCursor(source, targetUs, endUs)
+				this.#videoCursors.set(item.id, cursor)
+			}
+
+			return cursor.next(targetUs)
+		})
+	}
+
+	protected abstract createCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor
+
+	protected sample(timecode: Ms) {
+		return this.#sampler.sample(this.timeline, timecode)
+	}
+
+	async cancel() {
+		await Promise.all([...this.#videoCursors.values()].map(c => c.cancel()))
+		this.#videoCursors.clear()
+	}
+}
+
+/**
+ * forward-only frame cursor optimized for export purposes.
+ * it uses mediabunny internally so the support for non-clients
+ * should be done from mediabunny custom decoder/encoder
+ */
+
+export class CursorVisualSampler extends BaseVisualSampler {
+	#lastTimecode = -Infinity
+
+	next(timecode: Ms) {
+		if (timecode < this.#lastTimecode)
+			throw new Error(`Forward-only cursor regression: ${timecode}ms < ${this.#lastTimecode}ms`)
+
+		this.#lastTimecode = timecode
+		return this.sample(timecode)
+	}
+
+	protected createCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
+		const video = this.driver.decodeVideo({source, start: startUs / 1_000_000, end: endUs / 1_000_000})
+		const reader = video.readable.getReader()
+
+		let current: VideoFrame | null = null
+		let nextPromise: Promise<VideoFrame | null> | null = null
+		let ended = false
+
+		const readNext = async () => {
+			if (ended) return null
+			const {done, value} = await reader.read()
+			if (done) return (ended = true, null)
+
+			const frame = new VideoFrame(value)
+			value.close()
+			return frame
+		}
+
+		return {
+			async next(targetUs: number): Promise<VideoFrame | undefined> {
+				current ??= await readNext()
+				if (!current) return undefined
+
+				while (true) {
+					nextPromise ??= readNext()
+					const nextFrame = await nextPromise
+
+					if (!nextFrame) return new VideoFrame(current)
+
+					const currentUs = current.timestamp ?? -Infinity
+					const nextUs = nextFrame.timestamp ?? currentUs
+
+					if (nextUs < targetUs) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
+					}
+
+					const useNext = Math.abs(nextUs - targetUs) < Math.abs(currentUs - targetUs)
+
+					if (useNext) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
+					}
+
+					return new VideoFrame(current)
+				}
+			},
+
+			async cancel() {
+				const pending = nextPromise
+				nextPromise = null
+				ended = true
+
+				const buffered = await pending?.catch(() => null)
+				buffered?.close()
+
+				current?.close()
+				current = null
+
+				video.cancel()
+			}
+		}
+	}
+}
+
+export class ReverseCursorVisualSampler extends BaseVisualSampler {
+	#lastTimecode = Infinity
+
+	next(timecode: Ms) {
+		if (timecode > this.#lastTimecode)
+			throw new Error(`Reverse-only cursor regression: ${timecode}ms > ${this.#lastTimecode}ms`)
+
+		this.#lastTimecode = timecode
+		return this.sample(timecode)
+	}
+
+	protected createCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
+		const windowUs = 1_000_000
+		const prefetchThreshold = windowUs * 0.5
+
+		let frames: VideoFrame[] = []
+		let windowStart = Infinity
+		let windowEnd = -Infinity
+		let input: Input | null = null
+		let sink: VideoSampleSink | null = null
+		let prefetchPromise: Promise<{frames: VideoFrame[], windowStart: number, windowEnd: number}> | null = null
+		let activeFetches = 0
+		let idle: Promise<void> = Promise.resolve()
+		let resolveIdle: (() => void) | null = null
+		let canceled = false
+
+		const clear = () => {
+			for (const frame of frames)
+				frame.close()
+			frames = []
+		}
+
+		const startFetch = () => {
+			if (activeFetches++ === 0)
+				idle = new Promise<void>(resolve => resolveIdle = resolve)
+		}
+
+		const endFetch = () => {
+			if (--activeFetches === 0) {
+				resolveIdle?.()
+				resolveIdle = null
+			}
+		}
+
+		const getSink = async () => {
+			if (sink) return sink
+
+			input = new Input({
+				source: await loadDecoderSource(source),
+				formats: ALL_FORMATS,
+			})
+
+			const track = await input.getPrimaryVideoTrack()
+			sink = track && await track.canDecode()
+				? new VideoSampleSink(track)
+				: null
+
+			return sink
+		}
+
+		const fetchFrames = async (targetUs: number) => {
+			startFetch()
+			const wEnd = Math.min(endUs, targetUs + 1)
+			const wStart = Math.max(startUs, wEnd - windowUs)
+			const newFrames: VideoFrame[] = []
+
+			const videoSink = await getSink()
+			if (videoSink) {
+				for await (const sample of videoSink.samples(wStart / 1_000_000, wEnd / 1_000_000)) {
+					newFrames.push(sample.toVideoFrame())
+					sample.close()
+				}
+			}
+
+			endFetch()
+			return {frames: newFrames, windowStart: wStart, windowEnd: wEnd}
+		}
+
+		const loadWindow = async (targetUs: number) => {
+			clear()
+			const result = await fetchFrames(targetUs)
+			frames = result.frames
+			windowStart = result.windowStart
+			windowEnd = result.windowEnd
+		}
+
+		return {
+			async next(targetUs: number): Promise<VideoFrame | undefined> {
+				if (canceled)
+					return undefined
+
+				if (targetUs < windowStart || targetUs > windowEnd) {
+					if (prefetchPromise) {
+						const prefetched = await prefetchPromise
+						prefetchPromise = null
+
+						if (canceled) {
+							for (const f of prefetched.frames) f.close()
+							return undefined
+						}
+
+						if (targetUs >= prefetched.windowStart && targetUs <= prefetched.windowEnd) {
+							clear()
+							frames = prefetched.frames
+							windowStart = prefetched.windowStart
+							windowEnd = prefetched.windowEnd
+						} else {
+							for (const f of prefetched.frames) f.close()
+							await loadWindow(targetUs)
+						}
+					} else {
+						await loadWindow(targetUs)
+					}
+				}
+
+				if (!prefetchPromise && targetUs < windowStart + prefetchThreshold && windowStart > startUs)
+					prefetchPromise = fetchFrames(windowStart - 1)
+
+				let best: VideoFrame | undefined
+				let bestDistance = Infinity
+
+				for (const frame of frames) {
+					const distance = Math.abs((frame.timestamp ?? targetUs) - targetUs)
+					if (distance < bestDistance) {
+						best = frame
+						bestDistance = distance
+					}
+				}
+
+				return best ? new VideoFrame(best) : undefined
+			},
+
+			async cancel() {
+				canceled = true
+				const pending = prefetchPromise
+				prefetchPromise = null
+
+				const prefetched = await pending?.catch(() => null)
+				if (prefetched)
+					for (const f of prefetched.frames) f.close()
+
+				await idle
+				clear()
+				input?.dispose()
+				input = null
+				sink = null
+			}
+		}
+	}
+}
+
+const toUs = (ms: Ms) => Math.round(ms * 1_000)

package/s/timeline/renderers/export/parts/produce-audio.ts

@@ -0,0 +1,64 @@
+
+import {AudioMix} from "./audio-mix.js"
+import {ms} from "../../../../units/ms.js"
+import {resampleToPlanar} from "./resamplers.js"
+import {applyGainToPlanar} from "./audio-gain.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createAudioSampler} from "../../parts/samplers/audio/sampler.js"
+
+export function produceAudio({
+	timeline,
+	resolveMedia
+}: {
+	timeline: TimelineFile,
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+	const mixer = new AudioMix()
+	const audio = streamAudio(timeline, resolveMedia)
+	const stream = new TransformStream<AudioData, AudioData>()
+	const writer = stream.writable.getWriter()
+
+	async function produce() {
+		for await (const chunk of mixer.mix(audio)) {
+			const data = new AudioData({
+				format: 'f32-planar',
+				sampleRate: chunk.sampleRate,
+				numberOfFrames: chunk.frames,
+				numberOfChannels: chunk.channels,
+				timestamp: Math.round(
+					(chunk.startFrame / chunk.sampleRate) * 1_000_000
+				),
+				data: new Float32Array(chunk.planar)
+			})
+
+			await writer.write(data)
+		}
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+
+async function *streamAudio(timeline: TimelineFile, resolveMedia: (hash: string) => DecoderSource) {
+	const audioSampler = createAudioSampler(resolveMedia)
+
+	for await (const {sample, timestamp, gain}
+		of audioSampler.sampleAudio(timeline, ms(0))) {
+
+		const {data} = resampleToPlanar(sample, 48000)
+		applyGainToPlanar(data, gain)
+
+		yield {
+			planes: data,
+			sampleRate: 48000,
+			timestamp
+		}
+
+		sample.close()
+	}
+}
+

package/s/timeline/renderers/export/parts/produce-video.ts

@@ -0,0 +1,49 @@
+
+import {Fps} from "../../../../units/fps.js"
+import {CursorVisualSampler} from "./cursor.js"
+import {Driver} from "../../../../driver/driver.js"
+import {fixedStep} from "../../parts/schedulers.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {computeItemDuration} from "../../parts/handy.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export function produceVideo({
+	timeline,
+	fps,
+	driver,
+	resolveMedia,
+}: {
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+	timeline: TimelineFile
+	fps: Fps
+}) {
+
+	const stream = new TransformStream<VideoFrame, VideoFrame>()
+	const writer = stream.writable.getWriter()
+	const sampler = new CursorVisualSampler(driver, resolveMedia, timeline)
+	const dt = 1 / fps
+	const duration = computeItemDuration(timeline.rootId, timeline)
+
+	async function produce() {
+		await fixedStep({fps, duration}, async (timecode, i) => {
+			const layers = await sampler.next(timecode)
+			const composed = await driver.composite(layers)
+
+			const frame = new VideoFrame(composed, {
+				timestamp: Math.round(i * dt * 1_000_000),
+				duration: Math.round(dt * 1_000_000)
+			})
+
+			await writer.write(frame)
+			composed.close()
+		})
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+

package/s/timeline/renderers/export/parts/resamplers.ts

@@ -0,0 +1,48 @@
+
+const resampleLinear = (
+	src: Float32Array,
+	fromRate: number,
+	targetRate: number
+) => {
+	if (fromRate === targetRate)
+		return src
+
+	const ratio = targetRate / fromRate
+	const outFrames = Math.max(1, Math.round(src.length * ratio))
+	const out = new Float32Array(outFrames)
+
+	for (let i = 0; i < outFrames; i++) {
+		const t = i / ratio
+		const i0 = Math.floor(t)
+		const i1 = Math.min(i0 + 1, src.length - 1)
+		const frac = t - i0
+		out[i] = src[i0] * (1 - frac) + src[i1] * frac
+	}
+
+	return out
+}
+
+export const resampleToPlanar = (
+	sample: {
+		numberOfFrames: number
+		numberOfChannels: number
+		sampleRate: number
+		copyTo: (dest: Float32Array, options: {planeIndex: number; format: 'f32-planar'}) => void
+	},
+	targetRate: number
+): {data: Float32Array[]; frames: number} => {
+	const channels = sample.numberOfChannels
+	const data = new Array<Float32Array>(channels)
+	let frames = 0
+
+	for (let ch = 0; ch < channels; ch++) {
+		const plane = new Float32Array(sample.numberOfFrames)
+		sample.copyTo(plane, {planeIndex: ch, format: 'f32-planar'})
+		const resampled = resampleLinear(plane, sample.sampleRate, targetRate)
+		data[ch] = resampled
+		frames = resampled.length
+	}
+
+	return {data, frames}
+}
+

package/s/timeline/renderers/export/produce.ts

@@ -0,0 +1,28 @@
+
+import {Fps} from "../../../units/fps.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {produceAudio} from "./parts/produce-audio.js"
+import {produceVideo} from "./parts/produce-video.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+export function produce(opts: {
+	timeline: TimelineFile
+	fps: Fps
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+
+	const audio = produceAudio({...opts})
+	const video = produceVideo({...opts})
+
+	return opts.driver.encode({
+		video,
+		audio,
+		config: {
+			audio: {codec: 'opus', bitrate: 128000},
+			video: {codec: 'vp9', bitrate: 1000000}
+		}
+	})
+}
+