@omnimedia/omnitool 1.1.0-8 → 1.1.0-80

package/s/timeline/renderers/export/parts/produce-audio.ts

@@ -0,0 +1,64 @@
+
+import {AudioMix} from "./audio-mix.js"
+import {ms} from "../../../../units/ms.js"
+import {resampleToPlanar} from "./resamplers.js"
+import {applyGainToPlanar} from "./audio-gain.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createAudioSampler} from "../../parts/samplers/audio/sampler.js"
+
+export function produceAudio({
+	timeline,
+	resolveMedia
+}: {
+	timeline: TimelineFile,
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+	const mixer = new AudioMix()
+	const audio = streamAudio(timeline, resolveMedia)
+	const stream = new TransformStream<AudioData, AudioData>()
+	const writer = stream.writable.getWriter()
+
+	async function produce() {
+		for await (const chunk of mixer.mix(audio)) {
+			const data = new AudioData({
+				format: 'f32-planar',
+				sampleRate: chunk.sampleRate,
+				numberOfFrames: chunk.frames,
+				numberOfChannels: chunk.channels,
+				timestamp: Math.round(
+					(chunk.startFrame / chunk.sampleRate) * 1_000_000
+				),
+				data: new Float32Array(chunk.planar)
+			})
+
+			await writer.write(data)
+		}
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+
+async function *streamAudio(timeline: TimelineFile, resolveMedia: (hash: string) => DecoderSource) {
+	const audioSampler = createAudioSampler(resolveMedia)
+
+	for await (const {sample, timestamp, gain}
+		of audioSampler.sampleAudio(timeline, ms(0))) {
+
+		const {data} = resampleToPlanar(sample, 48000)
+		applyGainToPlanar(data, gain)
+
+		yield {
+			planes: data,
+			sampleRate: 48000,
+			timestamp
+		}
+
+		sample.close()
+	}
+}
+

package/s/timeline/renderers/export/parts/produce-video.ts

@@ -0,0 +1,49 @@
+
+import {Fps} from "../../../../units/fps.js"
+import {CursorVisualSampler} from "./cursor.js"
+import {Driver} from "../../../../driver/driver.js"
+import {fixedStep} from "../../parts/schedulers.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {computeItemDuration} from "../../parts/handy.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export function produceVideo({
+	timeline,
+	fps,
+	driver,
+	resolveMedia,
+}: {
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+	timeline: TimelineFile
+	fps: Fps
+}) {
+
+	const stream = new TransformStream<VideoFrame, VideoFrame>()
+	const writer = stream.writable.getWriter()
+	const sampler = new CursorVisualSampler(driver, resolveMedia, timeline)
+	const dt = 1 / fps
+	const duration = computeItemDuration(timeline.rootId, timeline)
+
+	async function produce() {
+		await fixedStep({fps, duration}, async (timecode, i) => {
+			const layers = await sampler.next(timecode)
+			const composed = await driver.composite(layers)
+
+			const frame = new VideoFrame(composed, {
+				timestamp: Math.round(i * dt * 1_000_000),
+				duration: Math.round(dt * 1_000_000)
+			})
+
+			await writer.write(frame)
+			composed.close()
+		})
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+

package/s/timeline/renderers/export/parts/resamplers.ts

@@ -0,0 +1,48 @@
+
+const resampleLinear = (
+	src: Float32Array,
+	fromRate: number,
+	targetRate: number
+) => {
+	if (fromRate === targetRate)
+		return src
+
+	const ratio = targetRate / fromRate
+	const outFrames = Math.max(1, Math.round(src.length * ratio))
+	const out = new Float32Array(outFrames)
+
+	for (let i = 0; i < outFrames; i++) {
+		const t = i / ratio
+		const i0 = Math.floor(t)
+		const i1 = Math.min(i0 + 1, src.length - 1)
+		const frac = t - i0
+		out[i] = src[i0] * (1 - frac) + src[i1] * frac
+	}
+
+	return out
+}
+
+export const resampleToPlanar = (
+	sample: {
+		numberOfFrames: number
+		numberOfChannels: number
+		sampleRate: number
+		copyTo: (dest: Float32Array, options: {planeIndex: number; format: 'f32-planar'}) => void
+	},
+	targetRate: number
+): {data: Float32Array[]; frames: number} => {
+	const channels = sample.numberOfChannels
+	const data = new Array<Float32Array>(channels)
+	let frames = 0
+
+	for (let ch = 0; ch < channels; ch++) {
+		const plane = new Float32Array(sample.numberOfFrames)
+		sample.copyTo(plane, {planeIndex: ch, format: 'f32-planar'})
+		const resampled = resampleLinear(plane, sample.sampleRate, targetRate)
+		data[ch] = resampled
+		frames = resampled.length
+	}
+
+	return {data, frames}
+}
+

package/s/timeline/renderers/export/produce.ts

@@ -0,0 +1,28 @@
+
+import {Fps} from "../../../units/fps.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {produceAudio} from "./parts/produce-audio.js"
+import {produceVideo} from "./parts/produce-video.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+export function produce(opts: {
+	timeline: TimelineFile
+	fps: Fps
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+
+	const audio = produceAudio({...opts})
+	const video = produceVideo({...opts})
+
+	return opts.driver.encode({
+		video,
+		audio,
+		config: {
+			audio: {codec: 'opus', bitrate: 128000},
+			video: {codec: 'vp9', bitrate: 1000000}
+		}
+	})
+}
+

package/s/timeline/renderers/parts/handy.ts

@@ -0,0 +1,377 @@
+
+import {ms, Ms} from '../../../units/ms.js'
+import {Id, TimelineFile} from '../../parts/basics.js'
+import { SampleContext } from './samplers/visual/parts/types.js'
+import {I6, Mat6, mul6, transformToMat6} from '../../utils/matrix.js'
+import {resolveScalarAnimation, resolveSpatialTransform} from '../../utils/anim.js'
+import {ContainerItem, Item, Kind, PlayableItem, SpatialItem} from '../../parts/item.js'
+
+function isPlayableItem(item: Item.Any): item is PlayableItem {
+	return 'duration' in item
+}
+
+type WalkAtCallbacks = {
+	sequence: (x: Item.Sequence, localTime: Ms, ancestors: AncestorAt[]) => void
+	stack: (x: Item.Stack, localTime: Ms, ancestors: AncestorAt[]) => void
+	video: (x: Item.Video, localTime: Ms, ancestors: AncestorAt[]) => void
+	text: (x: Item.Text, localTime: Ms, ancestors: AncestorAt[]) => void
+	audio: (x: Item.Audio, localTime: Ms, ancestors: AncestorAt[]) => void
+}
+
+type WalkCallbacks = {
+	sequence?: (x: Item.Sequence, matrix: Mat6, ancestors: AncestorAt[]) => void
+	stack?: (x: Item.Stack, matrix: Mat6, ancestors: AncestorAt[]) => void
+	video?: (x: Item.Video, matrix: Mat6, ancestors: AncestorAt[]) => void
+	text?: (x: Item.Text, matrix: Mat6, ancestors: AncestorAt[]) => void
+	audio?: (x: Item.Audio) => void
+}
+
+interface Props {
+	timeline: TimelineFile
+	timecode: Ms
+}
+
+export interface AncestorAt {
+	item: ContainerItem
+	localTime: Ms
+}
+
+interface At {
+	item: Item.Any
+	localTime: Ms
+	ancestors: AncestorAt[]
+}
+
+export function itemsAt(p: Props): At[] {
+	const results: At[] = []
+	const itemMap = new Map(p.timeline.items.map(item => [item.id, item]))
+
+	walkAt(p.timeline.rootId, itemMap, p.timecode, {
+		sequence: () => { },
+		stack: () => { },
+		video: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		text: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		audio: (item, localTime, ancestors) => results.push({ item, localTime, ancestors })
+	})
+
+	return results
+}
+
+interface FromProps {
+	timeline: TimelineFile
+	from: Ms
+}
+
+export function itemsFrom(p: FromProps): At[] {
+	const results: At[] = []
+	const itemMap = new Map(p.timeline.items.map(item => [item.id, item]))
+
+	walkFrom(p.timeline.rootId, itemMap, p.from, {
+		sequence: () => { },
+		stack: () => { },
+		video: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		text: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		audio: (item, localTime, ancestors) => results.push({ item, localTime, ancestors })
+	})
+
+	return results
+}
+
+export function computeWorldMatrix(
+	items: Map<Id, Item.Any>,
+	ancestors: AncestorAt[],
+	item: Item.Any,
+	localTime: Ms,
+): Mat6 {
+	let world = I6
+
+	for (const ancestor of ancestors) {
+		world = applySpatialIfAny(items, ancestor.item, world, ancestor.localTime)
+	}
+
+	return applySpatialIfAny(items, item, world, localTime)
+}
+
+function applySpatialIfAny(
+	items: Map<Id, Item.Any>,
+	item: Item.Any,
+	parentMatrix: Mat6,
+	time: Ms,
+) {
+	if ("spatialId" in item && item.spatialId) {
+		const spatial = items.get(item.spatialId) as SpatialItem | undefined
+		if (spatial?.enabled) {
+			const local = transformToMat6(resolveSpatialTransform(spatial, time))
+			return mul6(local, parentMatrix)
+		}
+	}
+	return parentMatrix
+}
+
+export function walk(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	parentMatrix: Mat6,
+	localTime: Ms,
+	callbacks: WalkCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	const currentMatrix = applySpatialIfAny(items, item, parentMatrix, localTime)
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack?.(item, currentMatrix, ancestors)
+			for (const childId of item.childrenIds) {
+				walk(childId, items, currentMatrix, localTime, callbacks, [...ancestors, {item, localTime}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence?.(item, currentMatrix, ancestors)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+				if (!isPlayableItem(child)) {
+					continue
+				}
+
+				walk(
+					childId,
+					items,
+					currentMatrix,
+					localTime,
+					callbacks,
+					[...ancestors, {item, localTime}]
+				)
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio?.(item)
+			break
+	}
+}
+
+
+function walkAt(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	time: Ms,
+	callbacks: WalkAtCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack(item, time, ancestors)
+			for (const childId of item.childrenIds) {
+				walkAt(childId, items, time, callbacks, [...ancestors, {item, localTime: time}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence(item, time, ancestors)
+
+			let offset = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+				if (!isPlayableItem(child)) {
+					continue
+				}
+
+				if (time >= offset && time < offset + child.duration) {
+					const localTime = ms(time - offset)
+					walkAt(
+						childId,
+						items,
+						localTime,
+						callbacks,
+						[...ancestors, {item, localTime: time}]
+					)
+					break
+				}
+
+				offset = ms(offset + child.duration)
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video(item, time, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text(item, time, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio(item, time, ancestors)
+			break
+	}
+}
+
+function walkFrom(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	from: Ms,
+	callbacks: WalkAtCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack(item, from, ancestors)
+			for (const childId of item.childrenIds) {
+				walkFrom(childId, items, from, callbacks, [...ancestors, {item, localTime: from}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence(item, from, ancestors)
+
+			let offset = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+				if (!isPlayableItem(child)) {
+					continue
+				}
+
+				const end = ms(offset + child.duration)
+				if (from >= end) {
+					offset = end
+					continue
+				}
+
+				const localTime = ms(Math.max(0, from - offset))
+				walkFrom(
+					childId,
+					items,
+					localTime,
+					callbacks,
+					[...ancestors, {item, localTime: from}]
+				)
+
+				offset = end
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video(item, from, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text(item, from, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio(item, from, ancestors)
+			break
+	}
+}
+
+export function computeItemDuration(
+	id: number,
+	timeline: TimelineFile
+): Ms {
+	const item = timeline.items.find(item => item.id === id)
+
+	if (!item) return ms(0)
+
+	switch (item.kind) {
+		case Kind.Sequence: {
+			const children = item.childrenIds
+				.map(childId => timeline.items.find(x => x.id === childId))
+				.filter(Boolean) as Item.Any[]
+
+			let total = ms(0)
+
+			for (let i = 0; i < children.length; i++) {
+				const child = children[i]
+
+				if (child.kind === Kind.Transition) {
+					const prev = children[i - 1]
+					const next = children[i + 1]
+
+					if (prev && next && prev.kind !== Kind.Transition && next.kind !== Kind.Transition) {
+						const prevDur = computeItemDuration(prev.id, timeline)
+						const nextDur = computeItemDuration(next.id, timeline)
+						const overlap = Math.max(0, Math.min(child.duration, prevDur, nextDur))
+
+						total = ms(total - overlap)
+					}
+					continue
+				}
+
+				total = ms(total + computeItemDuration(child.id, timeline))
+			}
+
+			return total
+		}
+
+		case Kind.Stack: {
+			let longest = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const duration = computeItemDuration(childId, timeline)
+				if (duration > longest) {
+					longest = duration
+				}
+			}
+
+			return longest
+		}
+
+		default: {
+			if (!isPlayableItem(item))
+				return ms(0)
+
+			return item.duration
+		}
+	}
+}
+
+export function computeOpacity(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+) {
+	if (!("animationId" in item) || item.animationId === undefined)
+		return 1
+
+	const animation = ctx.items.get(item.animationId) as Item.Animation | undefined
+	return animation?.enabled && animation.anims.opacity
+		? resolveScalarAnimation(time, animation.anims.opacity)
+		: 1
+}
+

package/s/timeline/renderers/parts/samplers/audio/parts/find.ts

@@ -0,0 +1,19 @@
+
+import {ActiveStream} from "./types.js"
+
+export function findEarliestStream(streams: ActiveStream[]) {
+	let earliest = {
+		index: 0,
+		stream: streams[0],
+		time: streams[0].timelineTime()
+	}
+
+	for (const [index, stream] of streams.entries()) {
+		const time = stream.timelineTime()
+		if (time < earliest.time) {
+			earliest = {time, stream, index}
+		}
+	}
+
+	return earliest
+}

package/s/timeline/renderers/parts/samplers/audio/parts/init.ts

@@ -0,0 +1,60 @@
+
+import {AudioSinkPool} from "./sink.js"
+import {ActiveStream} from "./types.js"
+import {itemsFrom} from "../../../handy.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Kind} from "../../../../../parts/item.js"
+import {seconds} from "../../../../../../units/seconds.js"
+
+export async function initStreams(
+	pool: AudioSinkPool,
+	items: ReturnType<typeof itemsFrom>,
+	from: Ms
+): Promise<ActiveStream[]> {
+	const streams = await Promise.all(
+		items.map(async ({item, localTime}) => {
+			if (item.kind !== Kind.Audio)
+				return
+
+			const sink = await pool.getSink(item.mediaHash)
+			if (!sink)
+				return
+
+			const mediaTime = item.start + localTime
+			const offset = seconds((from - mediaTime) / 1000)
+			const iter = sink.samples(mediaTime / 1000)
+
+			const first = await iter.next()
+			if (first.done)
+				return
+
+			let currentSample = first.value
+			let nextPromise = iter.next()
+
+			return {
+				offset,
+				gain: item.gain ?? 1,
+				get currentSample() {return currentSample},
+				timelineTime: () => seconds(offset + currentSample.timestamp),
+				output: () => ({
+					sample: currentSample,
+					timestamp: offset + currentSample.timestamp,
+					gain: item.gain ?? 1
+				}),
+				advance: async () => {
+					const result = await nextPromise
+					if (result.done)
+						return false
+
+					currentSample = result.value
+					nextPromise = iter.next()
+
+					return true
+				}
+			}
+		})
+	)
+
+	return streams.filter((stream): stream is ActiveStream => !!stream)
+}
+

package/s/timeline/renderers/parts/samplers/audio/parts/sink.ts

@@ -0,0 +1,38 @@
+
+import {ALL_FORMATS, Input, AudioSampleSink} from "mediabunny"
+
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../../../driver/utils/load-decoder-source.js"
+
+type SinkState = {
+	input: Input
+	sink: AudioSampleSink | null
+}
+
+export class AudioSinkPool {
+	readonly #sinks = new Map<string, SinkState>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource
+	) {}
+
+	async getSink(hash: string) {
+		const existing = this.#sinks.get(hash)
+
+		if (existing)
+			return existing.sink
+
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(this.resolveMedia(hash)),
+		})
+
+		const audioTrack = await input.getPrimaryAudioTrack()
+		const canDecodeAudio = !!audioTrack && await audioTrack.canDecode()
+		const sink = canDecodeAudio && audioTrack ? new AudioSampleSink(audioTrack) : null
+
+		this.#sinks.set(hash, {input, sink})
+
+		return sink
+	}
+}

package/s/timeline/renderers/parts/samplers/audio/parts/types.ts

@@ -0,0 +1,16 @@
+
+import {AudioSample} from "mediabunny"
+import {Seconds} from "../../../../../../units/seconds.js"
+
+export type ActiveStream = {
+	offset: Seconds
+	gain: number
+	currentSample: AudioSample
+	timelineTime: () => Seconds
+	output: () => {
+		sample: AudioSample
+		timestamp: number
+		gain: number
+	}
+	advance: () => Promise<boolean>
+}