@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/timeline/renderers/parts/handy.ts

@@ -0,0 +1,441 @@
+
+import {ms, Ms} from '../../../units/ms.js'
+import {Id, TimelineFile} from '../../parts/basics.js'
+import {Keyframes, TransformAnimation} from '../../types.js'
+import {SampleContext} from './samplers/visual/parts/types.js'
+import {I6, Mat6, mul6, transformToMat6} from '../../utils/matrix.js'
+import {ContainerItem, Item, Kind, PlayableItem} from '../../parts/item.js'
+import {resolveScalarAnimation, resolveTransformAnimation} from '../../utils/anim.js'
+
+function isPlayableItem(item: Item.Any): item is PlayableItem {
+	return 'duration' in item
+}
+
+type WalkAtCallbacks = {
+	sequence: (x: Item.Sequence, localTime: Ms, ancestors: AncestorAt[]) => void
+	stack: (x: Item.Stack, localTime: Ms, ancestors: AncestorAt[]) => void
+	video: (x: Item.Video, localTime: Ms, ancestors: AncestorAt[]) => void
+	image: (x: Item.Image, localTime: Ms, ancestors: AncestorAt[]) => void
+	text: (x: Item.Text, localTime: Ms, ancestors: AncestorAt[]) => void
+	caption: (x: Item.Caption, localTime: Ms, ancestors: AncestorAt[]) => void
+	audio: (x: Item.Audio, localTime: Ms, ancestors: AncestorAt[]) => void
+}
+
+type WalkCallbacks = {
+	sequence?: (x: Item.Sequence, matrix: Mat6, ancestors: AncestorAt[]) => void
+	stack?: (x: Item.Stack, matrix: Mat6, ancestors: AncestorAt[]) => void
+	video?: (x: Item.Video, matrix: Mat6, ancestors: AncestorAt[]) => void
+	image?: (x: Item.Image, matrix: Mat6, ancestors: AncestorAt[]) => void
+	text?: (x: Item.Text, matrix: Mat6, ancestors: AncestorAt[]) => void
+	caption?: (x: Item.Caption, matrix: Mat6, ancestors: AncestorAt[]) => void
+	audio?: (x: Item.Audio) => void
+}
+
+interface Props {
+	timeline: TimelineFile
+	timecode: Ms
+}
+
+export interface AncestorAt {
+	item: ContainerItem
+	localTime: Ms
+}
+
+interface At {
+	item: Item.Any
+	localTime: Ms
+	ancestors: AncestorAt[]
+}
+
+export function itemsAt(p: Props): At[] {
+	const results: At[] = []
+	const itemMap = new Map(p.timeline.items.map(item => [item.id, item]))
+
+	walkAt(p.timeline.rootId, itemMap, p.timecode, {
+		sequence: () => { },
+		stack: () => { },
+		video: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		image: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		text: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		caption: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		audio: (item, localTime, ancestors) => results.push({ item, localTime, ancestors })
+	})
+
+	return results
+}
+
+interface FromProps {
+	timeline: TimelineFile
+	from: Ms
+}
+
+export function itemsFrom(p: FromProps): At[] {
+	const results: At[] = []
+	const itemMap = new Map(p.timeline.items.map(item => [item.id, item]))
+
+	walkFrom(p.timeline.rootId, itemMap, p.from, {
+		sequence: () => { },
+		stack: () => { },
+		video: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		image: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		text: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		caption: (item, localTime, ancestors) => results.push({ item, localTime, ancestors }),
+		audio: (item, localTime, ancestors) => results.push({ item, localTime, ancestors })
+	})
+
+	return results
+}
+
+export function computeWorldMatrix(
+	items: Map<Id, Item.Any>,
+	ancestors: AncestorAt[],
+	item: Item.Any,
+	localTime: Ms,
+): Mat6 {
+	let world = I6
+
+	for (const ancestor of ancestors) {
+		world = applySpatialIfAny(items, ancestor.item, world, ancestor.localTime)
+	}
+
+	return applySpatialIfAny(items, item, world, localTime)
+}
+
+function applySpatialIfAny(
+	items: Map<Id, Item.Any>,
+	item: Item.Any,
+	parentMatrix: Mat6,
+	time: Ms,
+) {
+	let matrix = parentMatrix
+	if ("spatialId" in item && item.spatialId) {
+		const spatial = items.get(item.spatialId) as Item.Spatial | undefined
+		if (spatial?.enabled) {
+			const local = transformToMat6(spatial.transform)
+			matrix = mul6(local, matrix)
+		}
+	}
+
+	if ("animationIds" in item && item.animationIds) {
+		for (const id of item.animationIds) {
+			const animation = items.get(id) as Item.Animation | undefined
+			const anim = animation?.anims.transform
+			if (animation?.enabled && anim && transformActiveAt(anim, time)) {
+				const local = transformToMat6(resolveTransformAnimation(time, anim))
+				matrix = mul6(local, matrix)
+			}
+		}
+	}
+
+	return matrix
+}
+
+export function walk(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	parentMatrix: Mat6,
+	localTime: Ms,
+	callbacks: WalkCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	const currentMatrix = applySpatialIfAny(items, item, parentMatrix, localTime)
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack?.(item, currentMatrix, ancestors)
+			for (const childId of item.childrenIds) {
+				walk(childId, items, currentMatrix, localTime, callbacks, [...ancestors, {item, localTime}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence?.(item, currentMatrix, ancestors)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+				if (!isPlayableItem(child)) {
+					continue
+				}
+
+				walk(
+					childId,
+					items,
+					currentMatrix,
+					localTime,
+					callbacks,
+					[...ancestors, {item, localTime}]
+				)
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Image:
+			callbacks.image?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Caption:
+			callbacks.caption?.(item, currentMatrix, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio?.(item)
+			break
+	}
+}
+
+
+function walkAt(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	time: Ms,
+	callbacks: WalkAtCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack(item, time, ancestors)
+			for (const childId of item.childrenIds) {
+				walkAt(childId, items, time, callbacks, [...ancestors, {item, localTime: time}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence(item, time, ancestors)
+
+			let offset = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+
+				const duration = computeItemDurationFromMap(child.id, items)
+				if (duration <= 0)
+					continue
+
+				if (time >= offset && time < offset + duration) {
+					const localTime = ms(time - offset)
+					walkAt(
+						childId,
+						items,
+						localTime,
+						callbacks,
+						[...ancestors, {item, localTime: time}]
+					)
+					break
+				}
+
+				offset = ms(offset + duration)
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video(item, time, ancestors)
+			break
+
+		case Kind.Image:
+			callbacks.image(item, time, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text(item, time, ancestors)
+			break
+
+		case Kind.Caption:
+			callbacks.caption(item, time, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio(item, time, ancestors)
+			break
+	}
+}
+
+function walkFrom(
+	id: Id,
+	items: Map<Id, Item.Any>,
+	from: Ms,
+	callbacks: WalkAtCallbacks,
+	ancestors: AncestorAt[] = []
+) {
+	const item = items.get(id)
+	if (!item) return
+
+	switch (item.kind) {
+		case Kind.Stack:
+			callbacks.stack(item, from, ancestors)
+			for (const childId of item.childrenIds) {
+				walkFrom(childId, items, from, callbacks, [...ancestors, {item, localTime: from}])
+			}
+			break
+
+		case Kind.Sequence: {
+			callbacks.sequence(item, from, ancestors)
+
+			let offset = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const child = items.get(childId)
+
+				if (!child)
+					continue
+
+				const duration = computeItemDurationFromMap(child.id, items)
+				if (duration <= 0)
+					continue
+
+				const end = ms(offset + duration)
+				if (from >= end) {
+					offset = end
+					continue
+				}
+
+				const localTime = ms(Math.max(0, from - offset))
+				walkFrom(
+					childId,
+					items,
+					localTime,
+					callbacks,
+					[...ancestors, {item, localTime: from}]
+				)
+
+				offset = end
+			}
+
+			break
+		}
+
+		case Kind.Video:
+			callbacks.video(item, from, ancestors)
+			break
+
+		case Kind.Image:
+			callbacks.image(item, from, ancestors)
+			break
+
+		case Kind.Text:
+			callbacks.text(item, from, ancestors)
+			break
+
+		case Kind.Caption:
+			callbacks.caption(item, from, ancestors)
+			break
+
+		case Kind.Audio:
+			callbacks.audio(item, from, ancestors)
+			break
+	}
+}
+
+export function computeItemDuration(
+	id: number,
+	timeline: TimelineFile
+): Ms {
+	return computeItemDurationFromMap(
+		id,
+		new Map(timeline.items.map(item => [item.id, item]))
+	)
+}
+
+function computeItemDurationFromMap(
+	id: number,
+	items: Map<Id, Item.Any>
+): Ms {
+	const item = items.get(id)
+
+	if (!item) return ms(0)
+
+	switch (item.kind) {
+		case Kind.Sequence: {
+			let duration = ms(0)
+
+			for (const childId of item.childrenIds)
+				duration = ms(duration + computeItemDurationFromMap(childId, items))
+
+			return duration
+		}
+
+		case Kind.Stack: {
+			let longest = ms(0)
+
+			for (const childId of item.childrenIds) {
+				const duration = computeItemDurationFromMap(childId, items)
+				if (duration > longest) {
+					longest = duration
+				}
+			}
+
+			return longest
+		}
+
+		default: {
+			if (!isPlayableItem(item))
+				return ms(0)
+
+			return item.duration
+		}
+	}
+}
+
+export function computeOpacity(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+) {
+	if (!("animationIds" in item) || item.animationIds === undefined)
+		return 1
+
+	let opacity = 1
+	for (const id of item.animationIds) {
+		const animation = ctx.items.get(id) as Item.Animation | undefined
+		const anim = animation?.anims.opacity
+		if (animation?.enabled && anim && keyframesActiveAt(anim.track, time))
+			opacity = resolveScalarAnimation(time, anim)
+	}
+	return opacity
+}
+
+function keyframesActiveAt(keys: Keyframes, time: Ms) {
+	if (keys.length === 0)
+		return false
+
+	let start = keys[0][0]
+	let end = keys[0][0]
+	for (const [keyTime] of keys) {
+		start = Math.min(start, keyTime)
+		end = Math.max(end, keyTime)
+	}
+
+	return time >= start && time <= end
+}
+
+function transformActiveAt(anim: TransformAnimation, time: Ms) {
+	return (
+		keyframesActiveAt(anim.track.position.x, time) ||
+		keyframesActiveAt(anim.track.position.y, time) ||
+		keyframesActiveAt(anim.track.scale.x, time) ||
+		keyframesActiveAt(anim.track.scale.y, time) ||
+		keyframesActiveAt(anim.track.rotation, time)
+	)
+}

package/s/timeline/renderers/parts/samplers/audio/parts/find.ts

@@ -0,0 +1,19 @@
+
+import {ActiveStream} from "./types.js"
+
+export function findEarliestStream(streams: ActiveStream[]) {
+	let earliest = {
+		index: 0,
+		stream: streams[0],
+		time: streams[0].timelineTime()
+	}
+
+	for (const [index, stream] of streams.entries()) {
+		const time = stream.timelineTime()
+		if (time < earliest.time) {
+			earliest = {time, stream, index}
+		}
+	}
+
+	return earliest
+}

package/s/timeline/renderers/parts/samplers/audio/parts/init.ts

@@ -0,0 +1,60 @@
+
+import {AudioSinkPool} from "./sink.js"
+import {ActiveStream} from "./types.js"
+import {itemsFrom} from "../../../handy.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Kind} from "../../../../../parts/item.js"
+import {seconds} from "../../../../../../units/seconds.js"
+
+export async function initStreams(
+	pool: AudioSinkPool,
+	items: ReturnType<typeof itemsFrom>,
+	from: Ms
+): Promise<ActiveStream[]> {
+	const streams = await Promise.all(
+		items.map(async ({item, localTime}) => {
+			if (item.kind !== Kind.Audio)
+				return
+
+			const sink = await pool.getSink(item.mediaHash)
+			if (!sink)
+				return
+
+			const mediaTime = item.start + localTime
+			const offset = seconds((from - mediaTime) / 1000)
+			const iter = sink.samples(mediaTime / 1000)
+
+			const first = await iter.next()
+			if (first.done)
+				return
+
+			let currentSample = first.value
+			let nextPromise = iter.next()
+
+			return {
+				offset,
+				gain: item.gain ?? 1,
+				get currentSample() {return currentSample},
+				timelineTime: () => seconds(offset + currentSample.timestamp),
+				output: () => ({
+					sample: currentSample,
+					timestamp: offset + currentSample.timestamp,
+					gain: item.gain ?? 1
+				}),
+				advance: async () => {
+					const result = await nextPromise
+					if (result.done)
+						return false
+
+					currentSample = result.value
+					nextPromise = iter.next()
+
+					return true
+				}
+			}
+		})
+	)
+
+	return streams.filter((stream): stream is ActiveStream => !!stream)
+}
+

package/s/timeline/renderers/parts/samplers/audio/parts/sink.ts

@@ -0,0 +1,38 @@
+
+import {ALL_FORMATS, Input, AudioSampleSink} from "mediabunny"
+
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../../../driver/utils/load-decoder-source.js"
+
+type SinkState = {
+	input: Input
+	sink: AudioSampleSink | null
+}
+
+export class AudioSinkPool {
+	readonly #sinks = new Map<string, SinkState>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource
+	) {}
+
+	async getSink(hash: string) {
+		const existing = this.#sinks.get(hash)
+
+		if (existing)
+			return existing.sink
+
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(this.resolveMedia(hash)),
+		})
+
+		const audioTrack = await input.getPrimaryAudioTrack()
+		const canDecodeAudio = !!audioTrack && await audioTrack.canDecode()
+		const sink = canDecodeAudio && audioTrack ? new AudioSampleSink(audioTrack) : null
+
+		this.#sinks.set(hash, {input, sink})
+
+		return sink
+	}
+}

package/s/timeline/renderers/parts/samplers/audio/parts/types.ts

@@ -0,0 +1,16 @@
+
+import {AudioSample} from "mediabunny"
+import {Seconds} from "../../../../../../units/seconds.js"
+
+export type ActiveStream = {
+	offset: Seconds
+	gain: number
+	currentSample: AudioSample
+	timelineTime: () => Seconds
+	output: () => {
+		sample: AudioSample
+		timestamp: number
+		gain: number
+	}
+	advance: () => Promise<boolean>
+}

package/s/timeline/renderers/parts/samplers/audio/sampler.ts

@@ -0,0 +1,35 @@
+
+import {AudioSample} from "mediabunny"
+
+import {itemsFrom} from "../../handy.js"
+import {initStreams} from "./parts/init.js"
+import {AudioSinkPool} from "./parts/sink.js"
+import {Ms} from "../../../../../units/ms.js"
+import {findEarliestStream} from "./parts/find.js"
+import {TimelineFile} from "../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../driver/fns/schematic.js"
+
+export function createAudioSampler(resolveMedia: (hash: string) => DecoderSource) {
+	const sinkPool = new AudioSinkPool(resolveMedia)
+
+	return {
+		async *sampleAudio(timeline: TimelineFile, from: Ms) {
+			const items = itemsFrom({timeline, from})
+			const streams = await initStreams(sinkPool, items, from)
+
+			while (streams.length > 0) {
+				const {stream, index} = findEarliestStream(streams)
+
+				yield stream.output()
+
+				const advancing = await stream.advance()
+
+				if (!advancing) {
+					streams.splice(index, 1)
+				}
+			}
+
+		}
+	}
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/defaults.ts

@@ -0,0 +1,23 @@
+
+import {VideoSink} from "./video-sink.js"
+import {ImageSink} from "./image-sink.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Item} from "../../../../../parts/item.js"
+
+export type VideoSampler = (item: Item.Video, time: Ms) => Promise<VideoFrame | undefined>
+export type ImageSampler = (item: Item.Image, time: Ms) => Promise<VideoFrame | undefined>
+
+export function createDefaultVideoSampler(sink: VideoSink): VideoSampler {
+	return async (item, time) => {
+		const s = await sink.getSink(item.mediaHash)
+		const mediaTime = item.start + time
+		const sample = await s?.getSample(mediaTime / 1000)
+		const frame = sample?.toVideoFrame()
+		sample?.close()
+		return frame ?? undefined
+	}
+}
+
+export function createDefaultImageSampler(sink: ImageSink): ImageSampler {
+	return async (_item, time) => await sink.getFrame(_item.mediaHash, time)
+}

package/s/timeline/renderers/parts/samplers/visual/parts/image-sink.ts

@@ -0,0 +1,51 @@
+
+import {Ms} from "../../../../../../units/ms.js"
+import {Hash} from "../../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+
+type CachedImage = {
+	bitmap: ImageBitmap
+}
+
+export class ImageSink {
+	readonly #images = new Map<Hash, CachedImage>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource,
+	) {}
+
+	async getFrame(hash: Hash, time: Ms) {
+		const image = await this.#getImage(hash)
+		return new VideoFrame(image.bitmap, {
+			timestamp: Math.round(time * 1000),
+		})
+	}
+
+	async #getImage(hash: Hash) {
+		const existing = this.#images.get(hash)
+		if (existing)
+			return existing
+
+		const source = this.resolveMedia(hash)
+		const blob = source instanceof Blob
+			? source
+			: await fetch(source).then(response => response.blob())
+		const image = {bitmap: await createImageBitmap(blob)}
+
+		this.#images.set(hash, image)
+		return image
+	}
+
+	disposeAll() {
+		for (const image of this.#images.values())
+			image.bitmap.close()
+		this.#images.clear()
+	}
+
+	dispose(hash: Hash) {
+		const image = this.#images.get(hash)
+		image?.bitmap.close()
+		this.#images.delete(hash)
+	}
+}
+