@omnimedia/omnitool 1.1.0-3 → 1.1.0-31

package/s/timeline/parts/compositor/parts/tree-builder.ts

@@ -0,0 +1,196 @@
+import {Item, Kind} from "../../item.js"
+import {ImageLayer, Layer} from "../../../../driver/fns/schematic.js"
+import {I6, Mat6, mul6, transformToMat6} from "../../../utils/matrix.js"
+
+export type AudioStreamComponent = {
+	getStream: () => AsyncGenerator<AudioData>
+}
+export type AudioPlaybackComponent = {
+	onTimeUpdate: (time: number) => void
+}
+export type VisualComponent = {
+	sampleAt: (time: number) => Promise<Layer[]>
+}
+
+export type Node<T> = {
+	duration: number
+	visuals?: VisualComponent
+	audio?: T
+}
+
+interface Sampler<T> {
+	video(item: Item.Video, parentMatrix: Mat6): Promise<Node<T>>
+	audio(item: Item.Audio): Promise<Node<T>>
+	dispose(): Promise<void>
+}
+
+const requireItem = (items: Map<number, Item.Any>, id: number) => items.get(id)!
+export const getWorldMat6 = (
+	items: Map<number, Item.Any>,
+	item: Item.Text | Item.Sequence | Item.Stack | Item.Video,
+	parent?: Mat6
+): Mat6 => {
+	let world = parent ?? I6
+	if (item.spatialId) {
+		const spatial = requireItem(items, item.spatialId) as Item.Spatial
+		const local = transformToMat6(spatial.transform)
+		world = mul6(local, world)
+	}
+	return world
+}
+
+export type WebcodecsSampler = Sampler<AudioStreamComponent>
+export interface HTMLSampler extends Sampler<AudioPlaybackComponent> {
+	setPaused(v: boolean): void
+}
+
+export abstract class TreeBuilder<T> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: Sampler<T>) {}
+
+	async build(root: Item.Any, parentMatrix?: Mat6): Promise<Node<T>> {
+		switch (root.kind) {
+			case Kind.Video: return this.sampler.video(root, getWorldMat6(this.items, root, parentMatrix))
+			case Kind.Audio: return this.sampler.audio(root)
+			case Kind.Text: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				const styleItem = root.styleId !== undefined
+					? this.items.get(root.styleId) as Item.TextStyle
+					: undefined
+				return {
+					duration: root.duration,
+					visuals: {
+						sampleAt: async (t) => {
+							if (t < 0 || t >= root.duration)
+								return []
+							else return [{
+								kind: "text",
+								content: root.content,
+								style: styleItem?.style,
+								matrix
+							}]
+						}
+					}
+				}
+			}
+			case Kind.Gap: return {
+				duration: root.duration,
+				visuals: {
+					sampleAt: async () => []
+				}
+			}
+			case Kind.Stack: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				const children = await Promise.all(root.childrenIds.map(id => this.build(requireItem(this.items, id), matrix)))
+				return this.#composeStack(children)
+			}
+			case Kind.Sequence: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				return this.#composeSequence(root, matrix)
+			}
+			default: return {duration: 0}
+		}
+	}
+
+	abstract composeAudio_Stack(children: Node<T>[]): T | undefined
+	abstract composeAudio_Sequence(children: Node<T>[]): T | undefined
+
+	// Visual composition is the same for both builders, so it lives here.
+	#composeVisuals_Stack(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				const layers = await Promise.all(children.map(c => c.visuals ? c.visuals.sampleAt(time) : Promise.resolve([])))
+				return layers.flat()
+			}
+		}
+	}
+
+	#composeVisuals_Sequence(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) return child.visuals ? child.visuals.sampleAt(localTime) : []
+					localTime -= child.duration
+				}
+				return []
+			}
+		}
+	}
+
+	#composeStack(children: Node<T>[]): Node<T> {
+		const duration = Math.max(0, ...children.map(k => (Number.isFinite(k.duration) ? k.duration : 0)))
+		return {
+			duration,
+			visuals: this.#composeVisuals_Stack(children),
+			audio: this.composeAudio_Stack(children),
+		}
+	}
+
+	async #composeSequence(sequence: Item.Sequence, parentMatrix?: Mat6): Promise<Node<T>> {
+		const childItems = sequence.childrenIds.map(id => requireItem(this.items, id))
+		const children = await this.#processChildren(childItems, parentMatrix)
+		const duration = children.reduce((a, k) => a + k.duration, 0)
+		return {
+			duration,
+			visuals: this.#composeVisuals_Sequence(children),
+			audio: this.composeAudio_Sequence(children),
+		}
+	}
+
+	async #processChildren(childItems: Item.Any[], parentMatrix?: Mat6): Promise<Node<T>[]> {
+		const processedNodes: Node<T>[] = []
+		for (let i = 0; i < childItems.length; i++) {
+			const item = childItems[i]
+
+			if (item.kind !== Kind.Transition) {
+				processedNodes.push(await this.build(item, parentMatrix))
+				continue
+			}
+
+			const outgoingNode = processedNodes.pop()
+			const incomingItem = childItems[i + 1]
+
+			if (!outgoingNode || !incomingItem || incomingItem.kind === Kind.Transition) {
+				if (outgoingNode) processedNodes.push(outgoingNode)
+				continue
+			}
+
+			const incomingNode = await this.build(incomingItem, parentMatrix)
+			const transitionNode = await this.#createTransitionNode(item, outgoingNode, incomingNode)
+			processedNodes.push(transitionNode)
+			i++
+		}
+		return processedNodes
+	}
+
+	async #createTransitionNode(transitionItem: Item.Transition, outgoingNode: Node<T>, incomingNode: Node<T>): Promise<Node<T>> {
+		const overlap = Math.max(0, Math.min(transitionItem.duration, outgoingNode.duration, incomingNode.duration))
+		const start = Math.max(0, outgoingNode.duration - overlap)
+		const combinedDuration = outgoingNode.duration + incomingNode.duration - overlap
+		return {
+			duration: combinedDuration,
+			visuals: {
+				sampleAt: async (t) => {
+					if (!outgoingNode.visuals || !incomingNode.visuals) return []
+					if (t < start) return outgoingNode.visuals.sampleAt(t)
+					if (t < outgoingNode.duration) {
+						const localTime = t - start
+						const progress = overlap > 0 ? (localTime / overlap) : 1
+						const from = await outgoingNode.visuals.sampleAt(t) as ImageLayer[]
+						const to = await incomingNode.visuals.sampleAt(localTime) as ImageLayer[]
+						if(!from[0]?.frame || !to[0]?.frame) return []
+						return [{
+							kind: "transition",
+							name: "circle",
+							progress,
+							from: from[0].frame,
+							to: to[0].frame,
+						}]
+					}
+					return incomingNode.visuals.sampleAt(t - outgoingNode.duration + overlap)
+				}
+			},
+			audio: this.composeAudio_Sequence([outgoingNode, incomingNode])
+		}
+	}
+}

package/s/timeline/parts/compositor/parts/webcodecs-tree.ts

@@ -0,0 +1,30 @@
+import {Item} from "../../item.js"
+import {AudioStreamComponent, Node, WebcodecsSampler, TreeBuilder} from "./tree-builder.js"
+
+class WebCodecsNodeBuilder extends TreeBuilder<AudioStreamComponent> {
+	composeAudio_Stack(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+}
+
+export function buildWebCodecsNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: WebcodecsSampler) {
+	const builder = new WebCodecsNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/playback.ts

@@ -0,0 +1,94 @@
+import {signal} from "@e280/strata"
+
+import {TimelineFile} from "../basics.js"
+import {realtime} from "./parts/schedulers.js"
+import {Driver} from "../../../driver/driver.js"
+import {makeHtmlSampler} from "./samplers/html.js"
+import {buildHTMLNodeTree} from "./parts/html-tree.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {AudioPlaybackComponent, HTMLSampler, Node} from "./parts/tree-builder.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	readonly currentTime = signal(0)
+	#controller = realtime(
+		compositeTime => this.#tick(compositeTime),
+		currentTime => this.currentTime(currentTime)
+	)
+
+	constructor(
+		private driver: Driver,
+		public canvas: HTMLCanvasElement,
+		private root: Node<AudioPlaybackComponent>,
+		private sampler: HTMLSampler,
+		private resolveMedia: ResolveMedia = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#controller.setFPS(30)
+	}
+
+	get context() {
+		return this.canvas.getContext("2d")!
+	}
+
+	static async create(driver: Driver, timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		const sampler = makeHtmlSampler(() => "/assets/temp/gl.mp4")
+		const root = await buildHTMLNodeTree(rootItem, items, sampler)
+		const canvas = document.createElement("canvas")
+		canvas.width = 1920
+		canvas.height = 1080
+		return new this(driver, canvas, root, sampler)
+	}
+
+	async #tick(ms: number) {
+		const duration = this.root.duration
+		const tt = ms > duration ? duration : ms
+		this.root.audio?.onTimeUpdate(tt)
+		const layers = await this.root.visuals?.sampleAt(tt) ?? []
+		const frame = await this.driver.composite(layers)
+		this.context.drawImage(frame, 0, 0)
+		frame.close()
+		if (ms >= duration) this.pause()
+	}
+
+	async play() {
+		if (!this.#controller.isPlaying()) {
+			this.sampler.setPaused!(false)
+			this.#controller.play()
+		}
+	}
+
+	pause() {
+		if(this.#controller.isPlaying()) {
+			this.#controller.pause()
+			this.sampler.setPaused!(true)
+		}
+	}
+
+	async seek(ms: number) {
+		this.pause()
+		this.#controller.seek(ms)
+		this.root.audio?.onTimeUpdate(ms)
+		const layers = await this.root.visuals?.sampleAt(ms) ?? []
+		const frame = await this.driver.composite(layers)
+		this.context.drawImage(frame, 0, 0)
+		frame.close()
+	}
+
+	setFPS(value: number) {
+		this.#controller.setFPS(value)
+	}
+
+	/**
+	 call this whenever your timeline state changes
+	*/
+	async update(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		this.root = await buildHTMLNodeTree(rootItem, items, this.sampler)
+		await this.seek(this.currentTime())
+	}
+}
+

package/s/timeline/parts/compositor/samplers/html.ts

@@ -0,0 +1,115 @@
+import {Item} from "../../item.js"
+import {HTMLSampler} from "../parts/tree-builder.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+const toUrl = (src: DecoderSource) => (src instanceof Blob ? URL.createObjectURL(src) : String(src))
+
+export function makeHtmlSampler(resolveMedia: (hash: string) => DecoderSource): HTMLSampler {
+	const videoElements = new Map<number, HTMLVideoElement>()
+	const audioElements = new Map<number, HTMLAudioElement>()
+
+	function getOrCreateVideoElement(clip: Item.Video) {
+		let video = videoElements.get(clip.id)
+		if (!video) {
+			video = document.createElement("video")
+			video.playsInline = true
+			video.muted = true
+			video.preload = "auto"
+			video.crossOrigin = "anonymous"
+			video.src = toUrl(resolveMedia(clip.mediaHash))
+			videoElements.set(clip.id, video)
+		}
+		return video
+	}
+
+	function getOrCreateAudioElement(clip: Item.Audio) {
+		let audio = audioElements.get(clip.id)
+		if (!audio) {
+			audio = document.createElement("audio")
+			audio.preload = "auto"
+			audio.crossOrigin = "anonymous"
+			audio.src = toUrl(resolveMedia(clip.mediaHash))
+			audio.volume = 0.2
+			audioElements.set(clip.id, audio)
+		}
+		return audio
+	}
+
+	let paused = true
+
+	return {
+		async video(item, matrix) {
+			const video = getOrCreateVideoElement(item)
+			return {
+				duration: item.duration,
+				// if paused seek otherwise play
+				visuals: {
+					sampleAt: async (ms) => {
+						if (ms < 0 || ms >= item.duration)
+							return []
+
+						if(video.paused && paused) {
+							await seek(video, ms / 1000)
+						}
+
+						if(video.paused && !paused) {
+							await video.play()
+						}
+
+						const frame = new VideoFrame(video)
+						return frame ? [{kind: "image", frame, matrix}] : []
+					}
+				}
+			}
+		},
+		async audio(item) {
+			const audio = getOrCreateAudioElement(item)
+			return {
+				duration: item.duration,
+				audio: {
+					onTimeUpdate: async (ms) => {
+						const localTime = item.start + ms
+						if(audio.paused && paused) {
+							await seek(audio, localTime / 1000)
+						}
+						if(audio.paused && !paused) {
+							await audio.play()
+						}
+						return []
+					}
+				}
+			}
+		},
+		async dispose() {
+			const elements = [...videoElements.values(), ...audioElements.values()]
+      for (const element of elements) {
+        element.pause()
+        if (element.src.startsWith("blob:"))
+        	URL.revokeObjectURL(element.src)
+        element.remove()
+      }
+      videoElements.clear()
+      audioElements.clear()
+		},
+		async setPaused(p) {
+			paused = p
+			const elements = [...videoElements.values(), ...audioElements.values()]
+			for(const element of elements) {
+				if(p) element.pause()
+			}
+		},
+	}
+}
+
+function seek(media: HTMLVideoElement | HTMLAudioElement, time: number): Promise<void> {
+  return new Promise((resolve) => {
+    const onSeeked = () => {
+      media.removeEventListener("seeked", onSeeked)
+      resolve()
+    }
+    media.addEventListener("seeked", onSeeked)
+    if(media.fastSeek) {
+    	media.fastSeek(time)
+    } else media.currentTime = time
+  })
+}

package/s/timeline/parts/compositor/samplers/webcodecs.ts

@@ -0,0 +1,61 @@
+import {Item} from "../../item.js"
+import {Driver} from "../../../../driver/driver.js"
+import {WebcodecsSampler} from "../parts/tree-builder.js"
+import {VideoCursor} from "../../../utils/video-cursor.js"
+import {AudioStream} from "../../../utils/audio-stream.js"
+
+const toUs = (seconds: number) => Math.round(seconds * 1_000_000)
+
+export function makeWebCodecsSampler(
+	driver: Driver,
+	resolveMedia: (hash: string) => any
+): WebcodecsSampler {
+	const videoCursors = new Map<number, VideoCursor>()
+
+	async function getCursorForVideo(videoItem: Item.Video): Promise<VideoCursor> {
+		const existing = videoCursors.get(videoItem.id)
+		if (existing) return existing
+		const source = resolveMedia(videoItem.mediaHash)
+		const video = driver.decodeVideo({source})
+		const cursor = new VideoCursor(video.getReader())
+		videoCursors.set(videoItem.id, cursor)
+		return cursor
+	}
+
+	return {
+		async video(item, matrix) {
+			const cursor = await getCursorForVideo(item)
+			const baseUs = toUs(item.start ?? 0)
+			return {
+				duration: item.duration,
+				visuals: {
+					sampleAt: async (time: number) => {
+						const frame = await cursor.atOrNear(baseUs + toUs(time))
+						return frame ? [{kind: "image", frame, matrix}] : []
+					}
+				}
+			}
+		},
+		async audio(item) {
+			return {
+				duration: item.duration,
+				audio: {
+					getStream: async function*() {
+						const source = resolveMedia(item.mediaHash)
+						const startUs = item.start
+						const endUs = (item.start + item.duration)
+						const audio = driver.decodeAudio({source, start: startUs, end: endUs})
+						const audioStream = new AudioStream(audio.getReader())
+						yield* audioStream.stream()
+					},
+				}
+			}
+		},
+  	async dispose() {
+    	const tasks = Array.from([...videoCursors.values()], c => c.cancel())
+    	videoCursors.clear()
+    	await Promise.all(tasks)
+  	}
+	}
+}
+

package/s/timeline/parts/item.ts

@@ -1,12 +1,18 @@
+import {TextStyleOptions} from "pixi.js"
 
 import {Id, Hash} from "./basics.js"
+import {Transform} from "../types.js"
 
 export enum Kind {
 	Sequence,
 	Stack,
-	Clip,
+	Video,
+	Audio,
 	Text,
+	Gap,
+	Spatial,
 	Transition,
+	TextStyle
 }
 
 export enum Effect {
@@ -14,21 +20,50 @@ export enum Effect {
 }
 
 export namespace Item {
+  export type TextStyle = {
+    id: Id
+    kind: Kind.TextStyle
+		style: TextStyleOptions
+  }
+
+  export type Spatial = {
+    id: Id
+    kind: Kind.Spatial
+    transform: Transform
+  }
+
+	export type Gap = {
+		id: Id
+		kind: Kind.Gap
+		duration: number
+	}
+
 	export type Sequence = {
 		id: Id
 		kind: Kind.Sequence
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
 	}
 
 	export type Stack = {
 		id: Id
 		kind: Kind.Stack
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
 	}
 
-	export type Clip = {
+	export type Video = {
 		id: Id
-		kind: Kind.Clip
+		kind: Kind.Video
+		mediaHash: Hash
+		start: number
+		duration: number
+		spatialId?: Id
+	}
+
+	export type Audio = {
+		id: Id
+		kind: Kind.Audio
 		mediaHash: Hash
 		start: number
 		duration: number
@@ -38,6 +73,9 @@ export namespace Item {
 		id: Id
 		kind: Kind.Text
 		content: string
+		duration: number
+		spatialId?: Id
+		styleId?: Id
 	}
 
 	export type Transition = {
@@ -50,9 +88,13 @@ export namespace Item {
 	export type Any = (
 		| Sequence
 		| Stack
-		| Clip
+		| Video
+		| Audio
 		| Text
+		| Gap
 		| Transition
+		| Spatial
+		| TextStyle
 	)
 }
 

package/s/timeline/parts/media.ts

@@ -1,14 +1,35 @@
 
+import {ALL_FORMATS, Input} from "mediabunny"
+
 import {Datafile} from "../utils/datafile.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../driver/utils/load-decoder-source.js"
 
 export class Media {
 	duration = 0
+	hasVideo = false
+	hasAudio = false
+
 	constructor(public datafile: Datafile) {}
 
 	static async analyze(datafile: Datafile) {
 		const media = new this(datafile)
 		media.duration = 10
+		const {video, audio} = await this.#has("/assets/temp/gl.mp4")
+		media.hasAudio = audio
+		media.hasVideo = video
 		return media
 	}
+
+	static async #has(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		return {
+			audio: !!(await input.getPrimaryAudioTrack()),
+			video: !!(await input.getPrimaryVideoTrack())
+		}
+	}
 }
 

package/s/timeline/parts/waveform.ts

@@ -1,6 +1,6 @@
 import WaveSurfer from "wavesurfer.js"
 
-import {context} from "../../context.js"
+import {Driver} from "../../driver/driver.js"
 import {DecoderSource} from "../../driver/fns/schematic.js"
 
 export class Waveform {
@@ -19,9 +19,8 @@ export class Waveform {
 		})
 	}
 
-	static async init(source: DecoderSource, container: HTMLElement) {
-		const driver = await context.driver
-		const reader = driver.decode({ source }).audio.getReader()
+	static async init(driver: Driver, source: DecoderSource, container: HTMLElement) {
+		const reader = driver.decodeAudio({source}).getReader()
 
 		const peaks: number[] = []
 		let buffer: number[] = []