@omnimedia/omnitool 1.0.0 → 1.1.0-10

package/s/timeline/parts/compositor/export.ts

@@ -0,0 +1,77 @@
+import {TimelineFile} from "../basics.js"
+import {context} from "../../../context.js"
+import {fixedStep} from "./parts/schedulers.js"
+import {makeWebCodecsSampler} from "./samplers/webcodecs.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {buildWebCodecsNodeTree} from "./parts/webcodecs-tree.js"
+
+export class Export {
+	#sampler
+	constructor(
+		private framerate = 30,
+		private resolveMedia: (hash: string) => DecoderSource = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#sampler = makeWebCodecsSampler(this.resolveMedia)
+	}
+
+	async #build(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		return await buildWebCodecsNodeTree(rootItem, items, this.#sampler)
+	}
+
+	async render(timeline: TimelineFile) {
+		const root = await this.#build(timeline)
+
+		const driver = await context.driver
+		const videoStream = new TransformStream<VideoFrame, VideoFrame>()
+		const audioStream = new TransformStream<AudioData, AudioData>()
+
+		const encodePromise = driver.encode({
+			video: videoStream.readable,
+			audio: audioStream.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000},
+			},
+		})
+
+		const videoWriter = videoStream.writable.getWriter()
+		const audioWriter = audioStream.writable.getWriter()
+
+		const audioPromise = (async () => {
+			if (root.audio) {
+				for await (const chunk of root.audio.getStream()) {
+					await audioWriter.write(chunk)
+				}
+			}
+			await audioWriter.close()
+		})()
+
+		const videoPromise = (async () => {
+			let i = 0
+			const dt = 1 / this.framerate
+
+			await fixedStep(
+				{fps: this.framerate, duration: root.duration ?? 0},
+				async t => {
+					const layers = await root.visuals?.sampleAt(t) ?? []
+					const composed = await driver.composite(layers)
+					const vf = new VideoFrame(composed, {
+						timestamp: Math.round(i * dt * 1_000_000),
+						duration: Math.round(dt * 1_000_000),
+					})
+					await videoWriter.write(vf)
+					composed.close()
+					i++
+				}
+			)
+			await videoWriter.close()
+		})()
+
+		await audioPromise
+		await videoPromise
+		await encodePromise
+		// this.#sampler.dispose()
+	}
+}

package/s/timeline/parts/compositor/parts/html-tree.ts

@@ -0,0 +1,37 @@
+import {Item} from "../../item.js"
+import {AudioPlaybackComponent, HTMLSampler, Node, TreeBuilder} from "./tree-builder.js"
+
+class HTMLNodeBuilder extends TreeBuilder<AudioPlaybackComponent> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: HTMLSampler) {
+		super(items, sampler)
+	}
+
+	composeAudio_Stack(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				for (const child of children) {
+					if (child.audio) child.audio.onTimeUpdate(time)
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) {
+						if (child.audio) child.audio.onTimeUpdate(localTime)
+						break
+					}
+					localTime -= child.duration
+				}
+			}
+		}
+	}
+}
+
+export function buildHTMLNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: HTMLSampler) {
+	const builder = new HTMLNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/parts/schedulers.ts

@@ -0,0 +1,85 @@
+export type RealtimeController = {
+	play(): void
+	pause(): void
+	seek(t: number): void
+	dispose(): void
+	setFPS(v: number): void
+	isPlaying(): boolean
+}
+
+export const realtime = (onTick: (t: number) => void): RealtimeController => {
+  let playing = false
+  let rafId: number | null = null
+  let fps = 60
+
+  let frameDuration = 1000 / fps
+  let currentTimeS = 0
+  let lastTime = 0
+  let accumulator = 0
+
+  const tick = (now: number) => {
+    if (!playing) return
+
+    const deltaTime = now - lastTime
+    lastTime = now
+
+    accumulator += deltaTime
+
+    while (accumulator >= frameDuration) {
+      onTick(currentTimeS)
+      currentTimeS += frameDuration / 1000
+      accumulator -= frameDuration
+    }
+
+    rafId = requestAnimationFrame(tick)
+  }
+
+  return {
+    play() {
+      if (playing) return
+      playing = true
+      lastTime = performance.now()
+      rafId = requestAnimationFrame(tick)
+    },
+    pause() {
+      if (!playing) return
+      playing = false
+      if (rafId !== null) cancelAnimationFrame(rafId)
+      rafId = null
+    },
+    seek(t) {
+      currentTimeS = Math.max(0, t)
+      accumulator = 0
+    },
+    dispose() {
+      this.pause()
+    },
+    isPlaying() {
+      return playing
+    },
+    setFPS(v) {
+    	fps = v
+    	frameDuration = 1000 / fps
+    }
+  }
+}
+
+export type FixedStepOptions = {
+	fps: number
+	duration: number
+	abort?: AbortSignal
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: number, index: number) => Promise<void> | void
+) => {
+	const dt = 1 / opts.fps
+	const total = Math.ceil(opts.duration * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		if (opts.abort?.aborted) break
+		const t = i * dt
+		await onFrame(t, i)
+	}
+}

package/s/timeline/parts/compositor/parts/tree-builder.ts

@@ -0,0 +1,184 @@
+import {Item, Kind} from "../../item.js"
+import {ImageLayer, Layer} from "../../../../driver/fns/schematic.js"
+import {I6, Mat6, mul6, transformToMat6} from "../../../utils/matrix.js"
+
+export type AudioStreamComponent = {
+	getStream: () => AsyncGenerator<AudioData>
+}
+export type AudioPlaybackComponent = {
+	onTimeUpdate: (time: number) => void
+}
+export type VisualComponent = {
+	sampleAt: (time: number) => Promise<Layer[]>
+}
+
+export type Node<T> = {
+	duration: number
+	visuals?: VisualComponent
+	audio?: T
+}
+
+interface Sampler<T> {
+	video(item: Item.Video, parentMatrix: Mat6): Promise<Node<T>>
+	audio(item: Item.Audio): Promise<Node<T>>
+	dispose(): Promise<void>
+}
+
+const requireItem = (items: Map<number, Item.Any>, id: number) => items.get(id)!
+export const getWorldMat6 = (
+	items: Map<number, Item.Any>,
+	item: Item.Text | Item.Sequence | Item.Stack | Item.Video,
+	parent?: Mat6
+): Mat6 => {
+	let world = parent ?? I6
+	if (item.spatialId) {
+		const spatial = requireItem(items, item.spatialId) as Item.Spatial
+		const local = transformToMat6(spatial.transform)
+		world = mul6(local, world)
+	}
+	return world
+}
+
+export type WebcodecsSampler = Sampler<AudioStreamComponent>
+export interface HTMLSampler extends Sampler<AudioPlaybackComponent> {
+	setPaused(v: boolean): void
+}
+
+export abstract class TreeBuilder<T> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: Sampler<T>) {}
+
+	async build(root: Item.Any, parentMatrix?: Mat6): Promise<Node<T>> {
+		switch (root.kind) {
+			case Kind.Video: return this.sampler.video(root, getWorldMat6(this.items, root, parentMatrix))
+			case Kind.Audio: return this.sampler.audio(root)
+			case Kind.Text: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				return {
+					duration: Infinity,
+					visuals: {
+						sampleAt: async () => [{kind: "text", content: root.content, color: "white", fontSize: 48, matrix}]
+					}
+				}
+			}
+			case Kind.Gap: return {
+				duration: root.duration,
+				visuals: {
+					sampleAt: async () => []
+				}
+			}
+			case Kind.Stack: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				const children = await Promise.all(root.childrenIds.map(id => this.build(requireItem(this.items, id), matrix)))
+				return this.#composeStack(children)
+			}
+			case Kind.Sequence: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				return this.#composeSequence(root, matrix)
+			}
+			default: return {duration: 0}
+		}
+	}
+
+	abstract composeAudio_Stack(children: Node<T>[]): T | undefined
+	abstract composeAudio_Sequence(children: Node<T>[]): T | undefined
+
+	// Visual composition is the same for both builders, so it lives here.
+	#composeVisuals_Stack(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				const layers = await Promise.all(children.map(c => c.visuals ? c.visuals.sampleAt(time) : Promise.resolve([])))
+				return layers.flat()
+			}
+		}
+	}
+
+	#composeVisuals_Sequence(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) return child.visuals ? child.visuals.sampleAt(localTime) : []
+					localTime -= child.duration
+				}
+				return []
+			}
+		}
+	}
+
+	#composeStack(children: Node<T>[]): Node<T> {
+		const duration = Math.max(0, ...children.map(k => (Number.isFinite(k.duration) ? k.duration : 0)))
+		return {
+			duration,
+			visuals: this.#composeVisuals_Stack(children),
+			audio: this.composeAudio_Stack(children),
+		}
+	}
+
+	async #composeSequence(sequence: Item.Sequence, parentMatrix?: Mat6): Promise<Node<T>> {
+		const childItems = sequence.childrenIds.map(id => requireItem(this.items, id))
+		const children = await this.#processChildren(childItems, parentMatrix)
+		const duration = children.reduce((a, k) => a + k.duration, 0)
+		return {
+			duration,
+			visuals: this.#composeVisuals_Sequence(children),
+			audio: this.composeAudio_Sequence(children),
+		}
+	}
+
+	async #processChildren(childItems: Item.Any[], parentMatrix?: Mat6): Promise<Node<T>[]> {
+		const processedNodes: Node<T>[] = []
+		for (let i = 0; i < childItems.length; i++) {
+			const item = childItems[i]
+
+			if (item.kind !== Kind.Transition) {
+				processedNodes.push(await this.build(item, parentMatrix))
+				continue
+			}
+
+			const outgoingNode = processedNodes.pop()
+			const incomingItem = childItems[i + 1]
+
+			if (!outgoingNode || !incomingItem || incomingItem.kind === Kind.Transition) {
+				if (outgoingNode) processedNodes.push(outgoingNode)
+				continue
+			}
+
+			const incomingNode = await this.build(incomingItem, parentMatrix)
+			const transitionNode = await this.#createTransitionNode(item, outgoingNode, incomingNode)
+			processedNodes.push(transitionNode)
+			i++
+		}
+		return processedNodes
+	}
+
+	async #createTransitionNode(transitionItem: Item.Transition, outgoingNode: Node<T>, incomingNode: Node<T>): Promise<Node<T>> {
+		const overlap = Math.max(0, Math.min(transitionItem.duration, outgoingNode.duration, incomingNode.duration))
+		const start = Math.max(0, outgoingNode.duration - overlap)
+		const combinedDuration = outgoingNode.duration + incomingNode.duration - overlap
+		return {
+			duration: combinedDuration,
+			visuals: {
+				sampleAt: async (t) => {
+					if (!outgoingNode.visuals || !incomingNode.visuals) return []
+					if (t < start) return outgoingNode.visuals.sampleAt(t)
+					if (t < outgoingNode.duration) {
+						const localTime = t - start
+						const progress = overlap > 0 ? (localTime / overlap) : 1
+						const from = await outgoingNode.visuals.sampleAt(t) as ImageLayer[]
+						const to = await incomingNode.visuals.sampleAt(localTime) as ImageLayer[]
+						if(!from[0]?.frame || !to[0]?.frame) return []
+						return [{
+							kind: "transition",
+							name: "circle",
+							progress,
+							from: from[0].frame,
+							to: to[0].frame,
+						}]
+					}
+					return incomingNode.visuals.sampleAt(t - outgoingNode.duration + overlap)
+				}
+			},
+			audio: this.composeAudio_Sequence([outgoingNode, incomingNode])
+		}
+	}
+}

package/s/timeline/parts/compositor/parts/webcodecs-tree.ts

@@ -0,0 +1,30 @@
+import {Item} from "../../item.js"
+import {AudioStreamComponent, Node, WebcodecsSampler, TreeBuilder} from "./tree-builder.js"
+
+class WebCodecsNodeBuilder extends TreeBuilder<AudioStreamComponent> {
+	composeAudio_Stack(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+}
+
+export function buildWebCodecsNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: WebcodecsSampler) {
+	const builder = new WebCodecsNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/playback.ts

@@ -0,0 +1,81 @@
+import {TimelineFile} from "../basics.js"
+import {context} from "../../../context.js"
+import {realtime} from "./parts/schedulers.js"
+import {makeHtmlSampler} from "./samplers/html.js"
+import {buildHTMLNodeTree} from "./parts/html-tree.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {AudioPlaybackComponent, HTMLSampler, Node} from "./parts/tree-builder.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	#controller = realtime(t => this.#tick(t))
+
+	constructor(
+		public canvas: HTMLCanvasElement,
+		private root: Node<AudioPlaybackComponent>,
+		private sampler: HTMLSampler,
+		private resolveMedia: ResolveMedia = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#controller.setFPS(30)
+	}
+
+	get context() {
+		return this.canvas.getContext("2d")!
+	}
+
+	static async create(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		const sampler = makeHtmlSampler(() => "/assets/temp/gl.mp4")
+		const root = await buildHTMLNodeTree(rootItem, items, sampler)
+		const canvas = document.createElement("canvas")
+		canvas.width = 1920
+		canvas.height = 1080
+		return new this(canvas, root, sampler)
+	}
+
+	async #tick(t: number) {
+		const driver = await context.driver
+		const dur = this.root.duration
+		const tt = t > dur ? dur : t
+		this.root.audio?.onTimeUpdate(tt)
+		for (const layer of await this.root.visuals?.sampleAt(tt) ?? []) {
+			const frame = await driver.composite(layer)
+			this.context.drawImage(frame, 0, 0)
+			frame.close()
+		}
+		if (t >= dur) this.pause()
+	}
+
+	async play() {
+		if (!this.#controller.isPlaying()) {
+			this.sampler.setPaused!(false)
+			this.#controller.play()
+		}
+	}
+
+	pause() {
+		if(this.#controller.isPlaying()) {
+			this.#controller.pause()
+			this.sampler.setPaused!(true)
+		}
+	}
+
+	async seek(time: number) {
+		const driver = await context.driver
+		this.pause()
+		this.#controller.seek(time)
+		this.root.audio?.onTimeUpdate(time)
+		for (const draw of await this.root.visuals?.sampleAt(time) ?? []) {
+			const frame = await driver.composite(draw)
+			this.context.drawImage(frame, 0, 0)
+			frame.close()
+		}
+	}
+
+	setFPS(value: number) {
+		this.#controller.setFPS(value)
+	}
+}
+

package/s/timeline/parts/compositor/samplers/html.ts

@@ -0,0 +1,115 @@
+import {Item} from "../../item.js"
+import {HTMLSampler} from "../parts/tree-builder.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+const toUrl = (src: DecoderSource) => (src instanceof Blob ? URL.createObjectURL(src) : String(src))
+
+export function makeHtmlSampler(resolveMedia: (hash: string) => DecoderSource): HTMLSampler {
+	const videoElements = new Map<number, HTMLVideoElement>()
+	const audioElements = new Map<number, HTMLAudioElement>()
+
+	function getOrCreateVideoElement(clip: Item.Video) {
+		let video = videoElements.get(clip.id)
+		if (!video) {
+			video = document.createElement("video")
+			video.playsInline = true
+			video.muted = true
+			video.preload = "auto"
+			video.crossOrigin = "anonymous"
+			video.src = toUrl(resolveMedia(clip.mediaHash))
+			videoElements.set(clip.id, video)
+		}
+		return video
+	}
+
+	function getOrCreateAudioElement(clip: Item.Audio) {
+		let audio = audioElements.get(clip.id)
+		if (!audio) {
+			audio = document.createElement("audio")
+			audio.preload = "auto"
+			audio.crossOrigin = "anonymous"
+			audio.src = toUrl(resolveMedia(clip.mediaHash))
+			audio.volume = 0.2
+			audioElements.set(clip.id, audio)
+		}
+		return audio
+	}
+
+	let paused = true
+
+	return {
+		async video(item, matrix) {
+			const video = getOrCreateVideoElement(item)
+			return {
+				duration: item.duration,
+				// if paused seek otherwise play
+				visuals: {
+					sampleAt: async (t) => {
+						if (t < 0 || t >= item.duration)
+							return []
+
+						if(video.paused && paused) {
+							await seek(video, t)
+						}
+
+						if(video.paused && !paused) {
+							await video.play()
+						}
+
+						const frame = new VideoFrame(video)
+						return frame ? [{kind: "image", frame, matrix}] : []
+					}
+				}
+			}
+		},
+		async audio(item) {
+			const audio = getOrCreateAudioElement(item)
+			return {
+				duration: item.duration,
+				audio: {
+					onTimeUpdate: async (time) => {
+						const localTime = item.start + time
+						if(audio.paused && paused) {
+							await seek(audio, localTime)
+						}
+						if(audio.paused && !paused) {
+							await audio.play()
+						}
+						return []
+					}
+				}
+			}
+		},
+		async dispose() {
+			const elements = [...videoElements.values(), ...audioElements.values()]
+      for (const element of elements) {
+        element.pause()
+        if (element.src.startsWith("blob:"))
+        	URL.revokeObjectURL(element.src)
+        element.remove()
+      }
+      videoElements.clear()
+      audioElements.clear()
+		},
+		async setPaused(p) {
+			paused = p
+			const elements = [...videoElements.values(), ...audioElements.values()]
+			for(const element of elements) {
+				if(p) element.pause()
+			}
+		},
+	}
+}
+
+function seek(media: HTMLVideoElement | HTMLAudioElement, time: number): Promise<void> {
+  return new Promise((resolve) => {
+    const onSeeked = () => {
+      media.removeEventListener("seeked", onSeeked)
+      resolve()
+    }
+    media.addEventListener("seeked", onSeeked)
+    if(media.fastSeek) {
+    	media.fastSeek(time)
+    } else media.currentTime = time
+  })
+}

package/s/timeline/parts/compositor/samplers/webcodecs.ts

@@ -0,0 +1,60 @@
+import {Item} from "../../item.js"
+import {context} from "../../../../context.js"
+import {WebcodecsSampler} from "../parts/tree-builder.js"
+import {VideoCursor} from "../../../utils/video-cursor.js"
+import {AudioStream} from "../../../utils/audio-stream.js"
+
+const toUs = (seconds: number) => Math.round(seconds * 1_000_000)
+
+export function makeWebCodecsSampler(resolveMedia: (hash: string) => any): WebcodecsSampler {
+	const videoCursors = new Map<number, VideoCursor>()
+
+	async function getCursorForVideo(videoItem: Item.Video): Promise<VideoCursor> {
+		const existing = videoCursors.get(videoItem.id)
+		if (existing) return existing
+		const driver = await context.driver
+		const source = resolveMedia(videoItem.mediaHash)
+		const video = driver.decodeVideo({source})
+		const cursor = new VideoCursor(video.getReader())
+		videoCursors.set(videoItem.id, cursor)
+		return cursor
+	}
+
+	return {
+		async video(item, matrix) {
+			const cursor = await getCursorForVideo(item)
+			const baseUs = toUs(item.start ?? 0)
+			return {
+				duration: item.duration,
+				visuals: {
+					sampleAt: async (time: number) => {
+						const frame = await cursor.atOrNear(baseUs + toUs(time))
+						return frame ? [{kind: "image", frame, matrix}] : []
+					}
+				}
+			}
+		},
+		async audio(item) {
+			return {
+				duration: item.duration,
+				audio: {
+					getStream: async function*() {
+						const driver = await context.driver
+						const source = resolveMedia(item.mediaHash)
+						const startUs = item.start
+						const endUs = (item.start + item.duration)
+						const audio = driver.decodeAudio({source, start: startUs, end: endUs})
+						const audioStream = new AudioStream(audio.getReader())
+						yield* audioStream.stream()
+					},
+				}
+			}
+		},
+  	async dispose() {
+    	const tasks = Array.from([...videoCursors.values()], c => c.cancel())
+    	videoCursors.clear()
+    	await Promise.all(tasks)
+  	}
+	}
+}
+