@omnimedia/omnitool 1.1.0-8 → 1.1.0-81

package/s/timeline/parts/compositor/parts/tree-builder.ts

@@ -1,184 +0,0 @@
-import {Item, Kind} from "../../item.js"
-import {ImageLayer, Layer} from "../../../../driver/fns/schematic.js"
-import {I6, Mat6, mul6, transformToMat6} from "../../../utils/matrix.js"
-
-export type AudioStreamComponent = {
-	getStream: () => AsyncGenerator<AudioData>
-}
-export type AudioPlaybackComponent = {
-	onTimeUpdate: (time: number) => void
-}
-export type VisualComponent = {
-	sampleAt: (time: number) => Promise<Layer[]>
-}
-
-export type Node<T> = {
-	duration: number
-	visuals?: VisualComponent
-	audio?: T
-}
-
-interface Sampler<T> {
-	video(item: Item.Video, parentMatrix: Mat6): Promise<Node<T>>
-	audio(item: Item.Audio): Promise<Node<T>>
-	dispose(): Promise<void>
-}
-
-const requireItem = (items: Map<number, Item.Any>, id: number) => items.get(id)!
-export const getWorldMat6 = (
-	items: Map<number, Item.Any>,
-	item: Item.Text | Item.Sequence | Item.Stack | Item.Video,
-	parent?: Mat6
-): Mat6 => {
-	let world = parent ?? I6
-	if (item.spatialId) {
-		const spatial = requireItem(items, item.spatialId) as Item.Spatial
-		const local = transformToMat6(spatial.transform)
-		world = mul6(local, world)
-	}
-	return world
-}
-
-export type WebcodecsSampler = Sampler<AudioStreamComponent>
-export interface HTMLSampler extends Sampler<AudioPlaybackComponent> {
-	setPaused(v: boolean): void
-}
-
-export abstract class TreeBuilder<T> {
-	constructor(protected items: Map<number, Item.Any>, protected sampler: Sampler<T>) {}
-
-	async build(root: Item.Any, parentMatrix?: Mat6): Promise<Node<T>> {
-		switch (root.kind) {
-			case Kind.Video: return this.sampler.video(root, getWorldMat6(this.items, root, parentMatrix))
-			case Kind.Audio: return this.sampler.audio(root)
-			case Kind.Text: {
-				const matrix = getWorldMat6(this.items, root, parentMatrix)
-				return {
-					duration: Infinity,
-					visuals: {
-						sampleAt: async () => [{kind: "text", content: root.content, color: "white", fontSize: 48, matrix}]
-					}
-				}
-			}
-			case Kind.Gap: return {
-				duration: root.duration,
-				visuals: {
-					sampleAt: async () => []
-				}
-			}
-			case Kind.Stack: {
-				const matrix = getWorldMat6(this.items, root, parentMatrix)
-				const children = await Promise.all(root.childrenIds.map(id => this.build(requireItem(this.items, id), matrix)))
-				return this.#composeStack(children)
-			}
-			case Kind.Sequence: {
-				const matrix = getWorldMat6(this.items, root, parentMatrix)
-				return this.#composeSequence(root, matrix)
-			}
-			default: return {duration: 0}
-		}
-	}
-
-	abstract composeAudio_Stack(children: Node<T>[]): T | undefined
-	abstract composeAudio_Sequence(children: Node<T>[]): T | undefined
-
-	// Visual composition is the same for both builders, so it lives here.
-	#composeVisuals_Stack(children: Node<T>[]): VisualComponent {
-		return {
-			sampleAt: async (time) => {
-				const layers = await Promise.all(children.map(c => c.visuals ? c.visuals.sampleAt(time) : Promise.resolve([])))
-				return layers.flat()
-			}
-		}
-	}
-
-	#composeVisuals_Sequence(children: Node<T>[]): VisualComponent {
-		return {
-			sampleAt: async (time) => {
-				let localTime = time
-				for (const child of children) {
-					if (localTime < child.duration) return child.visuals ? child.visuals.sampleAt(localTime) : []
-					localTime -= child.duration
-				}
-				return []
-			}
-		}
-	}
-
-	#composeStack(children: Node<T>[]): Node<T> {
-		const duration = Math.max(0, ...children.map(k => (Number.isFinite(k.duration) ? k.duration : 0)))
-		return {
-			duration,
-			visuals: this.#composeVisuals_Stack(children),
-			audio: this.composeAudio_Stack(children),
-		}
-	}
-
-	async #composeSequence(sequence: Item.Sequence, parentMatrix?: Mat6): Promise<Node<T>> {
-		const childItems = sequence.childrenIds.map(id => requireItem(this.items, id))
-		const children = await this.#processChildren(childItems, parentMatrix)
-		const duration = children.reduce((a, k) => a + k.duration, 0)
-		return {
-			duration,
-			visuals: this.#composeVisuals_Sequence(children),
-			audio: this.composeAudio_Sequence(children),
-		}
-	}
-
-	async #processChildren(childItems: Item.Any[], parentMatrix?: Mat6): Promise<Node<T>[]> {
-		const processedNodes: Node<T>[] = []
-		for (let i = 0; i < childItems.length; i++) {
-			const item = childItems[i]
-
-			if (item.kind !== Kind.Transition) {
-				processedNodes.push(await this.build(item, parentMatrix))
-				continue
-			}
-
-			const outgoingNode = processedNodes.pop()
-			const incomingItem = childItems[i + 1]
-
-			if (!outgoingNode || !incomingItem || incomingItem.kind === Kind.Transition) {
-				if (outgoingNode) processedNodes.push(outgoingNode)
-				continue
-			}
-
-			const incomingNode = await this.build(incomingItem, parentMatrix)
-			const transitionNode = await this.#createTransitionNode(item, outgoingNode, incomingNode)
-			processedNodes.push(transitionNode)
-			i++
-		}
-		return processedNodes
-	}
-
-	async #createTransitionNode(transitionItem: Item.Transition, outgoingNode: Node<T>, incomingNode: Node<T>): Promise<Node<T>> {
-		const overlap = Math.max(0, Math.min(transitionItem.duration, outgoingNode.duration, incomingNode.duration))
-		const start = Math.max(0, outgoingNode.duration - overlap)
-		const combinedDuration = outgoingNode.duration + incomingNode.duration - overlap
-		return {
-			duration: combinedDuration,
-			visuals: {
-				sampleAt: async (t) => {
-					if (!outgoingNode.visuals || !incomingNode.visuals) return []
-					if (t < start) return outgoingNode.visuals.sampleAt(t)
-					if (t < outgoingNode.duration) {
-						const localTime = t - start
-						const progress = overlap > 0 ? (localTime / overlap) : 1
-						const from = await outgoingNode.visuals.sampleAt(t) as ImageLayer[]
-						const to = await incomingNode.visuals.sampleAt(localTime) as ImageLayer[]
-						if(!from[0]?.frame || !to[0]?.frame) return []
-						return [{
-							kind: "transition",
-							name: "circle",
-							progress,
-							from: from[0].frame,
-							to: to[0].frame,
-						}]
-					}
-					return incomingNode.visuals.sampleAt(t - outgoingNode.duration + overlap)
-				}
-			},
-			audio: this.composeAudio_Sequence([outgoingNode, incomingNode])
-		}
-	}
-}

package/s/timeline/parts/compositor/parts/webcodecs-tree.ts

@@ -1,30 +0,0 @@
-import {Item} from "../../item.js"
-import {AudioStreamComponent, Node, WebcodecsSampler, TreeBuilder} from "./tree-builder.js"
-
-class WebCodecsNodeBuilder extends TreeBuilder<AudioStreamComponent> {
-	composeAudio_Stack(children: Node<AudioStreamComponent>[]) {
-		return {
-			getStream: async function*() {
-				for (const child of children) {
-					if (child.audio)
-						yield* child.audio.getStream()
-				}
-			}
-		}
-	}
-	composeAudio_Sequence(children: Node<AudioStreamComponent>[]) {
-		return {
-			getStream: async function*() {
-				for (const child of children) {
-					if (child.audio)
-						yield* child.audio.getStream()
-				}
-			}
-		}
-	}
-}
-
-export function buildWebCodecsNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: WebcodecsSampler) {
-	const builder = new WebCodecsNodeBuilder(items, sampler)
-	return builder.build(root)
-}

package/s/timeline/parts/compositor/playback.ts

@@ -1,81 +0,0 @@
-import {TimelineFile} from "../basics.js"
-import {context} from "../../../context.js"
-import {realtime} from "./parts/schedulers.js"
-import {makeHtmlSampler} from "./samplers/html.js"
-import {buildHTMLNodeTree} from "./parts/html-tree.js"
-import {DecoderSource} from "../../../driver/fns/schematic.js"
-import {AudioPlaybackComponent, HTMLSampler, Node} from "./parts/tree-builder.js"
-
-type ResolveMedia = (hash: string) => DecoderSource
-
-export class VideoPlayer {
-	#controller = realtime(t => this.#tick(t))
-
-	constructor(
-		public canvas: HTMLCanvasElement,
-		private root: Node<AudioPlaybackComponent>,
-		private sampler: HTMLSampler,
-		private resolveMedia: ResolveMedia = _hash => "/assets/temp/gl.mp4"
-	) {
-		this.#controller.setFPS(30)
-	}
-
-	get context() {
-		return this.canvas.getContext("2d")!
-	}
-
-	static async create(timeline: TimelineFile) {
-		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
-		const items = new Map(timeline.items.map(i => [i.id, i]))
-		const sampler = makeHtmlSampler(() => "/assets/temp/gl.mp4")
-		const root = await buildHTMLNodeTree(rootItem, items, sampler)
-		const canvas = document.createElement("canvas")
-		canvas.width = 1920
-		canvas.height = 1080
-		return new this(canvas, root, sampler)
-	}
-
-	async #tick(t: number) {
-		const driver = await context.driver
-		const dur = this.root.duration
-		const tt = t > dur ? dur : t
-		this.root.audio?.onTimeUpdate(tt)
-		for (const layer of await this.root.visuals?.sampleAt(tt) ?? []) {
-			const frame = await driver.composite(layer)
-			this.context.drawImage(frame, 0, 0)
-			frame.close()
-		}
-		if (t >= dur) this.pause()
-	}
-
-	async play() {
-		if (!this.#controller.isPlaying()) {
-			this.sampler.setPaused!(false)
-			this.#controller.play()
-		}
-	}
-
-	pause() {
-		if(this.#controller.isPlaying()) {
-			this.#controller.pause()
-			this.sampler.setPaused!(true)
-		}
-	}
-
-	async seek(time: number) {
-		const driver = await context.driver
-		this.pause()
-		this.#controller.seek(time)
-		this.root.audio?.onTimeUpdate(time)
-		for (const draw of await this.root.visuals?.sampleAt(time) ?? []) {
-			const frame = await driver.composite(draw)
-			this.context.drawImage(frame, 0, 0)
-			frame.close()
-		}
-	}
-
-	setFPS(value: number) {
-		this.#controller.setFPS(value)
-	}
-}
-

package/s/timeline/parts/compositor/samplers/html.ts

@@ -1,115 +0,0 @@
-import {Item} from "../../item.js"
-import {HTMLSampler} from "../parts/tree-builder.js"
-import {DecoderSource} from "../../../../driver/fns/schematic.js"
-
-const toUrl = (src: DecoderSource) => (src instanceof Blob ? URL.createObjectURL(src) : String(src))
-
-export function makeHtmlSampler(resolveMedia: (hash: string) => DecoderSource): HTMLSampler {
-	const videoElements = new Map<number, HTMLVideoElement>()
-	const audioElements = new Map<number, HTMLAudioElement>()
-
-	function getOrCreateVideoElement(clip: Item.Video) {
-		let video = videoElements.get(clip.id)
-		if (!video) {
-			video = document.createElement("video")
-			video.playsInline = true
-			video.muted = true
-			video.preload = "auto"
-			video.crossOrigin = "anonymous"
-			video.src = toUrl(resolveMedia(clip.mediaHash))
-			videoElements.set(clip.id, video)
-		}
-		return video
-	}
-
-	function getOrCreateAudioElement(clip: Item.Audio) {
-		let audio = audioElements.get(clip.id)
-		if (!audio) {
-			audio = document.createElement("audio")
-			audio.preload = "auto"
-			audio.crossOrigin = "anonymous"
-			audio.src = toUrl(resolveMedia(clip.mediaHash))
-			audio.volume = 0.2
-			audioElements.set(clip.id, audio)
-		}
-		return audio
-	}
-
-	let paused = true
-
-	return {
-		async video(item, matrix) {
-			const video = getOrCreateVideoElement(item)
-			return {
-				duration: item.duration,
-				// if paused seek otherwise play
-				visuals: {
-					sampleAt: async (t) => {
-						if (t < 0 || t >= item.duration)
-							return []
-
-						if(video.paused && paused) {
-							await seek(video, t)
-						}
-
-						if(video.paused && !paused) {
-							await video.play()
-						}
-
-						const frame = new VideoFrame(video)
-						return frame ? [{kind: "image", frame, matrix}] : []
-					}
-				}
-			}
-		},
-		async audio(item) {
-			const audio = getOrCreateAudioElement(item)
-			return {
-				duration: item.duration,
-				audio: {
-					onTimeUpdate: async (time) => {
-						const localTime = item.start + time
-						if(audio.paused && paused) {
-							await seek(audio, localTime)
-						}
-						if(audio.paused && !paused) {
-							await audio.play()
-						}
-						return []
-					}
-				}
-			}
-		},
-		async dispose() {
-			const elements = [...videoElements.values(), ...audioElements.values()]
-      for (const element of elements) {
-        element.pause()
-        if (element.src.startsWith("blob:"))
-        	URL.revokeObjectURL(element.src)
-        element.remove()
-      }
-      videoElements.clear()
-      audioElements.clear()
-		},
-		async setPaused(p) {
-			paused = p
-			const elements = [...videoElements.values(), ...audioElements.values()]
-			for(const element of elements) {
-				if(p) element.pause()
-			}
-		},
-	}
-}
-
-function seek(media: HTMLVideoElement | HTMLAudioElement, time: number): Promise<void> {
-  return new Promise((resolve) => {
-    const onSeeked = () => {
-      media.removeEventListener("seeked", onSeeked)
-      resolve()
-    }
-    media.addEventListener("seeked", onSeeked)
-    if(media.fastSeek) {
-    	media.fastSeek(time)
-    } else media.currentTime = time
-  })
-}

package/s/timeline/parts/compositor/samplers/webcodecs.ts

@@ -1,60 +0,0 @@
-import {Item} from "../../item.js"
-import {context} from "../../../../context.js"
-import {WebcodecsSampler} from "../parts/tree-builder.js"
-import {VideoCursor} from "../../../utils/video-cursor.js"
-import {AudioStream} from "../../../utils/audio-stream.js"
-
-const toUs = (seconds: number) => Math.round(seconds * 1_000_000)
-
-export function makeWebCodecsSampler(resolveMedia: (hash: string) => any): WebcodecsSampler {
-	const videoCursors = new Map<number, VideoCursor>()
-
-	async function getCursorForVideo(videoItem: Item.Video): Promise<VideoCursor> {
-		const existing = videoCursors.get(videoItem.id)
-		if (existing) return existing
-		const driver = await context.driver
-		const source = resolveMedia(videoItem.mediaHash)
-		const video = driver.decodeVideo({source})
-		const cursor = new VideoCursor(video.getReader())
-		videoCursors.set(videoItem.id, cursor)
-		return cursor
-	}
-
-	return {
-		async video(item, matrix) {
-			const cursor = await getCursorForVideo(item)
-			const baseUs = toUs(item.start ?? 0)
-			return {
-				duration: item.duration,
-				visuals: {
-					sampleAt: async (time: number) => {
-						const frame = await cursor.atOrNear(baseUs + toUs(time))
-						return frame ? [{kind: "image", frame, matrix}] : []
-					}
-				}
-			}
-		},
-		async audio(item) {
-			return {
-				duration: item.duration,
-				audio: {
-					getStream: async function*() {
-						const driver = await context.driver
-						const source = resolveMedia(item.mediaHash)
-						const startUs = item.start
-						const endUs = (item.start + item.duration)
-						const audio = driver.decodeAudio({source, start: startUs, end: endUs})
-						const audioStream = new AudioStream(audio.getReader())
-						yield* audioStream.stream()
-					},
-				}
-			}
-		},
-  	async dispose() {
-    	const tasks = Array.from([...videoCursors.values()], c => c.cancel())
-    	videoCursors.clear()
-    	await Promise.all(tasks)
-  	}
-	}
-}
-

package/s/timeline/parts/waveform.ts

@@ -1,62 +0,0 @@
-import WaveSurfer from "wavesurfer.js"
-
-import {context} from "../../context.js"
-import {DecoderSource} from "../../driver/fns/schematic.js"
-
-export class Waveform {
-	wavesurfer: WaveSurfer
-
-	constructor(peaks: number[], container: HTMLElement, duration: number) {
-		this.wavesurfer = WaveSurfer.create({
-  		container,
-  		waveColor: 'rgb(200, 0, 200)',
-  		progressColor: 'rgb(100, 0, 100)',
-  		barWidth: 10,
-  		barRadius: 10,
-  		barGap: 2,
-  		peaks: [peaks],
-  		duration
-		})
-	}
-
-	static async init(source: DecoderSource, container: HTMLElement) {
-		const driver = await context.driver
-		const reader = driver.decodeAudio({source}).getReader()
-
-		const peaks: number[] = []
-		let buffer: number[] = []
-		const samplesPerPeak = 1024
-		const duration = await driver.getAudioDuration(source)
-
-		while (true) {
-			const {done, value: audioData} = await reader.read()
-			if (done) break
-
-			const frames = audioData.numberOfFrames
-			const plane = new Float32Array(frames)
-			audioData.copyTo(plane, {planeIndex: 0}) // Use left channel only
-
-			for (let i = 0; i < plane.length; i++) {
-				buffer.push(plane[i])
-				if (buffer.length >= samplesPerPeak) {
-					const chunk = buffer.splice(0, samplesPerPeak)
-					const min = Math.min(...chunk)
-					const max = Math.max(...chunk)
-					peaks.push(min, max)
-				}
-			}
-
-			audioData.close()
-		}
-
-		return new Waveform(peaks, container, duration ?? 0)
-	}
-
-	// set zoom(value: number) {
-	// 	this.wavesurfer.zoom(value)
-	// }
-
-	set width(value: number) {
-		this.wavesurfer.setOptions({width: value})
-	}
-}

package/s/timeline/sugar/builders.ts

@@ -1,102 +0,0 @@
-import {O} from "./o.js"
-import {Id} from "../parts/basics.js"
-import {Item} from "../parts/item.js"
-
-export class TimelineItem {
-	public readonly id: Id
-
-	constructor(public item: Item.Any) {
-		this.id = item.id
-	}
-
-	toJSON() {
-		return {
-			...this.item
-		}
-	}
-}
-
-abstract class VisualItem extends TimelineItem {
-  abstract spatial(spatial: Spatial): TimelineItem
-}
-
-export class Stack extends VisualItem {
-  constructor(private o: O, public item: Item.Stack) {
-  	super(item)
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-
-  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
-  	const result = fn(this.o)
-  	const items = Array.isArray(result) ? result : [result]
-    this.item.childrenIds.push(...items.map(c => c.item.id))
-    return this
-  }
-}
-
-export class Spatial extends TimelineItem {
-	constructor(public item: Item.Spatial) {super(item)}
-}
-
-export class Gap extends TimelineItem {
-	constructor(public item: Item.Gap) {super(item)}
-}
-
-export class Audio extends TimelineItem {
-	constructor(public item: Item.Audio) {super(item)}
-}
-
-export class Video extends VisualItem {
-  constructor(public item: Item.Video) {
-  	super(item)
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-}
-
-export class Text extends VisualItem {
-  constructor(public item: Item.Text) {
-  	super(item)
-  }
-
-  color(color: string) {
-  	this.item.color = color
-		return this
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-}
-
-export class Sequence extends VisualItem {
-	constructor(private o: O, public item: Item.Sequence) {
-		super(item)
-	}
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-
-  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
-  	const result = fn(this.o)
-  	const items = Array.isArray(result) ? result : [result]
-    this.item.childrenIds.push(...items.map(c => c.item.id))
-    return this
-  }
-}
-
-export class Transition extends TimelineItem {
-	constructor(public item: Item.Transition) {
-		super(item)
-	}
-}

package/s/timeline/sugar/omni-test.ts

@@ -1,38 +0,0 @@
-
-import {Omni} from "./omni.js"
-import {dummyData} from "../utils/dummy-data.js"
-
-//
-// create an omni context
-//
-
-const omni = new Omni()
-
-//
-// load in some media resources
-//
-
-const {mediaA, mediaB} = await omni.load({
-	mediaA: dummyData(),
-	mediaB: dummyData(),
-})
-
-//
-// create a timeline
-//
-
-const timeline = omni.timeline(o => o.sequence(
-	o.video(mediaA),
-	o.transition.crossfade(600),
-	o.stack(
-		o.video(mediaB),
-		o.text("hello world"),
-	),
-))
-
-//
-// log the timeline
-//
-
-console.log(JSON.stringify(timeline, undefined, "  "))
-

package/s/timeline/timeline.ts

@@ -1,22 +0,0 @@
-import {TimelineItem} from "./sugar/builders.js"
-import {Id, TimelineFile} from "./parts/basics.js"
-
-export class Timeline {
-  constructor(public root: TimelineItem, private items: Map<Id, TimelineItem>) {}
-
-  require<T extends TimelineItem>(id: Id): T {
-    const item = this.items.get(id)
-    return item as T
-  }
-
-  toJSON(): TimelineFile {
-    return {
-			format: "timeline",
-			info: "https://omniclip.app/",
-			version: 0,
-			rootId: this.root.item.id,
-			items: Array.from(this.items.values()).map(item => item.toJSON())
-    }
-  }
-}
-