@omnimedia/omnitool 1.1.0-4 → 1.1.0-7

package/s/features/transition/parts/uniforms.ts

@@ -0,0 +1,29 @@
+import {GLTransition} from "./types.js"
+
+export const uniforms = {
+	custom: (transition: GLTransition) => Object.fromEntries(
+		Object.entries(transition.defaultParams).map(([name, value]) => [
+			name,
+			{
+				value,
+				type: getUniformType(transition.paramsTypes[name])
+			}
+		])
+	),
+	basics: {
+		_fromR: {value: 1, type: "f32"},
+		_toR: {value: 1, type: "f32"},
+		ratio: {value: 1, type: "f32"},
+		progress: {value: 0, type: "f32"},
+		customUniform: {value: 0, type: "f32"},
+	}
+}
+
+const getUniformType = (type: string) => {
+	if(type === "f32" || type === "i32") {
+		return type
+	} else if(type === "float") {
+		return "f32"
+	}
+	else return `${type}<f32>`
+}

package/s/features/transition/parts/vertex.ts

@@ -0,0 +1,31 @@
+export const vertex = `
+	in vec2 aPosition;
+	varying vec2 _uv;                          // gl-transition
+	uniform mat3 projectionMatrix;
+	uniform vec4 uInputSize;
+	uniform vec4 uOutputFrame;
+	out vec2 vTextureCoord;
+	uniform vec4 uOutputTexture;
+
+	vec4 filterVertexPosition( void )
+	{
+			vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+
+			position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+			position.y = position.y * (2.0*uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+
+			return vec4(position, 0.0, 1.0);
+	}
+
+	vec2 filterTextureCoord( void )
+	{
+	return aPosition * (uOutputFrame.zw * uInputSize.zw);
+	}
+
+	void main(void)
+	{
+	gl_Position = filterVertexPosition();
+	vTextureCoord = filterTextureCoord();
+	_uv = vec2(0.5, 0.5) * (aPosition +vec2(1.0, 1.0));    // gl-transition
+	}
+`

package/s/features/transition/transition.ts

@@ -0,0 +1,60 @@
+//@ts-ignore
+import transitions from "gl-transitions"
+import {Filter, GlProgram, Sprite, Texture, ImageSource} from "pixi.js"
+
+import {vertex} from "./parts/vertex.js"
+import {uniforms} from "./parts/uniforms.js"
+import {fragment} from "./parts/fragment.js"
+import {GLTransition, TransitionOptions, TransitionRendererOptions} from "./parts/types.js"
+
+export function makeTransition({name, renderer}: TransitionOptions) {
+	const transition = transitions.find((t: GLTransition) => t.name === name) as GLTransition
+	const transitionSprite = new Sprite()
+	const transitionTexture = new Texture()
+	const sourceFrom = new ImageSource({})
+	const sourceTo = new ImageSource({})
+
+	const filter = new Filter({
+		glProgram: new GlProgram({
+			vertex,
+			fragment: fragment(transition.glsl),
+		}),
+		resources: {
+			from: sourceFrom,
+			to: sourceTo,
+			uniforms: {
+				...uniforms.basics,
+				...uniforms.custom(transition)
+			}
+		}
+	})
+
+	transitionSprite.filters = [filter]
+
+	return {
+		render({width, height, from, to, progress}: TransitionRendererOptions) {
+			if(transitionSprite.width !== width || transitionSprite.height !== height) {
+				transitionSprite.setSize({width, height})
+				transitionTexture.source.resize(width, height)
+			}
+
+			sourceFrom.resource = from
+			sourceTo.resource = to
+			sourceFrom.update()
+			sourceTo.update()
+
+			filter.resources.uniforms.uniforms.progress = progress
+
+			renderer.render({
+				container: transitionSprite,
+				target: transitionTexture,
+				clear: false,
+				width,
+				height
+			})
+
+			return transitionTexture
+		}
+	}
+}
+

package/s/index.html.ts

@@ -30,7 +30,7 @@ export default ssg.page(import.meta.url, async orb => ({
 		<section>
 			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
 			<button class=fetch>fetch</button>
-			<button class="import">import</button>
+			<input type="file" class="file-input">
 			<div class=results></div>
 			<div class=filmstrip-demo>
 				<label for="viewable-range">viewable range:</label>
@@ -47,6 +47,11 @@ export default ssg.page(import.meta.url, async orb => ({
 				<label for="width">width:</label>
 				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
 			</div>
+			<div class=player>
+				<input class="seek" type="number" min="0">
+				<button class=play>play</button>
+				<button class=stop>stop</button>
+			</div>
 		</section>
 	`,
 }))

package/s/timeline/parts/basics.ts

@@ -11,7 +11,7 @@ export type TimelineFile = {
 	info: "https://omniclip.app/"
 	format: "timeline"
 	version: number
-	root: Id
+	rootId: Id
 	items: Item.Any[]
 }
 

package/s/timeline/parts/compositor/export.ts

@@ -0,0 +1,77 @@
+import {TimelineFile} from "../basics.js"
+import {context} from "../../../context.js"
+import {fixedStep} from "./parts/schedulers.js"
+import {makeWebCodecsSampler} from "./samplers/webcodecs.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {buildWebCodecsNodeTree} from "./parts/webcodecs-tree.js"
+
+export class Export {
+	#sampler
+	constructor(
+		private framerate = 30,
+		private resolveMedia: (hash: string) => DecoderSource = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#sampler = makeWebCodecsSampler(this.resolveMedia)
+	}
+
+	async #build(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		return await buildWebCodecsNodeTree(rootItem, items, this.#sampler)
+	}
+
+	async render(timeline: TimelineFile) {
+		const root = await this.#build(timeline)
+
+		const driver = await context.driver
+		const videoStream = new TransformStream<VideoFrame, VideoFrame>()
+		const audioStream = new TransformStream<AudioData, AudioData>()
+
+		const encodePromise = driver.encode({
+			video: videoStream.readable,
+			audio: audioStream.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000},
+			},
+		})
+
+		const videoWriter = videoStream.writable.getWriter()
+		const audioWriter = audioStream.writable.getWriter()
+
+		const audioPromise = (async () => {
+			if (root.audio) {
+				for await (const chunk of root.audio.getStream()) {
+					await audioWriter.write(chunk)
+				}
+			}
+			await audioWriter.close()
+		})()
+
+		const videoPromise = (async () => {
+			let i = 0
+			const dt = 1 / this.framerate
+
+			await fixedStep(
+				{fps: this.framerate, duration: root.duration ?? 0},
+				async t => {
+					const layers = await root.visuals?.sampleAt(t) ?? []
+					const composed = await driver.composite(layers)
+					const vf = new VideoFrame(composed, {
+						timestamp: Math.round(i * dt * 1_000_000),
+						duration: Math.round(dt * 1_000_000),
+					})
+					await videoWriter.write(vf)
+					composed.close()
+					i++
+				}
+			)
+			await videoWriter.close()
+		})()
+
+		await audioPromise
+		await videoPromise
+		await encodePromise
+		// this.#sampler.dispose()
+	}
+}

package/s/timeline/parts/compositor/parts/html-tree.ts

@@ -0,0 +1,37 @@
+import {Item} from "../../item.js"
+import {AudioPlaybackComponent, HTMLSampler, Node, TreeBuilder} from "./tree-builder.js"
+
+class HTMLNodeBuilder extends TreeBuilder<AudioPlaybackComponent> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: HTMLSampler) {
+		super(items, sampler)
+	}
+
+	composeAudio_Stack(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				for (const child of children) {
+					if (child.audio) child.audio.onTimeUpdate(time)
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) {
+						if (child.audio) child.audio.onTimeUpdate(localTime)
+						break
+					}
+					localTime -= child.duration
+				}
+			}
+		}
+	}
+}
+
+export function buildHTMLNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: HTMLSampler) {
+	const builder = new HTMLNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/parts/schedulers.ts

@@ -0,0 +1,85 @@
+export type RealtimeController = {
+	play(): void
+	pause(): void
+	seek(t: number): void
+	dispose(): void
+	setFPS(v: number): void
+	isPlaying(): boolean
+}
+
+export const realtime = (onTick: (t: number) => void): RealtimeController => {
+  let playing = false
+  let rafId: number | null = null
+  let fps = 60
+
+  let frameDuration = 1000 / fps
+  let currentTimeS = 0
+  let lastTime = 0
+  let accumulator = 0
+
+  const tick = (now: number) => {
+    if (!playing) return
+
+    const deltaTime = now - lastTime
+    lastTime = now
+
+    accumulator += deltaTime
+
+    while (accumulator >= frameDuration) {
+      onTick(currentTimeS)
+      currentTimeS += frameDuration / 1000
+      accumulator -= frameDuration
+    }
+
+    rafId = requestAnimationFrame(tick)
+  }
+
+  return {
+    play() {
+      if (playing) return
+      playing = true
+      lastTime = performance.now()
+      rafId = requestAnimationFrame(tick)
+    },
+    pause() {
+      if (!playing) return
+      playing = false
+      if (rafId !== null) cancelAnimationFrame(rafId)
+      rafId = null
+    },
+    seek(t) {
+      currentTimeS = Math.max(0, t)
+      accumulator = 0
+    },
+    dispose() {
+      this.pause()
+    },
+    isPlaying() {
+      return playing
+    },
+    setFPS(v) {
+    	fps = v
+    	frameDuration = 1000 / fps
+    }
+  }
+}
+
+export type FixedStepOptions = {
+	fps: number
+	duration: number
+	abort?: AbortSignal
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: number, index: number) => Promise<void> | void
+) => {
+	const dt = 1 / opts.fps
+	const total = Math.ceil(opts.duration * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		if (opts.abort?.aborted) break
+		const t = i * dt
+		await onFrame(t, i)
+	}
+}

package/s/timeline/parts/compositor/parts/tree-builder.ts

@@ -0,0 +1,184 @@
+import {Item, Kind} from "../../item.js"
+import {ImageLayer, Layer} from "../../../../driver/fns/schematic.js"
+import {I6, Mat6, mul6, transformToMat6} from "../../../utils/matrix.js"
+
+export type AudioStreamComponent = {
+	getStream: () => AsyncGenerator<AudioData>
+}
+export type AudioPlaybackComponent = {
+	onTimeUpdate: (time: number) => void
+}
+export type VisualComponent = {
+	sampleAt: (time: number) => Promise<Layer[]>
+}
+
+export type Node<T> = {
+	duration: number
+	visuals?: VisualComponent
+	audio?: T
+}
+
+interface Sampler<T> {
+	video(item: Item.Video, parentMatrix: Mat6): Promise<Node<T>>
+	audio(item: Item.Audio): Promise<Node<T>>
+	dispose(): Promise<void>
+}
+
+const requireItem = (items: Map<number, Item.Any>, id: number) => items.get(id)!
+export const getWorldMat6 = (
+	items: Map<number, Item.Any>,
+	item: Item.Text | Item.Sequence | Item.Stack | Item.Video,
+	parent?: Mat6
+): Mat6 => {
+	let world = parent ?? I6
+	if (item.spatialId) {
+		const spatial = requireItem(items, item.spatialId) as Item.Spatial
+		const local = transformToMat6(spatial.transform)
+		world = mul6(local, world)
+	}
+	return world
+}
+
+export type WebcodecsSampler = Sampler<AudioStreamComponent>
+export interface HTMLSampler extends Sampler<AudioPlaybackComponent> {
+	setPaused(v: boolean): void
+}
+
+export abstract class TreeBuilder<T> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: Sampler<T>) {}
+
+	async build(root: Item.Any, parentMatrix?: Mat6): Promise<Node<T>> {
+		switch (root.kind) {
+			case Kind.Video: return this.sampler.video(root, getWorldMat6(this.items, root, parentMatrix))
+			case Kind.Audio: return this.sampler.audio(root)
+			case Kind.Text: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				return {
+					duration: Infinity,
+					visuals: {
+						sampleAt: async () => [{kind: "text", content: root.content, color: "white", fontSize: 48, matrix}]
+					}
+				}
+			}
+			case Kind.Gap: return {
+				duration: root.duration,
+				visuals: {
+					sampleAt: async () => []
+				}
+			}
+			case Kind.Stack: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				const children = await Promise.all(root.childrenIds.map(id => this.build(requireItem(this.items, id), matrix)))
+				return this.#composeStack(children)
+			}
+			case Kind.Sequence: {
+				const matrix = getWorldMat6(this.items, root, parentMatrix)
+				return this.#composeSequence(root, matrix)
+			}
+			default: return {duration: 0}
+		}
+	}
+
+	abstract composeAudio_Stack(children: Node<T>[]): T | undefined
+	abstract composeAudio_Sequence(children: Node<T>[]): T | undefined
+
+	// Visual composition is the same for both builders, so it lives here.
+	#composeVisuals_Stack(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				const layers = await Promise.all(children.map(c => c.visuals ? c.visuals.sampleAt(time) : Promise.resolve([])))
+				return layers.flat()
+			}
+		}
+	}
+
+	#composeVisuals_Sequence(children: Node<T>[]): VisualComponent {
+		return {
+			sampleAt: async (time) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) return child.visuals ? child.visuals.sampleAt(localTime) : []
+					localTime -= child.duration
+				}
+				return []
+			}
+		}
+	}
+
+	#composeStack(children: Node<T>[]): Node<T> {
+		const duration = Math.max(0, ...children.map(k => (Number.isFinite(k.duration) ? k.duration : 0)))
+		return {
+			duration,
+			visuals: this.#composeVisuals_Stack(children),
+			audio: this.composeAudio_Stack(children),
+		}
+	}
+
+	async #composeSequence(sequence: Item.Sequence, parentMatrix?: Mat6): Promise<Node<T>> {
+		const childItems = sequence.childrenIds.map(id => requireItem(this.items, id))
+		const children = await this.#processChildren(childItems, parentMatrix)
+		const duration = children.reduce((a, k) => a + k.duration, 0)
+		return {
+			duration,
+			visuals: this.#composeVisuals_Sequence(children),
+			audio: this.composeAudio_Sequence(children),
+		}
+	}
+
+	async #processChildren(childItems: Item.Any[], parentMatrix?: Mat6): Promise<Node<T>[]> {
+		const processedNodes: Node<T>[] = []
+		for (let i = 0; i < childItems.length; i++) {
+			const item = childItems[i]
+
+			if (item.kind !== Kind.Transition) {
+				processedNodes.push(await this.build(item, parentMatrix))
+				continue
+			}
+
+			const outgoingNode = processedNodes.pop()
+			const incomingItem = childItems[i + 1]
+
+			if (!outgoingNode || !incomingItem || incomingItem.kind === Kind.Transition) {
+				if (outgoingNode) processedNodes.push(outgoingNode)
+				continue
+			}
+
+			const incomingNode = await this.build(incomingItem, parentMatrix)
+			const transitionNode = await this.#createTransitionNode(item, outgoingNode, incomingNode)
+			processedNodes.push(transitionNode)
+			i++
+		}
+		return processedNodes
+	}
+
+	async #createTransitionNode(transitionItem: Item.Transition, outgoingNode: Node<T>, incomingNode: Node<T>): Promise<Node<T>> {
+		const overlap = Math.max(0, Math.min(transitionItem.duration, outgoingNode.duration, incomingNode.duration))
+		const start = Math.max(0, outgoingNode.duration - overlap)
+		const combinedDuration = outgoingNode.duration + incomingNode.duration - overlap
+		return {
+			duration: combinedDuration,
+			visuals: {
+				sampleAt: async (t) => {
+					if (!outgoingNode.visuals || !incomingNode.visuals) return []
+					if (t < start) return outgoingNode.visuals.sampleAt(t)
+					if (t < outgoingNode.duration) {
+						const localTime = t - start
+						const progress = overlap > 0 ? (localTime / overlap) : 1
+						const from = await outgoingNode.visuals.sampleAt(t) as ImageLayer[]
+						const to = await incomingNode.visuals.sampleAt(localTime) as ImageLayer[]
+						if(!from[0]?.frame || !to[0]?.frame) return []
+						return [{
+							kind: "transition",
+							name: "circle",
+							progress,
+							from: from[0].frame,
+							to: to[0].frame,
+						}]
+					}
+					return incomingNode.visuals.sampleAt(t - outgoingNode.duration + overlap)
+				}
+			},
+			audio: this.composeAudio_Sequence([outgoingNode, incomingNode])
+		}
+	}
+}

package/s/timeline/parts/compositor/parts/webcodecs-tree.ts

@@ -0,0 +1,30 @@
+import {Item} from "../../item.js"
+import {AudioStreamComponent, Node, WebcodecsSampler, TreeBuilder} from "./tree-builder.js"
+
+class WebCodecsNodeBuilder extends TreeBuilder<AudioStreamComponent> {
+	composeAudio_Stack(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioStreamComponent>[]) {
+		return {
+			getStream: async function*() {
+				for (const child of children) {
+					if (child.audio)
+						yield* child.audio.getStream()
+				}
+			}
+		}
+	}
+}
+
+export function buildWebCodecsNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: WebcodecsSampler) {
+	const builder = new WebCodecsNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/playback.ts

@@ -0,0 +1,81 @@
+import {TimelineFile} from "../basics.js"
+import {context} from "../../../context.js"
+import {realtime} from "./parts/schedulers.js"
+import {makeHtmlSampler} from "./samplers/html.js"
+import {buildHTMLNodeTree} from "./parts/html-tree.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {AudioPlaybackComponent, HTMLSampler, Node} from "./parts/tree-builder.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	#controller = realtime(t => this.#tick(t))
+
+	constructor(
+		public canvas: HTMLCanvasElement,
+		private root: Node<AudioPlaybackComponent>,
+		private sampler: HTMLSampler,
+		private resolveMedia: ResolveMedia = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#controller.setFPS(30)
+	}
+
+	get context() {
+		return this.canvas.getContext("2d")!
+	}
+
+	static async create(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		const sampler = makeHtmlSampler(() => "/assets/temp/gl.mp4")
+		const root = await buildHTMLNodeTree(rootItem, items, sampler)
+		const canvas = document.createElement("canvas")
+		canvas.width = 1920
+		canvas.height = 1080
+		return new this(canvas, root, sampler)
+	}
+
+	async #tick(t: number) {
+		const driver = await context.driver
+		const dur = this.root.duration
+		const tt = t > dur ? dur : t
+		this.root.audio?.onTimeUpdate(tt)
+		for (const layer of await this.root.visuals?.sampleAt(tt) ?? []) {
+			const frame = await driver.composite(layer)
+			this.context.drawImage(frame, 0, 0)
+			frame.close()
+		}
+		if (t >= dur) this.pause()
+	}
+
+	async play() {
+		if (!this.#controller.isPlaying()) {
+			this.sampler.setPaused!(false)
+			this.#controller.play()
+		}
+	}
+
+	pause() {
+		if(this.#controller.isPlaying()) {
+			this.#controller.pause()
+			this.sampler.setPaused!(true)
+		}
+	}
+
+	async seek(time: number) {
+		const driver = await context.driver
+		this.pause()
+		this.#controller.seek(time)
+		this.root.audio?.onTimeUpdate(time)
+		for (const draw of await this.root.visuals?.sampleAt(time) ?? []) {
+			const frame = await driver.composite(draw)
+			this.context.drawImage(frame, 0, 0)
+			frame.close()
+		}
+	}
+
+	setFPS(value: number) {
+		this.#controller.setFPS(value)
+	}
+}
+