@omnimedia/omnitool 1.1.0-4 → 1.1.0-40

package/s/driver/fns/work.ts

@@ -1,88 +1,86 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
 import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+
 export const setupDriverWork = (
 	Comrade.work<DriverSchematic>(shell => ({
 		async hello() {
 			await shell.host.world()
 		},
 
-		async decode({source, video, audio}) {
-			const loadSource = async () => {
-				if(source instanceof Blob) {
-					return new BlobSource(source)
-				} else {
-					return new UrlSource(source)
+		async decodeAudio({source, audio, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
+
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
 				}
+				await audioWriter.close()
 			}
+		},
+
+		async decodeVideo({source, video, start, end}) {
 			const input = new Input({
-				source: await loadSource(),
+				source: await loadSource(source),
 				formats: ALL_FORMATS
 			})
 
-			const [videoTrack, audioTrack] = await Promise.all([
-				input.getPrimaryVideoTrack(),
-				input.getPrimaryAudioTrack()
-			])
-
+			const videoTrack = await input.getPrimaryVideoTrack()
 			const videoDecodable = await videoTrack?.canDecode()
-			const audioDecodable = await audioTrack?.canDecode()
-
 			const videoWriter = video.getWriter()
-			const audioWriter = audio.getWriter()
 
-			await Promise.all([
-				(async () => {
-					if (videoDecodable && videoTrack) {
-						const sink = new VideoSampleSink(videoTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toVideoFrame()
-							await videoWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await videoWriter.close()
-					}
-				})(),
-				(async () => {
-					if (audioDecodable && audioTrack) {
-						const sink = new AudioSampleSink(audioTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toAudioData()
-							await audioWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await audioWriter.close()
-					}
-				})()
-			])
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
 		},
 
-		async encode({readables, config, bridge}) {
+		async encode({video, audio, config, bridge}) {
 			const output = new Output({
 				format: new Mp4OutputFormat(),
 				target: new StreamTarget(bridge, {chunked: true})
 			})
-			const videoSource = new VideoSampleSource(config.video)
-			output.addVideoTrack(videoSource)
 			// since AudioSample is not transferable it fails to transfer encoder bitrate config
 			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-			const audioSource = new AudioSampleSource(config.audio)
-			output.addAudioTrack(audioSource)
 
-			await output.start()
+			const promises = []
 
-			const videoReader = readables.video.getReader()
-			const audioReader = readables.audio.getReader()
-
-			await Promise.all([
-				(async () => {
+			if(video) {
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				const videoReader = video.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await videoReader.read()
 						if (done) break
@@ -90,8 +88,14 @@ export const setupDriverWork = (
 						await videoSource.add(sample)
 						sample.close()
 					}
-				})(),
-				(async () => {
+				})())
+			}
+
+			if(audio) {
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				const audioReader = audio.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await audioReader.read()
 						if (done) break
@@ -100,9 +104,11 @@ export const setupDriverWork = (
 						sample.close()
 						value.close()
 					}
-				})()
-			])
+				})())
+			}
 
+			await output.start()
+			await Promise.all(promises)
 			await output.finalize()
 		},
 
@@ -110,22 +116,18 @@ export const setupDriverWork = (
 			const {stage, renderer} = await renderPIXI(1920, 1080)
 			stage.removeChildren()
 
-			const {baseFrame, disposables} = await renderLayer(composition, stage)
+			const {dispose} = await renderLayer(composition, stage)
 			renderer.render(stage)
 
 			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
 			// if its very slow on eg edge try chrome
 			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: baseFrame?.timestamp,
-				duration: baseFrame?.duration ?? undefined,
+				timestamp: 0,
+				duration: 0,
 			})
 
-			baseFrame?.close()
 			renderer.clear()
-
-			for (const disposable of disposables) {
-				disposable.destroy(true)
-			}
+			dispose()
 
 			shell.transfer = [frame]
 			return frame
@@ -157,78 +159,88 @@ async function renderPIXI(width: number, height: number) {
 	return pixi
 }
 
+const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+
 type RenderableObject = Sprite | Text | Texture
 
 async function renderLayer(
 	layer: Layer | Composition,
 	parent: Container,
-	disposables: RenderableObject[] = []
 ) {
 	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
+		layer.reverse()
+		const disposers: (() => void)[] = []
 		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
+			const result = await renderLayer(child, parent)
+			disposers.push(result.dispose)
 		}
-		return {baseFrame, disposables}
-	}
-
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
+		return {dispose: () => disposers.forEach(d => d())}
 	}
 
 	switch (layer.kind) {
 		case 'text':
-			return renderTextLayer(layer, parent, disposables)
+			return renderTextLayer(layer, parent)
 		case 'image':
-			return renderImageLayer(layer, parent, disposables)
+			return renderImageLayer(layer, parent)
+		case 'transition':
+			return renderTransitionLayer(layer, parent)
+		case 'gap': {
+			pixi?.renderer.clear()
+			return {dispose: () => {}}
+		}
 		default:
 			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
+			return {dispose: () => {}}
 	}
 }
 
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
-
 function renderTextLayer(
 	layer: Extract<Layer, {kind: 'text'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const text = new Text({
 		text: layer.content,
-		style: {
-			fontFamily: 'sans-serif',
-			fontSize: layer.fontSize ?? 48,
-			fill: layer.color ?? 'white'
-		}
+		style: layer.style
 	})
-	applyTransform(text, layer)
+	applyTransform(text, layer.matrix)
 	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
+	return {dispose: () => text.destroy(true)}
 }
 
 function renderImageLayer(
 	layer: Extract<Layer, {kind: 'image'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const texture = Texture.from(layer.frame)
 	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
+	applyTransform(sprite, layer.matrix)
+	parent.addChild(sprite)
+	return {dispose: () => {
+		sprite.destroy(true)
+		texture.destroy(true)
+		layer.frame.close()
+	}}
+}
+
+function renderTransitionLayer(
+	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+	parent: Container,
+) {
+	const transition = transitions.get(name) ??
+		(transitions.set(name, makeTransition({
+			name: "circle",
+			renderer: pixi!.renderer
+		})),
+	  transitions.get(name)!
+	)
+	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+	const sprite = new Sprite(texture)
 	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
+	return {dispose: () => sprite.destroy(false)}
 }
 
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
+function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  if (!worldMatrix) return
+	const mx = mat6ToMatrix(worldMatrix)
+  target.setFromMatrix(mx)
 }

package/s/features/transition/parts/fragment.ts

@@ -0,0 +1,24 @@
+export const fragment = (glsl: string) => `
+	precision highp float;
+	varying vec2 vTextureCoord;
+	varying vec2 _uv;
+	uniform sampler2D from, to;
+	uniform float progress, ratio, _fromR, _toR;
+	uniform float customUniform;
+
+	vec4 getFromColor(vec2 uv){
+		return texture2D(from, .5+(uv-.5)*vec2(max(ratio/_fromR,1.), max(_fromR/ratio,1.)));
+	}
+	vec4 getToColor(vec2 uv){
+		return texture2D(to, .5+(uv-.5)*vec2(max(ratio/_toR,1.), max(_toR/ratio,1.)));
+	}
+
+	// gl-transition code here
+	${glsl}
+	// gl-transition code end
+
+	void main(){
+		vec2 uv = vTextureCoord.xy;
+		gl_FragColor = transition(vTextureCoord);
+	}
+`

package/s/features/transition/parts/types.ts

@@ -0,0 +1,94 @@
+import {Renderer} from "pixi.js"
+
+export interface TransitionOptions {
+	name: Transition
+	renderer: Renderer
+}
+
+export interface TransitionRendererOptions {
+	from: VideoFrame
+	to: VideoFrame
+	progress: number
+	width: number
+	height: number
+}
+
+export interface GLTransition {
+	author: string
+	createdAt: string
+	glsl: string
+	license: string
+	name: Transition
+	updatedAt: string
+	defaultParams: any
+	paramsTypes: any
+}
+
+export type Transition =
+  | "Bounce"
+  | "BowTieHorizontal"
+  | "BowTieVertical"
+  | "ButterflyWaveScrawler"
+  | "CircleCrop"
+  | "ColourDistance"
+  | "CrazyParametricFun"
+  | "CrossZoom"
+  | "Directional"
+  | "DoomScreenTransition"
+  | "Dreamy"
+  | "DreamyZoom"
+  | "GlitchDisplace"
+  | "GlitchMemories"
+  | "GridFlip"
+  | "InvertedPageCurl"
+  | "LinearBlur"
+  | "Mosaic"
+  | "PolkaDotsCurtain"
+  | "Radial"
+  | "SimpleZoom"
+  | "StereoViewer"
+  | "Swirl"
+  | "WaterDrop"
+  | "ZoomInCircles"
+  | "angular"
+  | "burn"
+  | "cannabisleaf"
+  | "circle"
+  | "circleopen"
+  | "colorphase"
+  | "crosshatch"
+  | "crosswarp"
+  | "cube"
+  | "directionalwarp"
+  | "directionalwipe"
+  | "displacement"
+  | "doorway"
+  | "fade"
+  | "fadecolor"
+  | "fadegrayscale"
+  | "flyeye"
+  | "heart"
+  | "hexagonalize"
+  | "kaleidoscope"
+  | "luma"
+  | "luminance_melt"
+  | "morph"
+  | "multiply_blend"
+  | "perlin"
+  | "pinwheel"
+  | "pixelize"
+  | "polar_function"
+  | "randomsquares"
+  | "ripple"
+  | "rotate_scale_fade"
+  | "squareswire"
+  | "squeeze"
+  | "swap"
+  | "undulatingBurnOut"
+  | "wind"
+  | "windowblinds"
+  | "windowslice"
+  | "wipeDown"
+  | "wipeLeft"
+  | "wipeRight"
+  | "wipeUp"

package/s/features/transition/parts/uniforms.ts

@@ -0,0 +1,29 @@
+import {GLTransition} from "./types.js"
+
+export const uniforms = {
+	custom: (transition: GLTransition) => Object.fromEntries(
+		Object.entries(transition.defaultParams).map(([name, value]) => [
+			name,
+			{
+				value,
+				type: getUniformType(transition.paramsTypes[name])
+			}
+		])
+	),
+	basics: {
+		_fromR: {value: 1, type: "f32"},
+		_toR: {value: 1, type: "f32"},
+		ratio: {value: 1, type: "f32"},
+		progress: {value: 0, type: "f32"},
+		customUniform: {value: 0, type: "f32"},
+	}
+}
+
+const getUniformType = (type: string) => {
+	if(type === "f32" || type === "i32") {
+		return type
+	} else if(type === "float") {
+		return "f32"
+	}
+	else return `${type}<f32>`
+}

package/s/features/transition/parts/vertex.ts

@@ -0,0 +1,31 @@
+export const vertex = `
+	in vec2 aPosition;
+	varying vec2 _uv;                          // gl-transition
+	uniform mat3 projectionMatrix;
+	uniform vec4 uInputSize;
+	uniform vec4 uOutputFrame;
+	out vec2 vTextureCoord;
+	uniform vec4 uOutputTexture;
+
+	vec4 filterVertexPosition( void )
+	{
+			vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+
+			position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+			position.y = position.y * (2.0*uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+
+			return vec4(position, 0.0, 1.0);
+	}
+
+	vec2 filterTextureCoord( void )
+	{
+	return aPosition * (uOutputFrame.zw * uInputSize.zw);
+	}
+
+	void main(void)
+	{
+	gl_Position = filterVertexPosition();
+	vTextureCoord = filterTextureCoord();
+	_uv = vec2(0.5, 0.5) * (aPosition +vec2(1.0, 1.0));    // gl-transition
+	}
+`

package/s/features/transition/transition.ts

@@ -0,0 +1,60 @@
+//@ts-ignore
+import transitions from "gl-transitions"
+import {Filter, GlProgram, Sprite, Texture, ImageSource} from "pixi.js"
+
+import {vertex} from "./parts/vertex.js"
+import {uniforms} from "./parts/uniforms.js"
+import {fragment} from "./parts/fragment.js"
+import {GLTransition, TransitionOptions, TransitionRendererOptions} from "./parts/types.js"
+
+export function makeTransition({name, renderer}: TransitionOptions) {
+	const transition = transitions.find((t: GLTransition) => t.name === name) as GLTransition
+	const transitionSprite = new Sprite()
+	const transitionTexture = new Texture()
+	const sourceFrom = new ImageSource({})
+	const sourceTo = new ImageSource({})
+
+	const filter = new Filter({
+		glProgram: new GlProgram({
+			vertex,
+			fragment: fragment(transition.glsl),
+		}),
+		resources: {
+			from: sourceFrom,
+			to: sourceTo,
+			uniforms: {
+				...uniforms.basics,
+				...uniforms.custom(transition)
+			}
+		}
+	})
+
+	transitionSprite.filters = [filter]
+
+	return {
+		render({width, height, from, to, progress}: TransitionRendererOptions) {
+			if(transitionSprite.width !== width || transitionSprite.height !== height) {
+				transitionSprite.setSize({width, height})
+				transitionTexture.source.resize(width, height)
+			}
+
+			sourceFrom.resource = from
+			sourceTo.resource = to
+			sourceFrom.update()
+			sourceTo.update()
+
+			filter.resources.uniforms.uniforms.progress = progress
+
+			renderer.render({
+				container: transitionSprite,
+				target: transitionTexture,
+				clear: false,
+				width,
+				height
+			})
+
+			return transitionTexture
+		}
+	}
+}
+

package/s/index.html.ts

@@ -30,7 +30,7 @@ export default ssg.page(import.meta.url, async orb => ({
 		<section>
 			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
 			<button class=fetch>fetch</button>
-			<button class="import">import</button>
+			<input type="file" class="file-input">
 			<div class=results></div>
 			<div class=filmstrip-demo>
 				<label for="viewable-range">viewable range:</label>
@@ -47,6 +47,11 @@ export default ssg.page(import.meta.url, async orb => ({
 				<label for="width">width:</label>
 				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
 			</div>
+			<div class=player>
+				<input class="seek" type="number" min="0">
+				<button class=play>play</button>
+				<button class=stop>stop</button>
+			</div>
 		</section>
 	`,
 }))

package/s/timeline/index.ts

@@ -5,6 +5,7 @@ export * from "./parts/media.js"
 export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
+export * from "./parts/compositor/playback.js"
 
 export * from "./sugar/o.js"
 export * from "./sugar/omni.js"

package/s/timeline/parts/basics.ts

@@ -11,7 +11,7 @@ export type TimelineFile = {
 	info: "https://omniclip.app/"
 	format: "timeline"
 	version: number
-	root: Id
+	rootId: Id
 	items: Item.Any[]
 }
 

package/s/timeline/parts/compositor/export.ts

@@ -0,0 +1,77 @@
+import {TimelineFile} from "../basics.js"
+import {fixedStep} from "./parts/schedulers.js"
+import {Driver} from "../../../driver/driver.js"
+import {makeWebCodecsSampler} from "./samplers/webcodecs.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {buildWebCodecsNodeTree} from "./parts/webcodecs-tree.js"
+
+export class Export {
+	#sampler
+	constructor(
+		private driver: Driver,
+		private framerate = 30,
+		private resolveMedia: (hash: string) => DecoderSource = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#sampler = makeWebCodecsSampler(this.driver, this.resolveMedia)
+	}
+
+	async #build(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		return await buildWebCodecsNodeTree(rootItem, items, this.#sampler)
+	}
+
+	async render(timeline: TimelineFile) {
+		const root = await this.#build(timeline)
+
+		const videoStream = new TransformStream<VideoFrame, VideoFrame>()
+		const audioStream = new TransformStream<AudioData, AudioData>()
+
+		const encodePromise = this.driver.encode({
+			video: videoStream.readable,
+			audio: audioStream.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000},
+			},
+		})
+
+		const videoWriter = videoStream.writable.getWriter()
+		const audioWriter = audioStream.writable.getWriter()
+
+		const audioPromise = (async () => {
+			if (root.audio) {
+				for await (const chunk of root.audio.getStream()) {
+					await audioWriter.write(chunk)
+				}
+			}
+			await audioWriter.close()
+		})()
+
+		const videoPromise = (async () => {
+			let i = 0
+			const dt = 1 / this.framerate
+
+			await fixedStep(
+				{fps: this.framerate, duration: root.duration ?? 0},
+				async t => {
+					const layers = await root.visuals?.sampleAt(t) ?? []
+					const composed = await this.driver.composite(layers)
+					const vf = new VideoFrame(composed, {
+						timestamp: Math.round(i * dt * 1_000_000),
+						duration: Math.round(dt * 1_000_000),
+					})
+					await videoWriter.write(vf)
+					composed.close()
+					i++
+				}
+			)
+			await videoWriter.close()
+		})()
+
+		await audioPromise
+		await videoPromise
+		await encodePromise
+		// this.#sampler.dispose()
+	}
+}