@omnimedia/omnitool 1.1.0-4 → 1.1.0-7

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-4",
+	"version": "1.1.0-7",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -37,7 +37,9 @@
 		"@e280/stz": "^0.0.0-34",
 		"@huggingface/transformers": "^3.7.1",
 		"comrade": "^0.0.3",
-		"mediabunny": "^1.4.4",
+		"gl-transitions": "^1.43.0",
+		"gsap": "^3.13.0",
+		"mediabunny": "^1.14.3",
 		"mp4-muxer": "^5.2.1",
 		"pixi.js": "^8.10.1",
 		"wavesurfer.js": "^7.10.0",

package/s/demo/demo.css

@@ -46,6 +46,11 @@
 		}
 	}
 
+	canvas {
+		width: 500px;
+		height: 300px;
+	}
+
 	#filmstrip {
 		display: flex;
 		overflow-x: scroll;

package/s/demo/routines/transcode-test.ts

@@ -10,7 +10,7 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 	const ctx = canvas.getContext("2d")
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const composed = await driver.composite([
@@ -30,9 +30,11 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 				return composed
 			}
 		})
+		const audio = driver.decodeAudio({source})
 
 		await driver.encode({
-			readables,
+			video,
+			audio,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transitions-test.ts

@@ -0,0 +1,43 @@
+import {Application, Sprite} from "pixi.js"
+
+import {Driver} from "../../driver/driver.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export async function setupTransitionsTest(driver: Driver, source: DecoderSource) {
+	const app = new Application()
+	await app.init({width: 300, height: 300, preference: "webgl"})
+	const sprite = new Sprite({width: 300, height: 300})
+
+	app.stage.addChild(sprite)
+
+	document.body.appendChild(app.canvas)
+	const transition = makeTransition({name: "circle", renderer: app.renderer})
+
+	async function run() {
+		const video = driver.decodeVideo({
+			source,
+			async onFrame(frame) {
+				const texture = transition.render({
+					from: frame,
+					to: frame,
+					progress: 0.7,
+					width: app.canvas.width,
+					height: app.canvas.height
+				})
+				sprite.texture = texture
+				return frame
+			}
+		})
+
+		await driver.encode({
+			video,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000}
+			}
+		})
+	}
+
+	return {run}
+}

package/s/driver/driver.ts

@@ -50,7 +50,7 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
@@ -61,19 +61,27 @@ export class Driver {
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
 			source: input.source,
 			video: videoTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return videoTransform.readable
+	}
+
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
 			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
-		}
+		return audioTransform.readable
 	}
 
-	async encode({readables, config}: EncoderInput) {
+	async encode({video, audio, config}: EncoderInput) {
 		const handle = await window.showSaveFilePicker()
 		const writable = await handle.createWritable()
 		// making bridge because file picker writable is not transferable
@@ -85,7 +93,7 @@ export class Driver {
 				await writable.close()
 			}
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
 	}
 
 	async composite(

package/s/driver/fns/schematic.ts

@@ -2,16 +2,26 @@
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Mat6} from "../../timeline/utils/matrix.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
 		}): Promise<void>
 
 		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
@@ -26,20 +36,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
 export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +71,36 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
-
 export type TextLayer = {
 	kind: 'text'
 	content: string
 	fontSize?: number
 	color?: string
-} & Transform
+	matrix?: Mat6
+}
 
 export type ImageLayer = {
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	kind: 'gap'
+}
+
+export type Audio = {
+	kind: "audio"
+	data: AudioData
+}
 
-export type Layer = TextLayer | ImageLayer
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer
 

package/s/driver/fns/work.ts

@@ -1,88 +1,86 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
 import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+
 export const setupDriverWork = (
 	Comrade.work<DriverSchematic>(shell => ({
 		async hello() {
 			await shell.host.world()
 		},
 
-		async decode({source, video, audio}) {
-			const loadSource = async () => {
-				if(source instanceof Blob) {
-					return new BlobSource(source)
-				} else {
-					return new UrlSource(source)
+		async decodeAudio({source, audio, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
+
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
 				}
+				await audioWriter.close()
 			}
+		},
+
+		async decodeVideo({source, video, start, end}) {
 			const input = new Input({
-				source: await loadSource(),
+				source: await loadSource(source),
 				formats: ALL_FORMATS
 			})
 
-			const [videoTrack, audioTrack] = await Promise.all([
-				input.getPrimaryVideoTrack(),
-				input.getPrimaryAudioTrack()
-			])
-
+			const videoTrack = await input.getPrimaryVideoTrack()
 			const videoDecodable = await videoTrack?.canDecode()
-			const audioDecodable = await audioTrack?.canDecode()
-
 			const videoWriter = video.getWriter()
-			const audioWriter = audio.getWriter()
 
-			await Promise.all([
-				(async () => {
-					if (videoDecodable && videoTrack) {
-						const sink = new VideoSampleSink(videoTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toVideoFrame()
-							await videoWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await videoWriter.close()
-					}
-				})(),
-				(async () => {
-					if (audioDecodable && audioTrack) {
-						const sink = new AudioSampleSink(audioTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toAudioData()
-							await audioWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await audioWriter.close()
-					}
-				})()
-			])
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
 		},
 
-		async encode({readables, config, bridge}) {
+		async encode({video, audio, config, bridge}) {
 			const output = new Output({
 				format: new Mp4OutputFormat(),
 				target: new StreamTarget(bridge, {chunked: true})
 			})
-			const videoSource = new VideoSampleSource(config.video)
-			output.addVideoTrack(videoSource)
 			// since AudioSample is not transferable it fails to transfer encoder bitrate config
 			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-			const audioSource = new AudioSampleSource(config.audio)
-			output.addAudioTrack(audioSource)
 
-			await output.start()
-
-			const videoReader = readables.video.getReader()
-			const audioReader = readables.audio.getReader()
+			const promises = []
 
-			await Promise.all([
-				(async () => {
+			if(video) {
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				const videoReader = video.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await videoReader.read()
 						if (done) break
@@ -90,8 +88,14 @@ export const setupDriverWork = (
 						await videoSource.add(sample)
 						sample.close()
 					}
-				})(),
-				(async () => {
+				})())
+			}
+
+			if(audio) {
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				const audioReader = audio.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await audioReader.read()
 						if (done) break
@@ -100,9 +104,11 @@ export const setupDriverWork = (
 						sample.close()
 						value.close()
 					}
-				})()
-			])
+				})())
+			}
 
+			await output.start()
+			await Promise.all(promises)
 			await output.finalize()
 		},
 
@@ -110,22 +116,18 @@ export const setupDriverWork = (
 			const {stage, renderer} = await renderPIXI(1920, 1080)
 			stage.removeChildren()
 
-			const {baseFrame, disposables} = await renderLayer(composition, stage)
+			const {dispose} = await renderLayer(composition, stage)
 			renderer.render(stage)
 
 			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
 			// if its very slow on eg edge try chrome
 			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: baseFrame?.timestamp,
-				duration: baseFrame?.duration ?? undefined,
+				timestamp: 0,
+				duration: 0,
 			})
 
-			baseFrame?.close()
 			renderer.clear()
-
-			for (const disposable of disposables) {
-				disposable.destroy(true)
-			}
+			dispose()
 
 			shell.transfer = [frame]
 			return frame
@@ -157,46 +159,43 @@ async function renderPIXI(width: number, height: number) {
 	return pixi
 }
 
+const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+
 type RenderableObject = Sprite | Text | Texture
 
 async function renderLayer(
 	layer: Layer | Composition,
 	parent: Container,
-	disposables: RenderableObject[] = []
 ) {
 	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
+		const disposers: (() => void)[] = []
 		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
+			const result = await renderLayer(child, parent)
+			disposers.push(result.dispose)
 		}
-		return {baseFrame, disposables}
-	}
-
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
+		return {dispose: () => disposers.forEach(d => d())}
 	}
 
 	switch (layer.kind) {
 		case 'text':
-			return renderTextLayer(layer, parent, disposables)
+			return renderTextLayer(layer, parent)
 		case 'image':
-			return renderImageLayer(layer, parent, disposables)
+			return renderImageLayer(layer, parent)
+		case 'transition':
+			return renderTransitionLayer(layer, parent)
+		case 'gap': {
+			pixi?.renderer.clear()
+			return {dispose: () => {}}
+		}
 		default:
 			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
+			return {dispose: () => {}}
 	}
 }
 
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
-
 function renderTextLayer(
 	layer: Extract<Layer, {kind: 'text'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const text = new Text({
 		text: layer.content,
@@ -206,29 +205,45 @@ function renderTextLayer(
 			fill: layer.color ?? 'white'
 		}
 	})
-	applyTransform(text, layer)
+	applyTransform(text, layer.matrix)
 	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
+	return {dispose: () => text.destroy(true)}
 }
 
 function renderImageLayer(
 	layer: Extract<Layer, {kind: 'image'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const texture = Texture.from(layer.frame)
 	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
+	applyTransform(sprite, layer.matrix)
+	parent.addChild(sprite)
+	return {dispose: () => {
+		sprite.destroy(true)
+		texture.destroy(true)
+		layer.frame.close()
+	}}
+}
+
+function renderTransitionLayer(
+	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+	parent: Container,
+) {
+	const transition = transitions.get(name) ??
+		(transitions.set(name, makeTransition({
+			name: "circle",
+			renderer: pixi!.renderer
+		})),
+	  transitions.get(name)!
+	)
+	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+	const sprite = new Sprite(texture)
 	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
+	return {dispose: () => sprite.destroy(false)}
 }
 
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
+function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  if (!worldMatrix) return
+	const mx = mat6ToMatrix(worldMatrix)
+  target.setFromMatrix(mx)
 }

package/s/features/transition/parts/fragment.ts

@@ -0,0 +1,24 @@
+export const fragment = (glsl: string) => `
+	precision highp float;
+	varying vec2 vTextureCoord;
+	varying vec2 _uv;
+	uniform sampler2D from, to;
+	uniform float progress, ratio, _fromR, _toR;
+	uniform float customUniform;
+
+	vec4 getFromColor(vec2 uv){
+		return texture2D(from, .5+(uv-.5)*vec2(max(ratio/_fromR,1.), max(_fromR/ratio,1.)));
+	}
+	vec4 getToColor(vec2 uv){
+		return texture2D(to, .5+(uv-.5)*vec2(max(ratio/_toR,1.), max(_toR/ratio,1.)));
+	}
+
+	// gl-transition code here
+	${glsl}
+	// gl-transition code end
+
+	void main(){
+		vec2 uv = vTextureCoord.xy;
+		gl_FragColor = transition(vTextureCoord);
+	}
+`

package/s/features/transition/parts/types.ts

@@ -0,0 +1,94 @@
+import {Renderer} from "pixi.js"
+
+export interface TransitionOptions {
+	name: Transition
+	renderer: Renderer
+}
+
+export interface TransitionRendererOptions {
+	from: VideoFrame
+	to: VideoFrame
+	progress: number
+	width: number
+	height: number
+}
+
+export interface GLTransition {
+	author: string
+	createdAt: string
+	glsl: string
+	license: string
+	name: Transition
+	updatedAt: string
+	defaultParams: any
+	paramsTypes: any
+}
+
+export type Transition =
+  | "Bounce"
+  | "BowTieHorizontal"
+  | "BowTieVertical"
+  | "ButterflyWaveScrawler"
+  | "CircleCrop"
+  | "ColourDistance"
+  | "CrazyParametricFun"
+  | "CrossZoom"
+  | "Directional"
+  | "DoomScreenTransition"
+  | "Dreamy"
+  | "DreamyZoom"
+  | "GlitchDisplace"
+  | "GlitchMemories"
+  | "GridFlip"
+  | "InvertedPageCurl"
+  | "LinearBlur"
+  | "Mosaic"
+  | "PolkaDotsCurtain"
+  | "Radial"
+  | "SimpleZoom"
+  | "StereoViewer"
+  | "Swirl"
+  | "WaterDrop"
+  | "ZoomInCircles"
+  | "angular"
+  | "burn"
+  | "cannabisleaf"
+  | "circle"
+  | "circleopen"
+  | "colorphase"
+  | "crosshatch"
+  | "crosswarp"
+  | "cube"
+  | "directionalwarp"
+  | "directionalwipe"
+  | "displacement"
+  | "doorway"
+  | "fade"
+  | "fadecolor"
+  | "fadegrayscale"
+  | "flyeye"
+  | "heart"
+  | "hexagonalize"
+  | "kaleidoscope"
+  | "luma"
+  | "luminance_melt"
+  | "morph"
+  | "multiply_blend"
+  | "perlin"
+  | "pinwheel"
+  | "pixelize"
+  | "polar_function"
+  | "randomsquares"
+  | "ripple"
+  | "rotate_scale_fade"
+  | "squareswire"
+  | "squeeze"
+  | "swap"
+  | "undulatingBurnOut"
+  | "wind"
+  | "windowblinds"
+  | "windowslice"
+  | "wipeDown"
+  | "wipeLeft"
+  | "wipeRight"
+  | "wipeUp"