@omnimedia/omnitool 1.1.0-5 → 1.1.0-8

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-5",
+	"version": "1.1.0-8",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -38,7 +38,8 @@
 		"@huggingface/transformers": "^3.7.1",
 		"comrade": "^0.0.3",
 		"gl-transitions": "^1.43.0",
-		"mediabunny": "^1.4.4",
+		"gsap": "^3.13.0",
+		"mediabunny": "^1.14.3",
 		"mp4-muxer": "^5.2.1",
 		"pixi.js": "^8.10.1",
 		"wavesurfer.js": "^7.10.0",

package/s/demo/demo.css

@@ -46,6 +46,11 @@
 		}
 	}
 
+	canvas {
+		width: 500px;
+		height: 300px;
+	}
+
 	#filmstrip {
 		display: flex;
 		overflow-x: scroll;

package/s/demo/routines/transcode-test.ts

@@ -10,7 +10,7 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 	const ctx = canvas.getContext("2d")
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const composed = await driver.composite([
@@ -30,9 +30,11 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 				return composed
 			}
 		})
+		const audio = driver.decodeAudio({source})
 
 		await driver.encode({
-			readables,
+			video,
+			audio,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transitions-test.ts

@@ -15,7 +15,7 @@ export async function setupTransitionsTest(driver: Driver, source: DecoderSource
 	const transition = makeTransition({name: "circle", renderer: app.renderer})
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const texture = transition.render({
@@ -31,7 +31,7 @@ export async function setupTransitionsTest(driver: Driver, source: DecoderSource
 		})
 
 		await driver.encode({
-			readables,
+			video,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/driver/driver.ts

@@ -50,7 +50,7 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
@@ -61,19 +61,27 @@ export class Driver {
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
 			source: input.source,
 			video: videoTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return videoTransform.readable
+	}
+
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
 			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
-		}
+		return audioTransform.readable
 	}
 
-	async encode({readables, config}: EncoderInput) {
+	async encode({video, audio, config}: EncoderInput) {
 		const handle = await window.showSaveFilePicker()
 		const writable = await handle.createWritable()
 		// making bridge because file picker writable is not transferable
@@ -85,7 +93,7 @@ export class Driver {
 				await writable.close()
 			}
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
 	}
 
 	async composite(

package/s/driver/fns/schematic.ts

@@ -2,16 +2,26 @@
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Mat6} from "../../timeline/utils/matrix.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
 		}): Promise<void>
 
 		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
@@ -26,20 +36,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
 export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +71,36 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
-
 export type TextLayer = {
 	kind: 'text'
 	content: string
 	fontSize?: number
 	color?: string
-} & Transform
+	matrix?: Mat6
+}
 
 export type ImageLayer = {
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	kind: 'gap'
+}
+
+export type Audio = {
+	kind: "audio"
+	data: AudioData
+}
 
-export type Layer = TextLayer | ImageLayer
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer
 

package/s/driver/fns/work.ts

@@ -1,88 +1,86 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
 import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+
 export const setupDriverWork = (
 	Comrade.work<DriverSchematic>(shell => ({
 		async hello() {
 			await shell.host.world()
 		},
 
-		async decode({source, video, audio}) {
-			const loadSource = async () => {
-				if(source instanceof Blob) {
-					return new BlobSource(source)
-				} else {
-					return new UrlSource(source)
+		async decodeAudio({source, audio, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
+
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
 				}
+				await audioWriter.close()
 			}
+		},
+
+		async decodeVideo({source, video, start, end}) {
 			const input = new Input({
-				source: await loadSource(),
+				source: await loadSource(source),
 				formats: ALL_FORMATS
 			})
 
-			const [videoTrack, audioTrack] = await Promise.all([
-				input.getPrimaryVideoTrack(),
-				input.getPrimaryAudioTrack()
-			])
-
+			const videoTrack = await input.getPrimaryVideoTrack()
 			const videoDecodable = await videoTrack?.canDecode()
-			const audioDecodable = await audioTrack?.canDecode()
-
 			const videoWriter = video.getWriter()
-			const audioWriter = audio.getWriter()
 
-			await Promise.all([
-				(async () => {
-					if (videoDecodable && videoTrack) {
-						const sink = new VideoSampleSink(videoTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toVideoFrame()
-							await videoWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await videoWriter.close()
-					}
-				})(),
-				(async () => {
-					if (audioDecodable && audioTrack) {
-						const sink = new AudioSampleSink(audioTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toAudioData()
-							await audioWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await audioWriter.close()
-					}
-				})()
-			])
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
 		},
 
-		async encode({readables, config, bridge}) {
+		async encode({video, audio, config, bridge}) {
 			const output = new Output({
 				format: new Mp4OutputFormat(),
 				target: new StreamTarget(bridge, {chunked: true})
 			})
-			const videoSource = new VideoSampleSource(config.video)
-			output.addVideoTrack(videoSource)
 			// since AudioSample is not transferable it fails to transfer encoder bitrate config
 			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-			const audioSource = new AudioSampleSource(config.audio)
-			output.addAudioTrack(audioSource)
 
-			await output.start()
-
-			const videoReader = readables.video.getReader()
-			const audioReader = readables.audio.getReader()
+			const promises = []
 
-			await Promise.all([
-				(async () => {
+			if(video) {
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				const videoReader = video.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await videoReader.read()
 						if (done) break
@@ -90,8 +88,14 @@ export const setupDriverWork = (
 						await videoSource.add(sample)
 						sample.close()
 					}
-				})(),
-				(async () => {
+				})())
+			}
+
+			if(audio) {
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				const audioReader = audio.getReader()
+				promises.push((async () => {
 					while (true) {
 						const {done, value} = await audioReader.read()
 						if (done) break
@@ -100,9 +104,11 @@ export const setupDriverWork = (
 						sample.close()
 						value.close()
 					}
-				})()
-			])
+				})())
+			}
 
+			await output.start()
+			await Promise.all(promises)
 			await output.finalize()
 		},
 
@@ -110,22 +116,18 @@ export const setupDriverWork = (
 			const {stage, renderer} = await renderPIXI(1920, 1080)
 			stage.removeChildren()
 
-			const {baseFrame, disposables} = await renderLayer(composition, stage)
+			const {dispose} = await renderLayer(composition, stage)
 			renderer.render(stage)
 
 			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
 			// if its very slow on eg edge try chrome
 			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: baseFrame?.timestamp,
-				duration: baseFrame?.duration ?? undefined,
+				timestamp: 0,
+				duration: 0,
 			})
 
-			baseFrame?.close()
 			renderer.clear()
-
-			for (const disposable of disposables) {
-				disposable.destroy(true)
-			}
+			dispose()
 
 			shell.transfer = [frame]
 			return frame
@@ -157,46 +159,43 @@ async function renderPIXI(width: number, height: number) {
 	return pixi
 }
 
+const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+
 type RenderableObject = Sprite | Text | Texture
 
 async function renderLayer(
 	layer: Layer | Composition,
 	parent: Container,
-	disposables: RenderableObject[] = []
 ) {
 	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
+		const disposers: (() => void)[] = []
 		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
+			const result = await renderLayer(child, parent)
+			disposers.push(result.dispose)
 		}
-		return {baseFrame, disposables}
-	}
-
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
+		return {dispose: () => disposers.forEach(d => d())}
 	}
 
 	switch (layer.kind) {
 		case 'text':
-			return renderTextLayer(layer, parent, disposables)
+			return renderTextLayer(layer, parent)
 		case 'image':
-			return renderImageLayer(layer, parent, disposables)
+			return renderImageLayer(layer, parent)
+		case 'transition':
+			return renderTransitionLayer(layer, parent)
+		case 'gap': {
+			pixi?.renderer.clear()
+			return {dispose: () => {}}
+		}
 		default:
 			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
+			return {dispose: () => {}}
 	}
 }
 
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
-
 function renderTextLayer(
 	layer: Extract<Layer, {kind: 'text'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const text = new Text({
 		text: layer.content,
@@ -206,29 +205,45 @@ function renderTextLayer(
 			fill: layer.color ?? 'white'
 		}
 	})
-	applyTransform(text, layer)
+	applyTransform(text, layer.matrix)
 	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
+	return {dispose: () => text.destroy(true)}
 }
 
 function renderImageLayer(
 	layer: Extract<Layer, {kind: 'image'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const texture = Texture.from(layer.frame)
 	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
+	applyTransform(sprite, layer.matrix)
+	parent.addChild(sprite)
+	return {dispose: () => {
+		sprite.destroy(true)
+		texture.destroy(true)
+		layer.frame.close()
+	}}
+}
+
+function renderTransitionLayer(
+	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+	parent: Container,
+) {
+	const transition = transitions.get(name) ??
+		(transitions.set(name, makeTransition({
+			name: "circle",
+			renderer: pixi!.renderer
+		})),
+	  transitions.get(name)!
+	)
+	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+	const sprite = new Sprite(texture)
 	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
+	return {dispose: () => sprite.destroy(false)}
 }
 
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
+function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  if (!worldMatrix) return
+	const mx = mat6ToMatrix(worldMatrix)
+  target.setFromMatrix(mx)
 }

package/s/index.html.ts

@@ -30,7 +30,7 @@ export default ssg.page(import.meta.url, async orb => ({
 		<section>
 			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
 			<button class=fetch>fetch</button>
-			<button class="import">import</button>
+			<input type="file" class="file-input">
 			<div class=results></div>
 			<div class=filmstrip-demo>
 				<label for="viewable-range">viewable range:</label>
@@ -47,6 +47,11 @@ export default ssg.page(import.meta.url, async orb => ({
 				<label for="width">width:</label>
 				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
 			</div>
+			<div class=player>
+				<input class="seek" type="number" min="0">
+				<button class=play>play</button>
+				<button class=stop>stop</button>
+			</div>
 		</section>
 	`,
 }))

package/s/timeline/index.ts

@@ -8,6 +8,7 @@ export * from "./parts/filmstrip.js"
 
 export * from "./sugar/o.js"
 export * from "./sugar/omni.js"
+export * from "./timeline.js"
 
 export * from "./utils/checksum.js"
 export * from "./utils/datafile.js"

package/s/timeline/parts/basics.ts

@@ -11,7 +11,7 @@ export type TimelineFile = {
 	info: "https://omniclip.app/"
 	format: "timeline"
 	version: number
-	root: Id
+	rootId: Id
 	items: Item.Any[]
 }
 

package/s/timeline/parts/compositor/export.ts

@@ -0,0 +1,77 @@
+import {TimelineFile} from "../basics.js"
+import {context} from "../../../context.js"
+import {fixedStep} from "./parts/schedulers.js"
+import {makeWebCodecsSampler} from "./samplers/webcodecs.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {buildWebCodecsNodeTree} from "./parts/webcodecs-tree.js"
+
+export class Export {
+	#sampler
+	constructor(
+		private framerate = 30,
+		private resolveMedia: (hash: string) => DecoderSource = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#sampler = makeWebCodecsSampler(this.resolveMedia)
+	}
+
+	async #build(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		return await buildWebCodecsNodeTree(rootItem, items, this.#sampler)
+	}
+
+	async render(timeline: TimelineFile) {
+		const root = await this.#build(timeline)
+
+		const driver = await context.driver
+		const videoStream = new TransformStream<VideoFrame, VideoFrame>()
+		const audioStream = new TransformStream<AudioData, AudioData>()
+
+		const encodePromise = driver.encode({
+			video: videoStream.readable,
+			audio: audioStream.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000},
+			},
+		})
+
+		const videoWriter = videoStream.writable.getWriter()
+		const audioWriter = audioStream.writable.getWriter()
+
+		const audioPromise = (async () => {
+			if (root.audio) {
+				for await (const chunk of root.audio.getStream()) {
+					await audioWriter.write(chunk)
+				}
+			}
+			await audioWriter.close()
+		})()
+
+		const videoPromise = (async () => {
+			let i = 0
+			const dt = 1 / this.framerate
+
+			await fixedStep(
+				{fps: this.framerate, duration: root.duration ?? 0},
+				async t => {
+					const layers = await root.visuals?.sampleAt(t) ?? []
+					const composed = await driver.composite(layers)
+					const vf = new VideoFrame(composed, {
+						timestamp: Math.round(i * dt * 1_000_000),
+						duration: Math.round(dt * 1_000_000),
+					})
+					await videoWriter.write(vf)
+					composed.close()
+					i++
+				}
+			)
+			await videoWriter.close()
+		})()
+
+		await audioPromise
+		await videoPromise
+		await encodePromise
+		// this.#sampler.dispose()
+	}
+}