@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/driver/fns/work.ts

@@ -1,237 +1,125 @@
 import {Comrade} from "@e280/comrade"
-import {
-	Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample,
-	AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource
-} from "mediabunny"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink,
+	AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-DOMAdapter.set(WebWorkerAdapter)
+import {DecoderSource, DriverSchematic} from "./schematic.js"
 
-export const setupDriverWork = Comrade.work<DriverSchematic>(({host}, rig) => ({
-
-	async hello() {
-		await host.world()
-	},
-
-	async decode({source, video, audio}) {
-		const loadSource = async () => {
-			if(source instanceof FileSystemFileHandle) {
-				const file = await source.getFile()
-				return new BlobSource(file)
-			} else {
-				return new UrlSource(source)
-			}
-		}
-		const input = new Input({
-			source: await loadSource(),
-			formats: ALL_FORMATS
-		})
-
-		const [videoTrack, audioTrack] = await Promise.all([
-			input.getPrimaryVideoTrack(),
-			input.getPrimaryAudioTrack()
-		])
-
-		const videoDecodable = await videoTrack?.canDecode()
-		const audioDecodable = await audioTrack?.canDecode()
-
-		const videoWriter = video.getWriter()
-		const audioWriter = audio.getWriter()
-
-		await Promise.all([
-			(async () => {
-				if (videoDecodable && videoTrack) {
-					const sink = new VideoSampleSink(videoTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toVideoFrame()
-						await videoWriter.write(frame)
-						sample.close()
-						frame.close()
-					}
-					await videoWriter.close()
-				}
-			})(),
-			(async () => {
-				if (audioDecodable && audioTrack) {
-					const sink = new AudioSampleSink(audioTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toAudioData()
-						await audioWriter.write(frame)
-						sample.close()
-						frame.close()
-					}
-					await audioWriter.close()
-				}
-			})()
-		])
-	},
-
-	async encode({readables, config, bridge}) {
-		const output = new Output({
-			format: new Mp4OutputFormat(),
-			target: new StreamTarget(bridge, {chunked: true})
-		})
-		const videoSource = new VideoSampleSource(config.video)
-		output.addVideoTrack(videoSource)
-		// since AudioSample is not transferable it fails to transfer encoder bitrate config
-		// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-		const audioSource = new AudioSampleSource(config.audio)
-		output.addAudioTrack(audioSource)
-
-		await output.start()
-
-		const videoReader = readables.video.getReader()
-		const audioReader = readables.audio.getReader()
-
-		await Promise.all([
-			(async () => {
-				while (true) {
-					const {done, value} = await videoReader.read()
-					if (done) break
-					const sample = new VideoSample(value)
-					await videoSource.add(sample)
-					sample.close()
-				}
-			})(),
-			(async () => {
-				while (true) {
-					const {done, value} = await audioReader.read()
-					if (done) break
-					const sample = new AudioSample(value)
-					await audioSource.add(sample)
-					sample.close()
-					value.close()
-				}
-			})()
-		])
-
-		await output.finalize()
-	},
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
 
-	async composite(composition) {
-		const {stage, renderer} = await renderPIXI(1920, 1080)
-		stage.removeChildren()
+export const setupDriverWork = (
+	Comrade.work<DriverSchematic>(shell => ({
+		async hello() {
+			await shell.host.world()
+		},
 
-		const {baseFrame, disposables} = await renderLayer(composition, stage)
-		renderer.render(stage)
+		async decodeAudio({source, audio, start, end, cancel}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
 
-		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-		// if its very slow on eg edge try chrome
-		const frame = new VideoFrame(renderer.canvas, {
-			timestamp: baseFrame?.timestamp,
-			duration: baseFrame?.duration ?? undefined,
-		})
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
 
-		baseFrame?.close()
-		renderer.clear()
+			if(!audioDecodable || !audioTrack)
+				return
 
-		for (const disposable of disposables) {
-			disposable.destroy(true)
-		}
+			const sink = new AudioSampleSink(audioTrack)
+			const samples = sink.samples(start, end)
 
-		rig.transfer = [frame]
-		return frame
-	}
-}))
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
+			}
 
+			for await (const sample of samples) {
+				const frame = sample.toAudioData()
+				sample.close()
+				await audioWriter.write(frame)
+				frame.close()
+			}
 
-let pixi: {
-	renderer: Renderer
-	stage: Container
-} | null = null
+			await audioWriter.close()
+		},
 
-async function renderPIXI(width: number, height: number) {
-	if (pixi)
-		return pixi
+		async decodeVideo({source, video, start, end, cancel}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
 
-	const renderer = await autoDetectRenderer({
-		width,
-		height,
-		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
-		background: "black",
-		preferWebGLVersion: 2
-	})
+			const videoTrack = await input.getPrimaryVideoTrack()
+			const videoDecodable = await videoTrack?.canDecode()
+			const videoWriter = video.getWriter()
 
-	const stage = new Container()
-	pixi = {renderer, stage}
+			if(!videoDecodable || !videoTrack)
+				return
 
-	return pixi
-}
+			const sink = new VideoSampleSink(videoTrack)
+			const samples = sink.samples(start, end)
 
-type RenderableObject = Sprite | Text | Texture
-
-async function renderLayer(
-	layer: Layer | Composition,
-	parent: Container,
-	disposables: RenderableObject[] = []
-) {
-	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
-		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
-		}
-		return {baseFrame, disposables}
-	}
+			cancel.onmessage = async () => {
+				await samples.return()
+				cancel.close()
+			}
 
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
-	}
+			for await (const sample of samples) {
+				const frame = sample.toVideoFrame()
+				sample.close()
+				await videoWriter.write(frame)
+				frame.close()
+			}
 
-	switch (layer.kind) {
-		case 'text':
-			return renderTextLayer(layer, parent, disposables)
-		case 'image':
-			return renderImageLayer(layer, parent, disposables)
-		default:
-			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
-	}
-}
+			await videoWriter.close()
+			input.dispose()
+		},
+
+		async encode({video, audio, config, writable}) {
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new StreamTarget(writable, {chunked: true})
+			})
+
+			async function encodeVideo() {
+				if(!video) return
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				for await (const frame of video) {
+					const sample = new VideoSample(frame)
+					await videoSource.add(sample)
+					sample.close()
+					frame.close()
+				}
+			}
 
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
+			async function encodeAudio() {
+				if(!audio) return
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				for await (const data of audio) {
+					const sample = new AudioSample(data)
+					await audioSource.add(sample)
+					sample.close()
+					data.close()
+				}
+			}
 
-function renderTextLayer(
-	layer: Extract<Layer, {kind: 'text'}>,
-	parent: Container,
-	disposables: RenderableObject[]
-) {
-	const text = new Text({
-		text: layer.content,
-		style: {
-			fontFamily: 'sans-serif',
-			fontSize: layer.fontSize ?? 48,
-			fill: layer.color ?? 'white'
-		}
-	})
-	applyTransform(text, layer)
-	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
-}
+			const audioTask = encodeAudio()
+			const videoTask = encodeVideo()
 
-function renderImageLayer(
-	layer: Extract<Layer, {kind: 'image'}>,
-	parent: Container,
-	disposables: RenderableObject[]
-) {
-	const texture = Texture.from(layer.frame)
-	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
-	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
-}
+			await output.start()
+			await Promise.all([audioTask, videoTask])
+			await output.finalize()
+		},
+	}))
+)
 
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
-}

package/s/driver/parts/compositor.ts

@@ -0,0 +1,319 @@
+
+import {pub} from "@e280/stz"
+import {
+	autoDetectRenderer,
+	ColorMatrixFilter,
+	Container,
+	FederatedPointerEvent,
+	Filter,
+	Graphics,
+	Renderer,
+	Sprite,
+	Text,
+	Texture
+} from "pixi.js"
+import * as PixiFilters from "pixi-filters"
+
+import {Id} from "../../timeline/index.js"
+import {Crop} from "../../timeline/parts/item.js"
+import {findPixiFilter} from "../utils/find-pixi-filter.js"
+import {Composition, FilterSpec, Layer} from "../fns/schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export class Compositor {
+	onPointerDown = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onPointerMove = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onPointerUp = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onDispose = pub<[{id: Id, object: Container}]>()
+
+	static async setup() {
+		const renderer = await autoDetectRenderer({
+			width: 1920,
+			height: 1080,
+			preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+			background: "black",
+			preferWebGLVersion: 2
+		})
+		const stage = new Container()
+		stage.interactive = true
+		return new this({renderer, stage})
+	}
+
+	constructor(public pixi: {renderer: Renderer, stage: Container}) {}
+
+	#transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+	// objects rendered for current Composition
+	#activeObjects = new Map<number, {sprite: Container, dispose: () => void}>()
+	#cropMasks = new WeakMap<Container, Graphics>()
+
+	async composite(
+		composition: Composition,
+	) {
+		const {stage, renderer} = this.pixi
+
+		this.#cleanup(this.#collectIds(composition))
+		const {dispose} = await this.#renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: 0,
+			duration: 0,
+		})
+
+		dispose()
+
+		return frame
+	}
+
+	/**
+	 * get object for current Composition
+	 * */
+	getActiveObject(id: Id) {
+		return this.#activeObjects.get(id)?.sprite
+	}
+
+	async #renderLayer(
+		layer: Layer | Composition,
+		parent: Container,
+	) {
+		if (Array.isArray(layer)) {
+			const disposers: (() => void)[] = []
+			for (const child of [...layer].reverse()) {
+				const result = await this.#renderLayer(child, parent)
+				disposers.push(result.dispose)
+			}
+			return {dispose: () => disposers.forEach(d => d())}
+		}
+
+		switch (layer.kind) {
+			case 'text':
+				return this.#renderTextLayer(layer, parent)
+			case 'image':
+				return this.#renderImageLayer(layer, parent)
+			case 'transition':
+				return this.#renderTransitionLayer(layer, parent)
+			case 'gap': {
+				this.pixi?.renderer.clear()
+				return {dispose: () => {}}
+			}
+			default:
+				console.warn('Unknown layer kind', (layer as any).kind)
+				return {dispose: () => {}}
+		}
+	}
+
+	#renderTextLayer(
+		layer: Extract<Layer, {kind: 'text'}>,
+		parent: Container,
+	) {
+		const sprite = this.#findOrCreate<Text>(layer)!
+		sprite.text = layer.content
+		if (layer.style)
+			sprite.style = layer.style
+		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
+		this.#applyCrop(sprite, layer.crop)
+		this.#applyFilters(sprite, layer.filters)
+		parent.addChild(sprite)
+		return {
+			dispose: () => {}
+		}
+	}
+
+	#renderImageLayer(
+		layer: Extract<Layer, {kind: 'image'}>,
+		parent: Container,
+	) {
+		const sprite = this.#findOrCreate<Sprite>(layer)!
+
+		if (sprite.texture && sprite.texture !== Texture.EMPTY) {
+			sprite.texture.destroy(true)
+		}
+
+		const texture = Texture.from(layer.frame)
+		sprite.texture = texture
+		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
+		this.#applyCrop(sprite, layer.crop)
+		this.#applyFilters(sprite, layer.filters)
+		parent.addChild(sprite)
+
+		return {
+			dispose: () => {
+				layer.frame.close()
+			}
+		}
+	}
+
+	#renderTransitionLayer(
+		{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+		parent: Container,
+	) {
+		const transition = this.#transitions.get(name) ??
+			(this.#transitions.set(name, makeTransition({
+				name,
+				renderer: this.pixi.renderer
+			})),
+	  	this.#transitions.get(name)!
+		)
+		const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+		const sprite = new Sprite(texture)
+		parent.addChild(sprite)
+		return {dispose: () => sprite.destroy(false)}
+	}
+
+	#applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  	if (!worldMatrix) return
+		const mx = mat6ToMatrix(worldMatrix)
+  	target.setFromMatrix(mx)
+	}
+
+	#applyAlpha(target: Container, alpha?: number) {
+		target.alpha = alpha ?? 1
+	}
+
+	#applyCrop(target: Container, crop?: Crop) {
+		const existing = this.#cropMasks.get(target)
+		if (existing) {
+			if (target.mask === existing)
+				target.mask = null
+			existing.removeFromParent()
+			existing.clear()
+		}
+
+		if (!crop || crop.every(value => value === 0))
+			return
+
+		const [top, right, bottom, left] = crop
+		const activeMask = target.mask
+		target.mask = null
+		const bounds = target.getLocalBounds()
+		target.mask = activeMask
+		const x = bounds.x + bounds.width * left
+		const y = bounds.y + bounds.height * top
+		const width = bounds.width * Math.max(0, 1 - left - right)
+		const height = bounds.height * Math.max(0, 1 - top - bottom)
+
+		const mask = existing ?? new Graphics()
+		mask.clear()
+		mask.beginFill(0xffffff)
+		mask.drawRect(x, y, width, height)
+		mask.endFill()
+		target.addChild(mask)
+		target.mask = mask
+		this.#cropMasks.set(target, mask)
+	}
+
+	#applyFilters(target: Container, specs: FilterSpec[] | undefined) {
+		if (!specs?.length) {
+			target.filters = null
+			return
+		}
+
+		target.filters = specs
+			.map((spec): Filter | undefined => {
+				switch (spec.type) {
+					case "ColorMatrixFilter": {
+						const {matrix, ...params} = spec.params ?? {}
+						const filter = new ColorMatrixFilter(params)
+						if (matrix)
+							filter.matrix = matrix
+						return filter
+					}
+					case "EmbossFilter":
+						return new PixiFilters.EmbossFilter(spec.params?.strength)
+					case "PixelateFilter":
+						return new PixiFilters.PixelateFilter(spec.params?.size)
+					default: {
+						const PixiFilter = findPixiFilter(spec.type)
+						return PixiFilter ? new PixiFilter(spec.params) : undefined
+					}
+				}
+			})
+			.filter((filter): filter is Filter => !!filter)
+	}
+
+	#findOrCreate<T = Container>(layer: Layer) {
+		const object = this.#activeObjects.get(layer.id)
+		if(!object) {
+			switch (layer.kind) {
+				case 'text': {
+					const text = new Text({
+						text: layer.content,
+						style: layer.style
+					})
+					text.eventMode = "static"
+					const down = (event: FederatedPointerEvent) => this.onPointerDown.publish({event, id: layer.id, object: text})
+					const up = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: text})
+					const move = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: text})
+
+					text.on("pointerdown", down)
+					text.on("pointerup", up)
+					text.on("pointermove", move)
+
+					return this.#activeObjects
+						.set(layer.id, {
+							sprite: text,
+							dispose: () => {
+								this.onDispose.publish({id: layer.id, object: text})
+								text.off("pointermove", move)
+								text.off("pointerdown", down)
+								text.off("pointerup", up)
+							}
+						})
+						.get(layer.id)?.sprite as T
+				}
+				case 'image': {
+					const sprite = new Sprite()
+					sprite.eventMode = "static"
+					const down = (event: FederatedPointerEvent) => this.onPointerDown.publish({event, id: layer.id, object: sprite})
+					const up = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: sprite})
+					const move = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: sprite})
+
+					sprite.on("pointerdown", down)
+					sprite.on("pointermove", move)
+					sprite.on("pointerup", up)
+
+					return this.#activeObjects
+						.set(layer.id, {
+							sprite,
+							dispose: () => {
+								this.onDispose.publish({id: layer.id, object: sprite})
+								sprite.off("pointerdown", down)
+								sprite.off("pointermove", move)
+								sprite.off("pointerup", up)
+							}
+						})
+						.get(layer.id)?.sprite as T
+				}
+			}
+		} else return object.sprite as T
+	}
+
+	#collectIds(layers: Layer | Composition): Set<number> {
+		const result = new Set<number>()
+		const traverse = (node: Layer | Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node) traverse(child)
+			} else {
+				result.add(node.id)
+			}
+		}
+		traverse(layers)
+		return result
+	}
+
+	#cleanup(activeIds: Set<number>) {
+		for (const id of this.#activeObjects.keys()) {
+			if (!activeIds.has(id)) {
+				const {sprite, dispose} = this.#activeObjects.get(id)!
+				dispose()
+				sprite.destroy(true)
+				this.#activeObjects.delete(id)
+			}
+		}
+	}
+}

package/s/driver/parts/machina.ts

@@ -1,27 +1,26 @@
-import {WebMediaInfo} from "web-demuxer"
 
-type Events =
-	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
-	| {type: "info", data: WebMediaInfo}
-	| {type: "encoderQueueSize", size: number}
-
-type Handler = (event: Events) => void
+// type Events =
+// 	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+// 	| {type: "info", data: WebMediaInfo}
+// 	| {type: "encoderQueueSize", size: number}
+//
+// type Handler = (event: Events) => void
 
 export class Machina {
 	count = 0
 
-	#handlers = new Map<number, Handler>()
-
-	register(id: number, handler: Handler) {
-		this.#handlers.set(id, handler)
-	}
-
-	unregister(id: number) {
-		this.#handlers.delete(id)
-	}
-
-	dispatch(id: number, event: Events) {
-		this.#handlers.get(id)?.(event)
-	}
+	// #handlers = new Map<number, Handler>()
+	//
+	// register(id: number, handler: Handler) {
+	// 	this.#handlers.set(id, handler)
+	// }
+	//
+	// unregister(id: number) {
+	// 	this.#handlers.delete(id)
+	// }
+	//
+	// dispatch(id: number, event: Events) {
+	// 	this.#handlers.get(id)?.(event)
+	// }
 }