@omnimedia/omnitool 1.1.0-8 → 1.1.0-80

package/s/driver/driver.ts

@@ -1,7 +1,10 @@
+
+import {is} from "@e280/stz"
 import {Comrade, tune, Thread} from "@e280/comrade"
 import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
 
 import {Machina} from "./parts/machina.js"
+import {Compositor} from "./parts/compositor.js"
 import {setupDriverHost} from "./fns/host.js"
 import {loadDecoderSource} from "./utils/load-decoder-source.js"
 import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
@@ -11,19 +14,21 @@ export type DriverOptions = {
 }
 
 export class Driver {
-	static async setup(options: DriverOptions) {
+	static async setup(options?: DriverOptions) {
 		const machina = new Machina()
 		const thread = await Comrade.thread<DriverSchematic>({
 			label: "OmnitoolDriver",
-			workerUrl: options.workerUrl,
+			workerUrl: options?.workerUrl ?? "/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js",
 			setupHost: setupDriverHost(machina),
 		})
-		return new this(machina, thread)
+		const compositor = await Compositor.setup()
+		return new this(machina, thread, compositor)
 	}
 
 	constructor(
 		public machina: Machina,
-		public thread: Thread<DriverSchematic>
+		public thread: Thread<DriverSchematic>,
+		public compositor: Compositor
 	) {}
 
 	async hello() {
@@ -37,7 +42,8 @@ export class Driver {
 		})
 
 		const audioTrack = await input.getPrimaryAudioTrack()
-		return await audioTrack?.computeDuration()
+		if (!audioTrack) throw new Error("primary audio track not found")
+		return await audioTrack.computeDuration()
 	}
 
 	async getVideoDuration(source: DecoderSource) {
@@ -52,73 +58,68 @@ export class Driver {
 
 	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
+		const {port1, port2} = new MessageChannel()
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
 				const frame = await input.onFrame?.(chunk) ?? chunk
-				// below code is to prevent mem leaks and hardware accelerated decoder stall
 				lastFrame?.close()
 				controller.enqueue(frame)
 				lastFrame = frame
 			}
 		})
-		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			video: videoTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return videoTransform.readable
+		return {
+			readable: videoTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
 	decodeAudio(input: DecoderInput) {
 		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+		const {port1, port2} = new MessageChannel()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			audio: audioTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return audioTransform.readable
+		return {
+			readable: audioTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
-	async encode({video, audio, config}: EncoderInput) {
-		const handle = await window.showSaveFilePicker()
-		const writable = await handle.createWritable()
-		// making bridge because file picker writable is not transferable
-		const bridge = new WritableStream<StreamTargetChunk>({
-			async write(chunk) {
-				await writable.write(chunk)
-			},
-			async close() {
-				await writable.close()
-			}
-		})
-		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
+	encode({audio, video, config}: EncoderInput) {
+		const {readable, writable} = new TransformStream<StreamTargetChunk, StreamTargetChunk>()
+		const transfer = [audio, video, writable].filter(is.happy)
+		const done = this.thread.work.encode[tune]({transfer})({audio, video, config, writable})
+		return {readable, done}
 	}
 
 	async composite(
 		composition: Composition,
 	) {
-		const transfer = this.#collectTransferablesFromComposition(composition)
-		return await this.thread.work.composite[tune]({transfer})(composition)
+		return await this.compositor.composite(composition)
 	}
 
-	#collectTransferablesFromComposition(composition: Composition) {
-		const transferables: Transferable[] = []
-
-		const visit = (node: Composition) => {
-			if (Array.isArray(node)) {
-				for (const child of node)
-					visit(child)
-			}
-			else if (node && typeof node === 'object' && 'kind' in node) {
-				if (node.kind === 'image' && node.frame instanceof VideoFrame)
-					transferables.push(node.frame)
-			}
-		}
-
-		visit(composition)
-		return transferables
-	}
 }
 

package/s/driver/fns/schematic.ts

@@ -1,8 +1,12 @@
 
+import {TextStyleOptions} from "pixi.js"
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Id} from "../../timeline/index.js"
+import {Crop} from "../../timeline/parts/item.js"
 import {Mat6} from "../../timeline/utils/matrix.js"
+import {FilterParams, FilterType} from "../../timeline/parts/filters.js"
 
 export type DriverSchematic = AsSchematic<{
 
@@ -12,6 +16,7 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeAudio(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			audio: WritableStream<AudioData>
 			start?: number
 			end?: number
@@ -19,14 +24,13 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeVideo(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			video: WritableStream<VideoFrame>
 			start?: number
 			end?: number
 		}): Promise<void>
 
-		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
-
-		composite(input: Composition): Promise<VideoFrame>
+		encode(input: EncoderInput & {writable: WritableStream<StreamTargetChunk>}): Promise<void>
 	}
 
 	// happens on the main thread
@@ -71,21 +75,36 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
+export type FilterSpec = {
+	[TFilter in FilterType]: {
+		type: TFilter
+		params?: FilterParams<TFilter>
+	}
+}[FilterType]
+
 export type TextLayer = {
+	id: Id
 	kind: 'text'
 	content: string
-	fontSize?: number
-	color?: string
+	style?: TextStyleOptions
 	matrix?: Mat6
+	alpha?: number
+	crop?: Crop
+	filters?: FilterSpec[]
 }
 
 export type ImageLayer = {
+	id: Id
 	kind: 'image'
 	frame: VideoFrame
 	matrix?: Mat6
+	alpha?: number
+	crop?: Crop
+	filters?: FilterSpec[]
 }
 
 export type TransitionLayer = {
+	id: Id
   kind: 'transition'
   name: string
   progress: number
@@ -94,13 +113,14 @@ export type TransitionLayer = {
 }
 
 export type GapLayer = {
+	id: Id
 	kind: 'gap'
 }
 
 export type Audio = {
+	id: Id
 	kind: "audio"
 	data: AudioData
 }
 
 export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer
-

package/s/driver/fns/work.ts

@@ -1,12 +1,9 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
-import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
-import {makeTransition} from "../../features/transition/transition.js"
-import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
+import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink,
+	AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-DOMAdapter.set(WebWorkerAdapter)
+import {DecoderSource, DriverSchematic} from "./schematic.js"
 
 const loadSource = async (source: DecoderSource) => {
 	if(source instanceof Blob) {
@@ -22,7 +19,7 @@ export const setupDriverWork = (
 			await shell.host.world()
 		},
 
-		async decodeAudio({source, audio, start, end}) {
+		async decodeAudio({source, audio, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -32,19 +29,29 @@ export const setupDriverWork = (
 			const audioDecodable = await audioTrack?.canDecode()
 			const audioWriter = audio.getWriter()
 
-			if (audioDecodable && audioTrack) {
-				const sink = new AudioSampleSink(audioTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toAudioData()
-					await audioWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await audioWriter.close()
+			if(!audioDecodable || !audioTrack)
+				return
+
+			const sink = new AudioSampleSink(audioTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
+			}
+
+			for await (const sample of samples) {
+				const frame = sample.toAudioData()
+				sample.close()
+				await audioWriter.write(frame)
+				frame.close()
 			}
+
+			await audioWriter.close()
 		},
 
-		async decodeVideo({source, video, start, end}) {
+		async decodeVideo({source, video, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -54,196 +61,65 @@ export const setupDriverWork = (
 			const videoDecodable = await videoTrack?.canDecode()
 			const videoWriter = video.getWriter()
 
-			if (videoDecodable && videoTrack) {
-				const sink = new VideoSampleSink(videoTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toVideoFrame()
-					await videoWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await videoWriter.close()
+			if(!videoDecodable || !videoTrack)
+				return
+
+			const sink = new VideoSampleSink(videoTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				await samples.return()
+				input.dispose()
+				cancel.close()
+			}
+
+			for await (const sample of samples) {
+				const frame = sample.toVideoFrame()
+				sample.close()
+				await videoWriter.write(frame)
+				frame.close()
 			}
+
+			await videoWriter.close()
 		},
 
-		async encode({video, audio, config, bridge}) {
+		async encode({video, audio, config, writable}) {
 			const output = new Output({
 				format: new Mp4OutputFormat(),
-				target: new StreamTarget(bridge, {chunked: true})
+				target: new StreamTarget(writable, {chunked: true})
 			})
-			// since AudioSample is not transferable it fails to transfer encoder bitrate config
-			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-
-			const promises = []
 
-			if(video) {
+			async function encodeVideo() {
+				if(!video) return
 				const videoSource = new VideoSampleSource(config.video)
 				output.addVideoTrack(videoSource)
-				const videoReader = video.getReader()
-				promises.push((async () => {
-					while (true) {
-						const {done, value} = await videoReader.read()
-						if (done) break
-						const sample = new VideoSample(value)
-						await videoSource.add(sample)
-						sample.close()
-					}
-				})())
+				for await (const frame of video) {
+					const sample = new VideoSample(frame)
+					await videoSource.add(sample)
+					sample.close()
+					frame.close()
+				}
 			}
 
-			if(audio) {
+			async function encodeAudio() {
+				if(!audio) return
 				const audioSource = new AudioSampleSource(config.audio)
 				output.addAudioTrack(audioSource)
-				const audioReader = audio.getReader()
-				promises.push((async () => {
-					while (true) {
-						const {done, value} = await audioReader.read()
-						if (done) break
-						const sample = new AudioSample(value)
-						await audioSource.add(sample)
-						sample.close()
-						value.close()
-					}
-				})())
+				for await (const data of audio) {
+					const sample = new AudioSample(data)
+					await audioSource.add(sample)
+					sample.close()
+					data.close()
+				}
 			}
 
+			const audioTask = encodeAudio()
+			const videoTask = encodeVideo()
+
 			await output.start()
-			await Promise.all(promises)
+			await Promise.all([audioTask, videoTask])
 			await output.finalize()
 		},
-
-		async composite(composition) {
-			const {stage, renderer} = await renderPIXI(1920, 1080)
-			stage.removeChildren()
-
-			const {dispose} = await renderLayer(composition, stage)
-			renderer.render(stage)
-
-			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-			// if its very slow on eg edge try chrome
-			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: 0,
-				duration: 0,
-			})
-
-			renderer.clear()
-			dispose()
-
-			shell.transfer = [frame]
-			return frame
-		}
 	}))
 )
 
-// TODO suspicious global, probably bad
-let pixi: {
-	renderer: Renderer
-	stage: Container
-} | null = null
-
-async function renderPIXI(width: number, height: number) {
-	if (pixi)
-		return pixi
-
-	const renderer = await autoDetectRenderer({
-		width,
-		height,
-		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
-		background: "black",
-		preferWebGLVersion: 2
-	})
-
-	const stage = new Container()
-	pixi = {renderer, stage}
-
-	return pixi
-}
-
-const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
-
-type RenderableObject = Sprite | Text | Texture
-
-async function renderLayer(
-	layer: Layer | Composition,
-	parent: Container,
-) {
-	if (Array.isArray(layer)) {
-		const disposers: (() => void)[] = []
-		for (const child of layer) {
-			const result = await renderLayer(child, parent)
-			disposers.push(result.dispose)
-		}
-		return {dispose: () => disposers.forEach(d => d())}
-	}
-
-	switch (layer.kind) {
-		case 'text':
-			return renderTextLayer(layer, parent)
-		case 'image':
-			return renderImageLayer(layer, parent)
-		case 'transition':
-			return renderTransitionLayer(layer, parent)
-		case 'gap': {
-			pixi?.renderer.clear()
-			return {dispose: () => {}}
-		}
-		default:
-			console.warn('Unknown layer kind', (layer as any).kind)
-			return {dispose: () => {}}
-	}
-}
-
-function renderTextLayer(
-	layer: Extract<Layer, {kind: 'text'}>,
-	parent: Container,
-) {
-	const text = new Text({
-		text: layer.content,
-		style: {
-			fontFamily: 'sans-serif',
-			fontSize: layer.fontSize ?? 48,
-			fill: layer.color ?? 'white'
-		}
-	})
-	applyTransform(text, layer.matrix)
-	parent.addChild(text)
-	return {dispose: () => text.destroy(true)}
-}
-
-function renderImageLayer(
-	layer: Extract<Layer, {kind: 'image'}>,
-	parent: Container,
-) {
-	const texture = Texture.from(layer.frame)
-	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer.matrix)
-	parent.addChild(sprite)
-	return {dispose: () => {
-		sprite.destroy(true)
-		texture.destroy(true)
-		layer.frame.close()
-	}}
-}
-
-function renderTransitionLayer(
-	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
-	parent: Container,
-) {
-	const transition = transitions.get(name) ??
-		(transitions.set(name, makeTransition({
-			name: "circle",
-			renderer: pixi!.renderer
-		})),
-	  transitions.get(name)!
-	)
-	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
-	const sprite = new Sprite(texture)
-	parent.addChild(sprite)
-	return {dispose: () => sprite.destroy(false)}
-}
-
-function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
-  if (!worldMatrix) return
-	const mx = mat6ToMatrix(worldMatrix)
-  target.setFromMatrix(mx)
-}