@omnimedia/omnitool 1.1.0-5 → 1.1.0-51

package/s/driver/driver.ts

@@ -1,7 +1,10 @@
+
+import {is} from "@e280/stz"
 import {Comrade, tune, Thread} from "@e280/comrade"
 import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
 
 import {Machina} from "./parts/machina.js"
+import {Compositor} from "./parts/compositor.js"
 import {setupDriverHost} from "./fns/host.js"
 import {loadDecoderSource} from "./utils/load-decoder-source.js"
 import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
@@ -11,19 +14,21 @@ export type DriverOptions = {
 }
 
 export class Driver {
-	static async setup(options: DriverOptions) {
+	static async setup(options?: DriverOptions) {
 		const machina = new Machina()
 		const thread = await Comrade.thread<DriverSchematic>({
 			label: "OmnitoolDriver",
-			workerUrl: options.workerUrl,
+			workerUrl: options?.workerUrl ?? "/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js",
 			setupHost: setupDriverHost(machina),
 		})
-		return new this(machina, thread)
+		const compositor = await Compositor.setup()
+		return new this(machina, thread, compositor)
 	}
 
 	constructor(
 		public machina: Machina,
-		public thread: Thread<DriverSchematic>
+		public thread: Thread<DriverSchematic>,
+		public compositor: Compositor
 	) {}
 
 	async hello() {
@@ -37,7 +42,8 @@ export class Driver {
 		})
 
 		const audioTrack = await input.getPrimaryAudioTrack()
-		return await audioTrack?.computeDuration()
+		if (!audioTrack) throw new Error("primary audio track not found")
+		return await audioTrack.computeDuration()
 	}
 
 	async getVideoDuration(source: DecoderSource) {
@@ -50,7 +56,7 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
@@ -61,56 +67,38 @@ export class Driver {
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
 			source: input.source,
 			video: videoTransform.writable,
-			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
-		}
+		return videoTransform.readable
 	}
 
-	async encode({readables, config}: EncoderInput) {
-		const handle = await window.showSaveFilePicker()
-		const writable = await handle.createWritable()
-		// making bridge because file picker writable is not transferable
-		const bridge = new WritableStream<StreamTargetChunk>({
-			async write(chunk) {
-				await writable.write(chunk)
-			},
-			async close() {
-				await writable.close()
-			}
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
+			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return audioTransform.readable
+	}
+
+	encode({audio, video, config}: EncoderInput) {
+		const {readable, writable} = new TransformStream<StreamTargetChunk, StreamTargetChunk>()
+		const transfer = [audio, video, writable].filter(is.happy)
+		const done = this.thread.work.encode[tune]({transfer})({audio, video, config, writable})
+		return {readable, done}
 	}
 
 	async composite(
 		composition: Composition,
 	) {
-		const transfer = this.#collectTransferablesFromComposition(composition)
-		return await this.thread.work.composite[tune]({transfer})(composition)
+		return await this.compositor.composite(composition)
 	}
 
-	#collectTransferablesFromComposition(composition: Composition) {
-		const transferables: Transferable[] = []
-
-		const visit = (node: Composition) => {
-			if (Array.isArray(node)) {
-				for (const child of node)
-					visit(child)
-			}
-			else if (node && typeof node === 'object' && 'kind' in node) {
-				if (node.kind === 'image' && node.frame instanceof VideoFrame)
-					transferables.push(node.frame)
-			}
-		}
-
-		visit(composition)
-		return transferables
-	}
 }
 

package/s/driver/fns/schematic.ts

@@ -1,22 +1,32 @@
 
+import {TextStyleOptions} from "pixi.js"
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Mat6} from "../../timeline/utils/matrix.js"
+import {Id} from "../../timeline/index.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
 		}): Promise<void>
 
-		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
+		}): Promise<void>
 
-		composite(input: Composition): Promise<VideoFrame>
+		encode(input: EncoderInput & {writable: WritableStream<StreamTargetChunk>}): Promise<void>
 	}
 
 	// happens on the main thread
@@ -26,20 +36,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
 export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +71,40 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
-
 export type TextLayer = {
+	id: Id
 	kind: 'text'
 	content: string
-	fontSize?: number
-	color?: string
-} & Transform
+	style?: TextStyleOptions
+	matrix?: Mat6
+}
 
 export type ImageLayer = {
+	id: Id
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+	id: Id
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	id: Id
+	kind: 'gap'
+}
+
+export type Audio = {
+	id: Id
+	kind: "audio"
+	data: AudioData
+}
 
-export type Layer = TextLayer | ImageLayer
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer
 

package/s/driver/fns/work.ts

@@ -1,234 +1,105 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {DOMAdapter, WebWorkerAdapter} from "pixi.js"
 import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {DecoderSource, DriverSchematic} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+
 export const setupDriverWork = (
 	Comrade.work<DriverSchematic>(shell => ({
 		async hello() {
 			await shell.host.world()
 		},
 
-		async decode({source, video, audio}) {
-			const loadSource = async () => {
-				if(source instanceof Blob) {
-					return new BlobSource(source)
-				} else {
-					return new UrlSource(source)
-				}
-			}
+		async decodeAudio({source, audio, start, end}) {
 			const input = new Input({
-				source: await loadSource(),
+				source: await loadSource(source),
 				formats: ALL_FORMATS
 			})
 
-			const [videoTrack, audioTrack] = await Promise.all([
-				input.getPrimaryVideoTrack(),
-				input.getPrimaryAudioTrack()
-			])
-
-			const videoDecodable = await videoTrack?.canDecode()
+			const audioTrack = await input.getPrimaryAudioTrack()
 			const audioDecodable = await audioTrack?.canDecode()
-
-			const videoWriter = video.getWriter()
 			const audioWriter = audio.getWriter()
 
-			await Promise.all([
-				(async () => {
-					if (videoDecodable && videoTrack) {
-						const sink = new VideoSampleSink(videoTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toVideoFrame()
-							await videoWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await videoWriter.close()
-					}
-				})(),
-				(async () => {
-					if (audioDecodable && audioTrack) {
-						const sink = new AudioSampleSink(audioTrack)
-						for await (const sample of sink.samples()) {
-							const frame = sample.toAudioData()
-							await audioWriter.write(frame)
-							sample.close()
-							frame.close()
-						}
-						await audioWriter.close()
-					}
-				})()
-			])
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await audioWriter.close()
+			}
 		},
 
-		async encode({readables, config, bridge}) {
-			const output = new Output({
-				format: new Mp4OutputFormat(),
-				target: new StreamTarget(bridge, {chunked: true})
+		async decodeVideo({source, video, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
 			})
-			const videoSource = new VideoSampleSource(config.video)
-			output.addVideoTrack(videoSource)
-			// since AudioSample is not transferable it fails to transfer encoder bitrate config
-			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-			const audioSource = new AudioSampleSource(config.audio)
-			output.addAudioTrack(audioSource)
-
-			await output.start()
 
-			const videoReader = readables.video.getReader()
-			const audioReader = readables.audio.getReader()
-
-			await Promise.all([
-				(async () => {
-					while (true) {
-						const {done, value} = await videoReader.read()
-						if (done) break
-						const sample = new VideoSample(value)
-						await videoSource.add(sample)
-						sample.close()
-					}
-				})(),
-				(async () => {
-					while (true) {
-						const {done, value} = await audioReader.read()
-						if (done) break
-						const sample = new AudioSample(value)
-						await audioSource.add(sample)
-						sample.close()
-						value.close()
-					}
-				})()
-			])
+			const videoTrack = await input.getPrimaryVideoTrack()
+			const videoDecodable = await videoTrack?.canDecode()
+			const videoWriter = video.getWriter()
 
-			await output.finalize()
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
 		},
 
-		async composite(composition) {
-			const {stage, renderer} = await renderPIXI(1920, 1080)
-			stage.removeChildren()
-
-			const {baseFrame, disposables} = await renderLayer(composition, stage)
-			renderer.render(stage)
-
-			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-			// if its very slow on eg edge try chrome
-			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: baseFrame?.timestamp,
-				duration: baseFrame?.duration ?? undefined,
+		async encode({video, audio, config, writable}) {
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new StreamTarget(writable, {chunked: true})
 			})
 
-			baseFrame?.close()
-			renderer.clear()
+			async function encodeVideo() {
+				if(!video) return
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				for await (const frame of video) {
+					const sample = new VideoSample(frame)
+					await videoSource.add(sample)
+					sample.close()
+				}
+			}
 
-			for (const disposable of disposables) {
-				disposable.destroy(true)
+			async function encodeAudio() {
+				if(!audio) return
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				for await (const data of audio) {
+					const sample = new AudioSample(data)
+					await audioSource.add(sample)
+					sample.close()
+					data.close()
+				}
 			}
 
-			shell.transfer = [frame]
-			return frame
-		}
+			const audioTask = encodeAudio()
+			const videoTask = encodeVideo()
+
+			await output.start()
+			await Promise.all([audioTask, videoTask])
+			await output.finalize()
+		},
 	}))
 )
 
-// TODO suspicious global, probably bad
-let pixi: {
-	renderer: Renderer
-	stage: Container
-} | null = null
-
-async function renderPIXI(width: number, height: number) {
-	if (pixi)
-		return pixi
-
-	const renderer = await autoDetectRenderer({
-		width,
-		height,
-		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
-		background: "black",
-		preferWebGLVersion: 2
-	})
-
-	const stage = new Container()
-	pixi = {renderer, stage}
-
-	return pixi
-}
-
-type RenderableObject = Sprite | Text | Texture
-
-async function renderLayer(
-	layer: Layer | Composition,
-	parent: Container,
-	disposables: RenderableObject[] = []
-) {
-	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
-		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
-		}
-		return {baseFrame, disposables}
-	}
-
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
-	}
-
-	switch (layer.kind) {
-		case 'text':
-			return renderTextLayer(layer, parent, disposables)
-		case 'image':
-			return renderImageLayer(layer, parent, disposables)
-		default:
-			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
-	}
-}
-
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
-
-function renderTextLayer(
-	layer: Extract<Layer, {kind: 'text'}>,
-	parent: Container,
-	disposables: RenderableObject[]
-) {
-	const text = new Text({
-		text: layer.content,
-		style: {
-			fontFamily: 'sans-serif',
-			fontSize: layer.fontSize ?? 48,
-			fill: layer.color ?? 'white'
-		}
-	})
-	applyTransform(text, layer)
-	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
-}
-
-function renderImageLayer(
-	layer: Extract<Layer, {kind: 'image'}>,
-	parent: Container,
-	disposables: RenderableObject[]
-) {
-	const texture = Texture.from(layer.frame)
-	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
-	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
-}
-
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
-}