@omnimedia/omnitool 1.0.0 → 1.1.0-10

package/s/driver/driver.test.ts

@@ -0,0 +1,15 @@
+
+import {Driver} from "./driver.js"
+import {Science, test, expect} from "@e280/science"
+
+const workerUrl = new URL("./driver.worker.bundle.js", import.meta.url)
+
+export default Science.suite({
+	"driver hello world": test(async() => {
+		const driver = await Driver.setup({workerUrl})
+		expect(driver.machina.count).is(0)
+		await driver.thread.work.hello()
+		expect(driver.machina.count).is(1)
+	}),
+})
+

package/s/driver/driver.ts

@@ -0,0 +1,124 @@
+import {Comrade, tune, Thread} from "@e280/comrade"
+import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
+
+import {Machina} from "./parts/machina.js"
+import {setupDriverHost} from "./fns/host.js"
+import {loadDecoderSource} from "./utils/load-decoder-source.js"
+import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
+
+export type DriverOptions = {
+	workerUrl: URL | string
+}
+
+export class Driver {
+	static async setup(options: DriverOptions) {
+		const machina = new Machina()
+		const thread = await Comrade.thread<DriverSchematic>({
+			label: "OmnitoolDriver",
+			workerUrl: options.workerUrl,
+			setupHost: setupDriverHost(machina),
+		})
+		return new this(machina, thread)
+	}
+
+	constructor(
+		public machina: Machina,
+		public thread: Thread<DriverSchematic>
+	) {}
+
+	async hello() {
+		return this.thread.work.hello()
+	}
+
+	async getAudioDuration(source: DecoderSource) {
+		const input = new Input({
+			source: await loadDecoderSource(source),
+			formats: ALL_FORMATS
+		})
+
+		const audioTrack = await input.getPrimaryAudioTrack()
+		return await audioTrack?.computeDuration()
+	}
+
+	async getVideoDuration(source: DecoderSource) {
+		const input = new Input({
+			source: await loadDecoderSource(source),
+			formats: ALL_FORMATS
+		})
+
+		const videoTrack = await input.getPrimaryVideoTrack()
+		return await videoTrack?.computeDuration()
+	}
+
+	decodeVideo(input: DecoderInput) {
+		let lastFrame: VideoFrame | null = null
+		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
+			async transform(chunk, controller) {
+				const frame = await input.onFrame?.(chunk) ?? chunk
+				// below code is to prevent mem leaks and hardware accelerated decoder stall
+				lastFrame?.close()
+				controller.enqueue(frame)
+				lastFrame = frame
+			}
+		})
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
+			source: input.source,
+			video: videoTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return videoTransform.readable
+	}
+
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
+			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return audioTransform.readable
+	}
+
+	async encode({video, audio, config}: EncoderInput) {
+		const handle = await window.showSaveFilePicker()
+		const writable = await handle.createWritable()
+		// making bridge because file picker writable is not transferable
+		const bridge = new WritableStream<StreamTargetChunk>({
+			async write(chunk) {
+				await writable.write(chunk)
+			},
+			async close() {
+				await writable.close()
+			}
+		})
+		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
+	}
+
+	async composite(
+		composition: Composition,
+	) {
+		const transfer = this.#collectTransferablesFromComposition(composition)
+		return await this.thread.work.composite[tune]({transfer})(composition)
+	}
+
+	#collectTransferablesFromComposition(composition: Composition) {
+		const transferables: Transferable[] = []
+
+		const visit = (node: Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node)
+					visit(child)
+			}
+			else if (node && typeof node === 'object' && 'kind' in node) {
+				if (node.kind === 'image' && node.frame instanceof VideoFrame)
+					transferables.push(node.frame)
+			}
+		}
+
+		visit(composition)
+		return transferables
+	}
+}
+

package/s/driver/driver.worker.bundle.ts

@@ -0,0 +1,7 @@
+
+import {Comrade} from "@e280/comrade"
+import {setupDriverWork} from "./fns/work.js"
+import {DriverSchematic} from "./fns/schematic.js"
+
+await Comrade.worker<DriverSchematic>(setupDriverWork)
+

package/s/driver/fns/host.ts

@@ -0,0 +1,13 @@
+
+import {Comrade} from "@e280/comrade"
+import {Machina} from "../parts/machina.js"
+import {DriverSchematic} from "./schematic.js"
+
+export const setupDriverHost = (machina: Machina) => (
+	Comrade.host<DriverSchematic>(_shell => ({
+		async world() {
+			machina.count++
+		},
+	}))
+)
+

package/s/driver/fns/schematic.ts

@@ -0,0 +1,106 @@
+
+import {AsSchematic} from "@e280/comrade"
+import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
+
+import {Mat6} from "../../timeline/utils/matrix.js"
+
+export type DriverSchematic = AsSchematic<{
+
+	// happens on the web worker
+	work: {
+		hello(): Promise<void>
+
+		decodeAudio(input: {
+			source: DecoderSource
+			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
+
+		composite(input: Composition): Promise<VideoFrame>
+	}
+
+	// happens on the main thread
+	host: {
+		world(): Promise<void>
+	}
+}>
+
+export interface EncoderInput {
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
+}
+
+export type DecoderSource = Blob | string | URL
+
+export interface DecoderInput {
+	source: DecoderSource
+	start?: number
+	end?: number
+	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
+}
+
+export interface MuxOpts {
+	config: {
+		video: {
+			width: number
+			height: number
+		},
+		audio?: {
+			codec: "opus" | "aac"
+			numberOfChannels: number
+			sampleRate: number
+		}
+	}
+}
+
+export type Composition = Layer | (Layer | Composition)[]
+
+export type TextLayer = {
+	kind: 'text'
+	content: string
+	fontSize?: number
+	color?: string
+	matrix?: Mat6
+}
+
+export type ImageLayer = {
+	kind: 'image'
+	frame: VideoFrame
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	kind: 'gap'
+}
+
+export type Audio = {
+	kind: "audio"
+	data: AudioData
+}
+
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer
+

package/s/driver/fns/work.ts

@@ -0,0 +1,249 @@
+import {Comrade} from "@e280/comrade"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
+import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
+
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
+
+DOMAdapter.set(WebWorkerAdapter)
+
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+
+export const setupDriverWork = (
+	Comrade.work<DriverSchematic>(shell => ({
+		async hello() {
+			await shell.host.world()
+		},
+
+		async decodeAudio({source, audio, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
+
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await audioWriter.close()
+			}
+		},
+
+		async decodeVideo({source, video, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const videoTrack = await input.getPrimaryVideoTrack()
+			const videoDecodable = await videoTrack?.canDecode()
+			const videoWriter = video.getWriter()
+
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
+		},
+
+		async encode({video, audio, config, bridge}) {
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new StreamTarget(bridge, {chunked: true})
+			})
+			// since AudioSample is not transferable it fails to transfer encoder bitrate config
+			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
+
+			const promises = []
+
+			if(video) {
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				const videoReader = video.getReader()
+				promises.push((async () => {
+					while (true) {
+						const {done, value} = await videoReader.read()
+						if (done) break
+						const sample = new VideoSample(value)
+						await videoSource.add(sample)
+						sample.close()
+					}
+				})())
+			}
+
+			if(audio) {
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				const audioReader = audio.getReader()
+				promises.push((async () => {
+					while (true) {
+						const {done, value} = await audioReader.read()
+						if (done) break
+						const sample = new AudioSample(value)
+						await audioSource.add(sample)
+						sample.close()
+						value.close()
+					}
+				})())
+			}
+
+			await output.start()
+			await Promise.all(promises)
+			await output.finalize()
+		},
+
+		async composite(composition) {
+			const {stage, renderer} = await renderPIXI(1920, 1080)
+			stage.removeChildren()
+
+			const {dispose} = await renderLayer(composition, stage)
+			renderer.render(stage)
+
+			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+			// if its very slow on eg edge try chrome
+			const frame = new VideoFrame(renderer.canvas, {
+				timestamp: 0,
+				duration: 0,
+			})
+
+			renderer.clear()
+			dispose()
+
+			shell.transfer = [frame]
+			return frame
+		}
+	}))
+)
+
+// TODO suspicious global, probably bad
+let pixi: {
+	renderer: Renderer
+	stage: Container
+} | null = null
+
+async function renderPIXI(width: number, height: number) {
+	if (pixi)
+		return pixi
+
+	const renderer = await autoDetectRenderer({
+		width,
+		height,
+		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+		background: "black",
+		preferWebGLVersion: 2
+	})
+
+	const stage = new Container()
+	pixi = {renderer, stage}
+
+	return pixi
+}
+
+const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+
+type RenderableObject = Sprite | Text | Texture
+
+async function renderLayer(
+	layer: Layer | Composition,
+	parent: Container,
+) {
+	if (Array.isArray(layer)) {
+		const disposers: (() => void)[] = []
+		for (const child of layer) {
+			const result = await renderLayer(child, parent)
+			disposers.push(result.dispose)
+		}
+		return {dispose: () => disposers.forEach(d => d())}
+	}
+
+	switch (layer.kind) {
+		case 'text':
+			return renderTextLayer(layer, parent)
+		case 'image':
+			return renderImageLayer(layer, parent)
+		case 'transition':
+			return renderTransitionLayer(layer, parent)
+		case 'gap': {
+			pixi?.renderer.clear()
+			return {dispose: () => {}}
+		}
+		default:
+			console.warn('Unknown layer kind', (layer as any).kind)
+			return {dispose: () => {}}
+	}
+}
+
+function renderTextLayer(
+	layer: Extract<Layer, {kind: 'text'}>,
+	parent: Container,
+) {
+	const text = new Text({
+		text: layer.content,
+		style: {
+			fontFamily: 'sans-serif',
+			fontSize: layer.fontSize ?? 48,
+			fill: layer.color ?? 'white'
+		}
+	})
+	applyTransform(text, layer.matrix)
+	parent.addChild(text)
+	return {dispose: () => text.destroy(true)}
+}
+
+function renderImageLayer(
+	layer: Extract<Layer, {kind: 'image'}>,
+	parent: Container,
+) {
+	const texture = Texture.from(layer.frame)
+	const sprite = new Sprite(texture)
+	applyTransform(sprite, layer.matrix)
+	parent.addChild(sprite)
+	return {dispose: () => {
+		sprite.destroy(true)
+		texture.destroy(true)
+		layer.frame.close()
+	}}
+}
+
+function renderTransitionLayer(
+	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+	parent: Container,
+) {
+	const transition = transitions.get(name) ??
+		(transitions.set(name, makeTransition({
+			name: "circle",
+			renderer: pixi!.renderer
+		})),
+	  transitions.get(name)!
+	)
+	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+	const sprite = new Sprite(texture)
+	parent.addChild(sprite)
+	return {dispose: () => sprite.destroy(false)}
+}
+
+function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  if (!worldMatrix) return
+	const mx = mat6ToMatrix(worldMatrix)
+  target.setFromMatrix(mx)
+}

package/s/driver/parts/constants.ts

@@ -0,0 +1,17 @@
+// for later: https://github.com/gpac/mp4box.js/issues/243
+export const encoderDefaultConfig: VideoEncoderConfig = {
+	codec: "avc1.640034",
+	avc: {format: "annexb"},
+	width: 1280,
+	height: 720,
+	bitrate: 9_000_000, // 9 Mbps
+	framerate: 60,
+	bitrateMode: "variable",
+	hardwareAcceleration: "no-preference" // prefer-hardware seems like 2x slower from what i been testing
+}
+
+export const audioEncoderDefaultConfig: AudioEncoderConfig = {
+	codec: "opus",
+	numberOfChannels: 2,
+	sampleRate: 44100
+}

package/s/driver/parts/machina.ts

@@ -0,0 +1,27 @@
+import {WebMediaInfo} from "web-demuxer"
+
+type Events =
+	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+	| {type: "info", data: WebMediaInfo}
+	| {type: "encoderQueueSize", size: number}
+
+type Handler = (event: Events) => void
+
+export class Machina {
+	count = 0
+
+	#handlers = new Map<number, Handler>()
+
+	register(id: number, handler: Handler) {
+		this.#handlers.set(id, handler)
+	}
+
+	unregister(id: number) {
+		this.#handlers.delete(id)
+	}
+
+	dispatch(id: number, event: Events) {
+		this.#handlers.get(id)?.(event)
+	}
+}
+

package/s/driver/utils/load-decoder-source.ts

@@ -0,0 +1,12 @@
+import {BlobSource, UrlSource} from "mediabunny"
+import {DecoderSource} from "../fns/schematic.js"
+
+// only streamable sources
+export async function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource> {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
+

package/s/driver/utils/sleep.ts

@@ -0,0 +1,3 @@
+export function sleep(ms: number) {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}

package/s/features/speech/transcribe/default-spec.ts

@@ -0,0 +1,11 @@
+
+import {TranscriberSpec} from "./types.js"
+
+export const defaultTranscriberSpec = (): TranscriberSpec => ({
+	model: "onnx-community/whisper-tiny_timestamped",
+	dtype: "q4",
+	device: "wasm",
+	chunkLength: 20,
+	strideLength: 3,
+})
+

package/s/features/speech/transcribe/parts/load-pipe.ts

@@ -0,0 +1,19 @@
+
+import {pipeline} from "@huggingface/transformers"
+
+import {TranscriberPipeOptions} from "../types.js"
+
+export async function loadPipe(options: TranscriberPipeOptions) {
+	const {spec, onLoading} = options
+
+	const pipe = await pipeline("automatic-speech-recognition", spec.model, {
+		device: spec.device,
+		dtype: spec.dtype,
+		progress_callback: (data: any) => {
+			onLoading({total: data.total, progress: data.progress})
+		},
+	})
+
+	return pipe
+}
+

package/s/features/speech/transcribe/parts/prep-audio.ts

@@ -0,0 +1,23 @@
+
+import {Driver} from "../../../../driver/driver.js"
+
+export async function prepAudio(driver: Driver, source: Blob) {
+	const arrayBuffer = await source.arrayBuffer()
+	const audioCTX = new AudioContext({sampleRate: 16000})
+	const audioData = await audioCTX.decodeAudioData(arrayBuffer)
+	let audio: Float32Array
+	if (audioData.numberOfChannels === 2) {
+		const SCALING_FACTOR = Math.sqrt(2)
+		const left = audioData.getChannelData(0)
+		const right = audioData.getChannelData(1)
+		audio = new Float32Array(left.length)
+		for (let i = 0; i < audioData.length; ++i) {
+			audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
+		}
+	} else {
+		audio = audioData.getChannelData(0)
+	}
+	const duration = await driver.getAudioDuration(source)
+	return {audio, duration}
+}
+