@omnimedia/omnitool 1.1.0-3 → 1.1.0-30

package/s/driver/fns/work.ts

@@ -1,142 +1,141 @@
 import {Comrade} from "@e280/comrade"
-import {
-	Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample,
-	AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource
-} from "mediabunny"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
+import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
-export const setupDriverWork = Comrade.work<DriverSchematic>(({host}, rig) => ({
-
-	async hello() {
-		await host.world()
-	},
+const loadSource = async (source: DecoderSource) => {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
+	} else {
+		return new UrlSource(source)
+	}
+}
 
-	async decode({source, video, audio}) {
-		const loadSource = async () => {
-			if(source instanceof FileSystemFileHandle) {
-				const file = await source.getFile()
-				return new BlobSource(file)
-			} else {
-				return new UrlSource(source)
+export const setupDriverWork = (
+	Comrade.work<DriverSchematic>(shell => ({
+		async hello() {
+			await shell.host.world()
+		},
+
+		async decodeAudio({source, audio, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const audioTrack = await input.getPrimaryAudioTrack()
+			const audioDecodable = await audioTrack?.canDecode()
+			const audioWriter = audio.getWriter()
+
+			if (audioDecodable && audioTrack) {
+				const sink = new AudioSampleSink(audioTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toAudioData()
+					await audioWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await audioWriter.close()
 			}
-		}
-		const input = new Input({
-			source: await loadSource(),
-			formats: ALL_FORMATS
-		})
-
-		const [videoTrack, audioTrack] = await Promise.all([
-			input.getPrimaryVideoTrack(),
-			input.getPrimaryAudioTrack()
-		])
-
-		const videoDecodable = await videoTrack?.canDecode()
-		const audioDecodable = await audioTrack?.canDecode()
-
-		const videoWriter = video.getWriter()
-		const audioWriter = audio.getWriter()
-
-		await Promise.all([
-			(async () => {
-				if (videoDecodable && videoTrack) {
-					const sink = new VideoSampleSink(videoTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toVideoFrame()
-						await videoWriter.write(frame)
+		},
+
+		async decodeVideo({source, video, start, end}) {
+			const input = new Input({
+				source: await loadSource(source),
+				formats: ALL_FORMATS
+			})
+
+			const videoTrack = await input.getPrimaryVideoTrack()
+			const videoDecodable = await videoTrack?.canDecode()
+			const videoWriter = video.getWriter()
+
+			if (videoDecodable && videoTrack) {
+				const sink = new VideoSampleSink(videoTrack)
+				for await (const sample of sink.samples(start, end)) {
+					const frame = sample.toVideoFrame()
+					await videoWriter.write(frame)
+					sample.close()
+					frame.close()
+				}
+				await videoWriter.close()
+			}
+		},
+
+		async encode({video, audio, config, bridge}) {
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new StreamTarget(bridge, {chunked: true})
+			})
+			// since AudioSample is not transferable it fails to transfer encoder bitrate config
+			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
+
+			const promises = []
+
+			if(video) {
+				const videoSource = new VideoSampleSource(config.video)
+				output.addVideoTrack(videoSource)
+				const videoReader = video.getReader()
+				promises.push((async () => {
+					while (true) {
+						const {done, value} = await videoReader.read()
+						if (done) break
+						const sample = new VideoSample(value)
+						await videoSource.add(sample)
 						sample.close()
-						frame.close()
 					}
-					await videoWriter.close()
-				}
-			})(),
-			(async () => {
-				if (audioDecodable && audioTrack) {
-					const sink = new AudioSampleSink(audioTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toAudioData()
-						await audioWriter.write(frame)
+				})())
+			}
+
+			if(audio) {
+				const audioSource = new AudioSampleSource(config.audio)
+				output.addAudioTrack(audioSource)
+				const audioReader = audio.getReader()
+				promises.push((async () => {
+					while (true) {
+						const {done, value} = await audioReader.read()
+						if (done) break
+						const sample = new AudioSample(value)
+						await audioSource.add(sample)
 						sample.close()
-						frame.close()
+						value.close()
 					}
-					await audioWriter.close()
-				}
-			})()
-		])
-	},
-
-	async encode({readables, config, bridge}) {
-		const output = new Output({
-			format: new Mp4OutputFormat(),
-			target: new StreamTarget(bridge, {chunked: true})
-		})
-		const videoSource = new VideoSampleSource(config.video)
-		output.addVideoTrack(videoSource)
-		// since AudioSample is not transferable it fails to transfer encoder bitrate config
-		// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-		const audioSource = new AudioSampleSource(config.audio)
-		output.addAudioTrack(audioSource)
-
-		await output.start()
-
-		const videoReader = readables.video.getReader()
-		const audioReader = readables.audio.getReader()
-
-		await Promise.all([
-			(async () => {
-				while (true) {
-					const {done, value} = await videoReader.read()
-					if (done) break
-					const sample = new VideoSample(value)
-					await videoSource.add(sample)
-					sample.close()
-				}
-			})(),
-			(async () => {
-				while (true) {
-					const {done, value} = await audioReader.read()
-					if (done) break
-					const sample = new AudioSample(value)
-					await audioSource.add(sample)
-					sample.close()
-					value.close()
-				}
-			})()
-		])
+				})())
+			}
 
-		await output.finalize()
-	},
+			await output.start()
+			await Promise.all(promises)
+			await output.finalize()
+		},
 
-	async composite(composition) {
-		const {stage, renderer} = await renderPIXI(1920, 1080)
-		stage.removeChildren()
+		async composite(composition) {
+			const {stage, renderer} = await renderPIXI(1920, 1080)
+			stage.removeChildren()
 
-		const {baseFrame, disposables} = await renderLayer(composition, stage)
-		renderer.render(stage)
+			const {dispose} = await renderLayer(composition, stage)
+			renderer.render(stage)
 
-		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-		// if its very slow on eg edge try chrome
-		const frame = new VideoFrame(renderer.canvas, {
-			timestamp: baseFrame?.timestamp,
-			duration: baseFrame?.duration ?? undefined,
-		})
+			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+			// if its very slow on eg edge try chrome
+			const frame = new VideoFrame(renderer.canvas, {
+				timestamp: 0,
+				duration: 0,
+			})
 
-		baseFrame?.close()
-		renderer.clear()
+			renderer.clear()
+			dispose()
 
-		for (const disposable of disposables) {
-			disposable.destroy(true)
+			shell.transfer = [frame]
+			return frame
 		}
+	}))
+)
 
-		rig.transfer = [frame]
-		return frame
-	}
-}))
-
-
+// TODO suspicious global, probably bad
 let pixi: {
 	renderer: Renderer
 	stage: Container
@@ -160,78 +159,88 @@ async function renderPIXI(width: number, height: number) {
 	return pixi
 }
 
+const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+
 type RenderableObject = Sprite | Text | Texture
 
 async function renderLayer(
 	layer: Layer | Composition,
 	parent: Container,
-	disposables: RenderableObject[] = []
 ) {
 	if (Array.isArray(layer)) {
-		let baseFrame: VideoFrame | undefined
+		layer.reverse()
+		const disposers: (() => void)[] = []
 		for (const child of layer) {
-			const result = await renderLayer(child, parent, disposables)
-			baseFrame ??= result.baseFrame
+			const result = await renderLayer(child, parent)
+			disposers.push(result.dispose)
 		}
-		return {baseFrame, disposables}
-	}
-
-	if (!isRenderableLayer(layer)) {
-		console.warn('Invalid layer', layer)
-		return {disposables}
+		return {dispose: () => disposers.forEach(d => d())}
 	}
 
 	switch (layer.kind) {
 		case 'text':
-			return renderTextLayer(layer, parent, disposables)
+			return renderTextLayer(layer, parent)
 		case 'image':
-			return renderImageLayer(layer, parent, disposables)
+			return renderImageLayer(layer, parent)
+		case 'transition':
+			return renderTransitionLayer(layer, parent)
+		case 'gap': {
+			pixi?.renderer.clear()
+			return {dispose: () => {}}
+		}
 		default:
 			console.warn('Unknown layer kind', (layer as any).kind)
-			return {disposables}
+			return {dispose: () => {}}
 	}
 }
 
-function isRenderableLayer(layer: any): layer is Layer {
-	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
-}
-
 function renderTextLayer(
 	layer: Extract<Layer, {kind: 'text'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const text = new Text({
 		text: layer.content,
-		style: {
-			fontFamily: 'sans-serif',
-			fontSize: layer.fontSize ?? 48,
-			fill: layer.color ?? 'white'
-		}
+		style: layer.style
 	})
-	applyTransform(text, layer)
+	applyTransform(text, layer.matrix)
 	parent.addChild(text)
-	disposables.push(text)
-	return {disposables}
+	return {dispose: () => text.destroy(true)}
 }
 
 function renderImageLayer(
 	layer: Extract<Layer, {kind: 'image'}>,
 	parent: Container,
-	disposables: RenderableObject[]
 ) {
 	const texture = Texture.from(layer.frame)
 	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer)
+	applyTransform(sprite, layer.matrix)
+	parent.addChild(sprite)
+	return {dispose: () => {
+		sprite.destroy(true)
+		texture.destroy(true)
+		layer.frame.close()
+	}}
+}
+
+function renderTransitionLayer(
+	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+	parent: Container,
+) {
+	const transition = transitions.get(name) ??
+		(transitions.set(name, makeTransition({
+			name: "circle",
+			renderer: pixi!.renderer
+		})),
+	  transitions.get(name)!
+	)
+	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+	const sprite = new Sprite(texture)
 	parent.addChild(sprite)
-	disposables.push(sprite, texture)
-	return {baseFrame: layer.frame, disposables}
+	return {dispose: () => sprite.destroy(false)}
 }
 
-function applyTransform(target: Sprite | Text, t: Transform = {}) {
-	if(t.x) target.x = t.x
-	if(t.y) target.y = t.y
-	if(t.scale) target.scale.set(t.scale)
-	if(t.opacity) target.alpha = t.opacity
-	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
+function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  if (!worldMatrix) return
+	const mx = mat6ToMatrix(worldMatrix)
+  target.setFromMatrix(mx)
 }

package/s/driver/utils/load-decoder-source.ts

@@ -2,10 +2,9 @@ import {BlobSource, UrlSource} from "mediabunny"
 import {DecoderSource} from "../fns/schematic.js"
 
 // only streamable sources
-export async function loadDecoderSource(source: DecoderSource) {
-	if(source instanceof FileSystemFileHandle) {
-		const file = await source.getFile()
-		return new BlobSource(file)
+export async function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource> {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
 	} else {
 		return new UrlSource(source)
 	}

package/s/features/speech/transcribe/default-spec.ts

@@ -0,0 +1,11 @@
+
+import {TranscriberSpec} from "./types.js"
+
+export const defaultTranscriberSpec = (): TranscriberSpec => ({
+	model: "onnx-community/whisper-tiny_timestamped",
+	dtype: "q4",
+	device: "wasm",
+	chunkLength: 20,
+	strideLength: 3,
+})
+

package/s/features/speech/transcribe/parts/load-pipe.ts

@@ -0,0 +1,19 @@
+
+import {pipeline} from "@huggingface/transformers"
+
+import {TranscriberPipeOptions} from "../types.js"
+
+export async function loadPipe(options: TranscriberPipeOptions) {
+	const {spec, onLoading} = options
+
+	const pipe = await pipeline("automatic-speech-recognition", spec.model, {
+		device: spec.device,
+		dtype: spec.dtype,
+		progress_callback: (data: any) => {
+			onLoading({total: data.total, progress: data.progress})
+		},
+	})
+
+	return pipe
+}
+

package/s/features/speech/transcribe/parts/prep-audio.ts

@@ -0,0 +1,23 @@
+
+import {Driver} from "../../../../driver/driver.js"
+
+export async function prepAudio(driver: Driver, source: Blob) {
+	const arrayBuffer = await source.arrayBuffer()
+	const audioCTX = new AudioContext({sampleRate: 16000})
+	const audioData = await audioCTX.decodeAudioData(arrayBuffer)
+	let audio: Float32Array
+	if (audioData.numberOfChannels === 2) {
+		const SCALING_FACTOR = Math.sqrt(2)
+		const left = audioData.getChannelData(0)
+		const right = audioData.getChannelData(1)
+		audio = new Float32Array(left.length)
+		for (let i = 0; i < audioData.length; ++i) {
+			audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
+		}
+	} else {
+		audio = audioData.getChannelData(0)
+	}
+	const duration = await driver.getAudioDuration(source)
+	return {audio, duration}
+}
+

package/s/features/speech/transcribe/parts/transcribe.ts

@@ -0,0 +1,70 @@
+
+import {WhisperTextStreamer} from "@huggingface/transformers"
+import {TranscribeOptions} from "../types.js"
+
+export async function transcribe(options: TranscribeOptions) {
+	const {pipe, spec, request, callbacks} = options
+
+	if (!pipe.processor.feature_extractor)
+		throw new Error("no feature_extractor")
+
+	const timePrecision = (
+		pipe.processor.feature_extractor?.config.chunk_length /
+		// @ts-ignore
+		pipe.model.config.max_source_positions
+	)
+
+	let chunkCount = 0
+	let startTime: number | null = null
+	let tokenCount = 0
+	let tokensPerSecond = 0
+
+	const chunkDuration = spec.chunkLength - spec.strideLength
+
+	const calculateProgress = () => {
+		const audioProgressSeconds = chunkCount * chunkDuration
+		return Math.min(audioProgressSeconds / request.duration, 1)
+	}
+
+	// TODO type error on pipe.tokenizer
+	const tokenizer = pipe.tokenizer as any
+
+	const streamer = new WhisperTextStreamer(tokenizer, {
+		time_precision: timePrecision,
+		token_callback_function: () => {
+			startTime ??= performance.now()
+			if (++tokenCount > 1) {
+				tokensPerSecond = (tokenCount / (performance.now() - startTime)) * 1000
+			}
+		},
+		callback_function: (textChunk: any) => {
+			// TODO
+			callbacks.onTranscription(textChunk)
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+		on_finalize: () => {
+			startTime = null
+			tokenCount = 0
+			chunkCount++
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+	})
+
+	const result = await pipe(new Float32Array(request.audio), {
+		top_k: 0,
+		do_sample: false,
+		chunk_length_s: spec.chunkLength,
+		stride_length_s: spec.strideLength,
+		language: request.language,
+		task: "transcribe",
+		return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+		force_full_sequences: false,
+		streamer,
+	})
+
+	return {
+		text: result.text,
+		chunks: result.chunks
+	}
+}
+

package/s/features/speech/transcribe/transcriber.ts

@@ -0,0 +1,46 @@
+
+import {Comrade} from "@e280/comrade"
+import {coalesce, queue, sub} from "@e280/stz"
+
+import {prepAudio} from "./parts/prep-audio.js"
+import {TranscriberOptions, TranscriberSchematic, TranscriptionOptions, TranscriptionReport} from "./types.js"
+
+export async function makeTranscriber({driver, spec, workerUrl, onLoading}: TranscriberOptions) {
+	const onReport = sub<[report: TranscriptionReport]>()
+	const onTranscription = sub<[transcription: string]>()
+
+	const thread = await Comrade.thread<TranscriberSchematic>({
+		label: "OmnitoolSpeechTranscriber",
+		workerUrl,
+		setupHost: () => ({
+			loading: async loading => onLoading(loading),
+			deliverReport: async report => onReport.pub(report),
+			deliverTranscription: async transcription => onTranscription.pub(transcription),
+		}),
+	})
+
+	await thread.work.prepare(spec)
+
+	return {
+		transcribe: queue(async(info: TranscriptionOptions) => {
+			const {source, language} = info
+			const {audio, duration} = await prepAudio(driver, source)
+
+			const detachCallbacks = coalesce(
+				onReport(info.onReport),
+				onTranscription(info.onTranscription),
+			)
+
+			const result = await thread.work.transcribe({
+				duration,
+				language,
+				audio: audio.buffer,
+			})
+
+			detachCallbacks()
+			return result
+		}),
+		dispose: thread.terminate()
+	}
+}
+

package/s/features/speech/transcribe/types.ts

@@ -0,0 +1,82 @@
+
+import {AsSchematic} from "@e280/comrade"
+import {DataType, DeviceType, Pipeline} from "@huggingface/transformers"
+
+import {Driver} from "../../../driver/driver.js"
+
+export type TranscriberSchematic = AsSchematic<{
+	work: {
+		prepare(spec: TranscriberSpec): Promise<void>
+		transcribe(request: TranscriptionRequest): Promise<Transcription>
+	},
+
+	host: {
+		loading(load: Loading): Promise<void>
+		deliverReport(report: TranscriptionReport): Promise<void>
+		deliverTranscription(transcription: string): Promise<void>
+	}
+}>
+
+export type Loading = {
+	total: number
+	progress: number
+}
+
+export type TranscribeOptions = {
+	pipe: Pipeline
+	spec: TranscriberSpec
+	request: TranscriptionRequest
+	callbacks: TranscriptionCallbacks
+}
+
+export type TranscriberPipeOptions = {
+	spec: TranscriberSpec
+	onLoading: (loading: Loading) => void
+}
+
+export type SpeechTime = [start: number, end: number]
+
+export type Transcription = {
+	text: string
+	chunks: {
+		text: string
+		timestamp: SpeechTime
+	}[]
+}
+
+export type TranscriberSpec = {
+	model: string
+	dtype: DataType
+	device: DeviceType
+	chunkLength: number
+	strideLength: number
+}
+
+export type TranscriptionOptions = {
+	source: Blob
+	language: string | null
+} & TranscriptionCallbacks
+
+export type TranscriptionRequest = {
+	audio: ArrayBufferLike
+	language: string | null
+	duration: number
+}
+
+export type TranscriptionReport = {
+	progress: number
+	tokensPerSecond: number
+}
+
+export type TranscriptionCallbacks = {
+	onReport: (report: TranscriptionReport) => void
+	onTranscription: (transcription: string) => void
+}
+
+export type TranscriberOptions = {
+	driver: Driver
+	spec: TranscriberSpec
+	workerUrl: URL | string
+	onLoading: (loading: Loading) => void
+}
+