@omnimedia/omnitool 1.1.0-3 → 1.1.0-5

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-3",
+	"version": "1.1.0-5",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -23,20 +23,22 @@
 		"test-debug": "node inspect x/tests.test.js"
 	},
 	"devDependencies": {
-		"@e280/science": "^0.0.5",
-		"@e280/scute": "^0.0.0-6",
-		"@types/node": "^24.0.14",
+		"@e280/science": "^0.0.6",
+		"@e280/scute": "^0.0.0-7",
+		"@types/node": "^24.2.0",
 		"http-server": "^14.1.1",
 		"npm-run-all": "^4.1.5",
-		"typescript": "^5.8.3"
+		"typescript": "^5.9.2"
 	},
 	"dependencies": {
 		"@benev/slate": "^0.3.9",
-		"@e280/comrade": "^0.0.0-18",
-		"@e280/renraku": "^0.5.0-19",
-		"@e280/stz": "^0.0.0-22",
+		"@e280/comrade": "^0.0.0-23",
+		"@e280/renraku": "^0.5.0-29",
+		"@e280/stz": "^0.0.0-34",
+		"@huggingface/transformers": "^3.7.1",
 		"comrade": "^0.0.3",
-		"mediabunny": "^1.1.1",
+		"gl-transitions": "^1.43.0",
+		"mediabunny": "^1.4.4",
 		"mp4-muxer": "^5.2.1",
 		"pixi.js": "^8.10.1",
 		"wavesurfer.js": "^7.10.0",

package/s/context.ts

@@ -1,6 +1,6 @@
 import {Driver} from "./driver/driver.js"
 
-const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url)
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
 
 export const context = {
 	driver: Driver.setup({workerUrl})

package/s/demo/demo.bundle.ts

@@ -2,6 +2,7 @@
 import {context} from "../context.js"
 import {waveformTest} from "./routines/waveform-test.js"
 import {filmstripTest} from "./routines/filmstrip-test.js"
+import {transcriberTest} from "./routines/transcriber-test.js"
 import {setupTranscodeTest} from "./routines/transcode-test.js"
 
 const driver = await context.driver
@@ -14,6 +15,7 @@ fetchButton?.addEventListener("click", startDemoFetch)
 importButton?.addEventListener("click", startDemoImport)
 
 waveformTest()
+const transcriber = await transcriberTest(driver)
 
 // hello world test
 {
@@ -26,9 +28,11 @@ waveformTest()
 async function startDemoImport()
 {
 	const [fileHandle] = await window.showOpenFilePicker()
-	const transcode = setupTranscodeTest(driver, fileHandle)
-	await filmstripTest(fileHandle)
+	const file = await fileHandle.getFile()
+	const transcode = setupTranscodeTest(driver, file)
+	await filmstripTest(file)
 	run(transcode, fileHandle.name)
+	await transcriber.transcribe(file)
 }
 
 async function startDemoFetch()

package/s/demo/routines/filmstrip-test.ts

@@ -1,6 +1,6 @@
 import {Filmstrip} from "../../timeline/parts/filmstrip.js"
 
-export async function filmstripTest(fileHandle: FileSystemFileHandle) {
+export async function filmstripTest(file: File) {
 	const rangeSlider = document.querySelector(".range") as HTMLInputElement
 	const rangeView = document.querySelector(".range-view")!
 	const rangeSizeSlider = document.querySelector(".range-size")! as HTMLInputElement
@@ -10,7 +10,7 @@ export async function filmstripTest(fileHandle: FileSystemFileHandle) {
 	const FPS_10 = 1000/10 / 1000
 	let rangeSize = 0.5
 	const filmstrip = await Filmstrip.init(
-			fileHandle,
+			file,
 			{
 				onChange(tiles) {
 					// Sort by time (optional, for clean ordering)

package/s/demo/routines/transcriber-test.ts

@@ -0,0 +1,34 @@
+import {Driver} from "../../driver/driver.js"
+import {makeTranscriber} from "../../features/speech/transcribe/transcriber.js"
+
+export async function transcriberTest(driver: Driver) {
+	const transcriber = await makeTranscriber({
+		driver,
+		spec: {
+			model: "onnx-community/whisper-tiny_timestamped",
+			device: "webgpu",
+			strideLength: 5,
+			chunkLength: 30,
+			dtype: "fp32"
+		},
+		workerUrl: new URL("/features/speech/transcribe/worker.bundle.min.js", import.meta.url),
+		onLoading({progress, total}) {
+	  	console.log(progress, total, "total")
+		},
+	})
+	return {
+		transcribe: async (file: File) => {
+			const result = await transcriber.transcribe({
+				source: file,
+				language: "english",
+				onReport(report) {
+	  			console.log("report", report)
+				},
+				onTranscription(transcription) {
+	  			console.log("transcript", transcription)
+				}
+			})
+			console.log(result, "transcript result")
+		}
+	}
+}

package/s/demo/routines/transitions-test.ts

@@ -0,0 +1,43 @@
+import {Application, Sprite} from "pixi.js"
+
+import {Driver} from "../../driver/driver.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export async function setupTransitionsTest(driver: Driver, source: DecoderSource) {
+	const app = new Application()
+	await app.init({width: 300, height: 300, preference: "webgl"})
+	const sprite = new Sprite({width: 300, height: 300})
+
+	app.stage.addChild(sprite)
+
+	document.body.appendChild(app.canvas)
+	const transition = makeTransition({name: "circle", renderer: app.renderer})
+
+	async function run() {
+		const readables = driver.decode({
+			source,
+			async onFrame(frame) {
+				const texture = transition.render({
+					from: frame,
+					to: frame,
+					progress: 0.7,
+					width: app.canvas.width,
+					height: app.canvas.height
+				})
+				sprite.texture = texture
+				return frame
+			}
+		})
+
+		await driver.encode({
+			readables,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000}
+			}
+		})
+	}
+
+	return {run}
+}

package/s/driver/fns/host.ts

@@ -3,10 +3,11 @@ import {Comrade} from "@e280/comrade"
 import {Machina} from "../parts/machina.js"
 import {DriverSchematic} from "./schematic.js"
 
-export const setupDriverHost = (machina: Machina) => Comrade.host<DriverSchematic>(({work}, rig) => ({
-
-	async world() {
-		machina.count++
-	}
-}))
+export const setupDriverHost = (machina: Machina) => (
+	Comrade.host<DriverSchematic>(_shell => ({
+		async world() {
+			machina.count++
+		},
+	}))
+)
 

package/s/driver/fns/schematic.ts

@@ -36,7 +36,7 @@ export interface EncoderInput {
 	}
 }
 
-export type DecoderSource = FileSystemFileHandle | string
+export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource

package/s/driver/fns/work.ts

@@ -1,142 +1,139 @@
 import {Comrade} from "@e280/comrade"
-import {
-	Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample,
-	AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource
-} from "mediabunny"
 import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
 import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
-export const setupDriverWork = Comrade.work<DriverSchematic>(({host}, rig) => ({
-
-	async hello() {
-		await host.world()
-	},
-
-	async decode({source, video, audio}) {
-		const loadSource = async () => {
-			if(source instanceof FileSystemFileHandle) {
-				const file = await source.getFile()
-				return new BlobSource(file)
-			} else {
-				return new UrlSource(source)
+export const setupDriverWork = (
+	Comrade.work<DriverSchematic>(shell => ({
+		async hello() {
+			await shell.host.world()
+		},
+
+		async decode({source, video, audio}) {
+			const loadSource = async () => {
+				if(source instanceof Blob) {
+					return new BlobSource(source)
+				} else {
+					return new UrlSource(source)
+				}
 			}
-		}
-		const input = new Input({
-			source: await loadSource(),
-			formats: ALL_FORMATS
-		})
-
-		const [videoTrack, audioTrack] = await Promise.all([
-			input.getPrimaryVideoTrack(),
-			input.getPrimaryAudioTrack()
-		])
-
-		const videoDecodable = await videoTrack?.canDecode()
-		const audioDecodable = await audioTrack?.canDecode()
-
-		const videoWriter = video.getWriter()
-		const audioWriter = audio.getWriter()
-
-		await Promise.all([
-			(async () => {
-				if (videoDecodable && videoTrack) {
-					const sink = new VideoSampleSink(videoTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toVideoFrame()
-						await videoWriter.write(frame)
+			const input = new Input({
+				source: await loadSource(),
+				formats: ALL_FORMATS
+			})
+
+			const [videoTrack, audioTrack] = await Promise.all([
+				input.getPrimaryVideoTrack(),
+				input.getPrimaryAudioTrack()
+			])
+
+			const videoDecodable = await videoTrack?.canDecode()
+			const audioDecodable = await audioTrack?.canDecode()
+
+			const videoWriter = video.getWriter()
+			const audioWriter = audio.getWriter()
+
+			await Promise.all([
+				(async () => {
+					if (videoDecodable && videoTrack) {
+						const sink = new VideoSampleSink(videoTrack)
+						for await (const sample of sink.samples()) {
+							const frame = sample.toVideoFrame()
+							await videoWriter.write(frame)
+							sample.close()
+							frame.close()
+						}
+						await videoWriter.close()
+					}
+				})(),
+				(async () => {
+					if (audioDecodable && audioTrack) {
+						const sink = new AudioSampleSink(audioTrack)
+						for await (const sample of sink.samples()) {
+							const frame = sample.toAudioData()
+							await audioWriter.write(frame)
+							sample.close()
+							frame.close()
+						}
+						await audioWriter.close()
+					}
+				})()
+			])
+		},
+
+		async encode({readables, config, bridge}) {
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new StreamTarget(bridge, {chunked: true})
+			})
+			const videoSource = new VideoSampleSource(config.video)
+			output.addVideoTrack(videoSource)
+			// since AudioSample is not transferable it fails to transfer encoder bitrate config
+			// so it needs to be hardcoded not set through constants eg QUALITY_LOW
+			const audioSource = new AudioSampleSource(config.audio)
+			output.addAudioTrack(audioSource)
+
+			await output.start()
+
+			const videoReader = readables.video.getReader()
+			const audioReader = readables.audio.getReader()
+
+			await Promise.all([
+				(async () => {
+					while (true) {
+						const {done, value} = await videoReader.read()
+						if (done) break
+						const sample = new VideoSample(value)
+						await videoSource.add(sample)
 						sample.close()
-						frame.close()
 					}
-					await videoWriter.close()
-				}
-			})(),
-			(async () => {
-				if (audioDecodable && audioTrack) {
-					const sink = new AudioSampleSink(audioTrack)
-					for await (const sample of sink.samples()) {
-						const frame = sample.toAudioData()
-						await audioWriter.write(frame)
+				})(),
+				(async () => {
+					while (true) {
+						const {done, value} = await audioReader.read()
+						if (done) break
+						const sample = new AudioSample(value)
+						await audioSource.add(sample)
 						sample.close()
-						frame.close()
+						value.close()
 					}
-					await audioWriter.close()
-				}
-			})()
-		])
-	},
-
-	async encode({readables, config, bridge}) {
-		const output = new Output({
-			format: new Mp4OutputFormat(),
-			target: new StreamTarget(bridge, {chunked: true})
-		})
-		const videoSource = new VideoSampleSource(config.video)
-		output.addVideoTrack(videoSource)
-		// since AudioSample is not transferable it fails to transfer encoder bitrate config
-		// so it needs to be hardcoded not set through constants eg QUALITY_LOW
-		const audioSource = new AudioSampleSource(config.audio)
-		output.addAudioTrack(audioSource)
-
-		await output.start()
-
-		const videoReader = readables.video.getReader()
-		const audioReader = readables.audio.getReader()
-
-		await Promise.all([
-			(async () => {
-				while (true) {
-					const {done, value} = await videoReader.read()
-					if (done) break
-					const sample = new VideoSample(value)
-					await videoSource.add(sample)
-					sample.close()
-				}
-			})(),
-			(async () => {
-				while (true) {
-					const {done, value} = await audioReader.read()
-					if (done) break
-					const sample = new AudioSample(value)
-					await audioSource.add(sample)
-					sample.close()
-					value.close()
-				}
-			})()
-		])
+				})()
+			])
 
-		await output.finalize()
-	},
+			await output.finalize()
+		},
 
-	async composite(composition) {
-		const {stage, renderer} = await renderPIXI(1920, 1080)
-		stage.removeChildren()
+		async composite(composition) {
+			const {stage, renderer} = await renderPIXI(1920, 1080)
+			stage.removeChildren()
 
-		const {baseFrame, disposables} = await renderLayer(composition, stage)
-		renderer.render(stage)
+			const {baseFrame, disposables} = await renderLayer(composition, stage)
+			renderer.render(stage)
 
-		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-		// if its very slow on eg edge try chrome
-		const frame = new VideoFrame(renderer.canvas, {
-			timestamp: baseFrame?.timestamp,
-			duration: baseFrame?.duration ?? undefined,
-		})
+			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+			// if its very slow on eg edge try chrome
+			const frame = new VideoFrame(renderer.canvas, {
+				timestamp: baseFrame?.timestamp,
+				duration: baseFrame?.duration ?? undefined,
+			})
 
-		baseFrame?.close()
-		renderer.clear()
-
-		for (const disposable of disposables) {
-			disposable.destroy(true)
-		}
+			baseFrame?.close()
+			renderer.clear()
 
-		rig.transfer = [frame]
-		return frame
-	}
-}))
+			for (const disposable of disposables) {
+				disposable.destroy(true)
+			}
 
+			shell.transfer = [frame]
+			return frame
+		}
+	}))
+)
 
+// TODO suspicious global, probably bad
 let pixi: {
 	renderer: Renderer
 	stage: Container

package/s/driver/utils/load-decoder-source.ts

@@ -2,10 +2,9 @@ import {BlobSource, UrlSource} from "mediabunny"
 import {DecoderSource} from "../fns/schematic.js"
 
 // only streamable sources
-export async function loadDecoderSource(source: DecoderSource) {
-	if(source instanceof FileSystemFileHandle) {
-		const file = await source.getFile()
-		return new BlobSource(file)
+export async function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource> {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
 	} else {
 		return new UrlSource(source)
 	}

package/s/features/speech/transcribe/default-spec.ts

@@ -0,0 +1,11 @@
+
+import {TranscriberSpec} from "./types.js"
+
+export const defaultTranscriberSpec = (): TranscriberSpec => ({
+	model: "onnx-community/whisper-tiny_timestamped",
+	dtype: "q4",
+	device: "wasm",
+	chunkLength: 20,
+	strideLength: 3,
+})
+

package/s/features/speech/transcribe/parts/load-pipe.ts

@@ -0,0 +1,19 @@
+
+import {pipeline} from "@huggingface/transformers"
+
+import {TranscriberPipeOptions} from "../types.js"
+
+export async function loadPipe(options: TranscriberPipeOptions) {
+	const {spec, onLoading} = options
+
+	const pipe = await pipeline("automatic-speech-recognition", spec.model, {
+		device: spec.device,
+		dtype: spec.dtype,
+		progress_callback: (data: any) => {
+			onLoading({total: data.total, progress: data.progress})
+		},
+	})
+
+	return pipe
+}
+

package/s/features/speech/transcribe/parts/prep-audio.ts

@@ -0,0 +1,23 @@
+
+import {Driver} from "../../../../driver/driver.js"
+
+export async function prepAudio(driver: Driver, source: Blob) {
+	const arrayBuffer = await source.arrayBuffer()
+	const audioCTX = new AudioContext({sampleRate: 16000})
+	const audioData = await audioCTX.decodeAudioData(arrayBuffer)
+	let audio: Float32Array
+	if (audioData.numberOfChannels === 2) {
+		const SCALING_FACTOR = Math.sqrt(2)
+		const left = audioData.getChannelData(0)
+		const right = audioData.getChannelData(1)
+		audio = new Float32Array(left.length)
+		for (let i = 0; i < audioData.length; ++i) {
+			audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
+		}
+	} else {
+		audio = audioData.getChannelData(0)
+	}
+	const duration = await driver.getAudioDuration(source)
+	return {audio, duration}
+}
+

package/s/features/speech/transcribe/parts/transcribe.ts

@@ -0,0 +1,70 @@
+
+import {WhisperTextStreamer} from "@huggingface/transformers"
+import {TranscribeOptions} from "../types.js"
+
+export async function transcribe(options: TranscribeOptions) {
+	const {pipe, spec, request, callbacks} = options
+
+	if (!pipe.processor.feature_extractor)
+		throw new Error("no feature_extractor")
+
+	const timePrecision = (
+		pipe.processor.feature_extractor?.config.chunk_length /
+		// @ts-ignore
+		pipe.model.config.max_source_positions
+	)
+
+	let chunkCount = 0
+	let startTime: number | null = null
+	let tokenCount = 0
+	let tokensPerSecond = 0
+
+	const chunkDuration = spec.chunkLength - spec.strideLength
+
+	const calculateProgress = () => {
+		const audioProgressSeconds = chunkCount * chunkDuration
+		return Math.min(audioProgressSeconds / request.duration, 1)
+	}
+
+	// TODO type error on pipe.tokenizer
+	const tokenizer = pipe.tokenizer as any
+
+	const streamer = new WhisperTextStreamer(tokenizer, {
+		time_precision: timePrecision,
+		token_callback_function: () => {
+			startTime ??= performance.now()
+			if (++tokenCount > 1) {
+				tokensPerSecond = (tokenCount / (performance.now() - startTime)) * 1000
+			}
+		},
+		callback_function: (textChunk: any) => {
+			// TODO
+			callbacks.onTranscription(textChunk)
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+		on_finalize: () => {
+			startTime = null
+			tokenCount = 0
+			chunkCount++
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+	})
+
+	const result = await pipe(new Float32Array(request.audio), {
+		top_k: 0,
+		do_sample: false,
+		chunk_length_s: spec.chunkLength,
+		stride_length_s: spec.strideLength,
+		language: request.language,
+		task: "transcribe",
+		return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+		force_full_sequences: false,
+		streamer,
+	})
+
+	return {
+		text: result.text,
+		chunks: result.chunks
+	}
+}
+

package/s/features/speech/transcribe/transcriber.ts

@@ -0,0 +1,46 @@
+
+import {Comrade} from "@e280/comrade"
+import {coalesce, queue, sub} from "@e280/stz"
+
+import {prepAudio} from "./parts/prep-audio.js"
+import {TranscriberOptions, TranscriberSchematic, TranscriptionOptions, TranscriptionReport} from "./types.js"
+
+export async function makeTranscriber({driver, spec, workerUrl, onLoading}: TranscriberOptions) {
+	const onReport = sub<[report: TranscriptionReport]>()
+	const onTranscription = sub<[transcription: string]>()
+
+	const thread = await Comrade.thread<TranscriberSchematic>({
+		label: "OmnitoolSpeechTranscriber",
+		workerUrl,
+		setupHost: () => ({
+			loading: async loading => onLoading(loading),
+			deliverReport: async report => onReport.pub(report),
+			deliverTranscription: async transcription => onTranscription.pub(transcription),
+		}),
+	})
+
+	await thread.work.prepare(spec)
+
+	return {
+		transcribe: queue(async(info: TranscriptionOptions) => {
+			const {source, language} = info
+			const {audio, duration} = await prepAudio(driver, source)
+
+			const detachCallbacks = coalesce(
+				onReport(info.onReport),
+				onTranscription(info.onTranscription),
+			)
+
+			const result = await thread.work.transcribe({
+				duration,
+				language,
+				audio: audio.buffer,
+			})
+
+			detachCallbacks()
+			return result
+		}),
+		dispose: thread.terminate()
+	}
+}
+