@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/driver/utils/find-pixi-filter.ts

@@ -0,0 +1,24 @@
+import {
+	AlphaFilter,
+	BlurFilter,
+	ColorMatrixFilter,
+	NoiseFilter,
+	Filter,
+} from "pixi.js"
+import * as PixiFilters from "pixi-filters"
+
+import type {FilterType} from "../../timeline/parts/filters.js"
+
+type FilterConstructor = new (options?: any) => Filter
+
+const pixiFilters: Partial<Record<FilterType, FilterConstructor>> = {
+	AlphaFilter,
+	BlurFilter,
+	ColorMatrixFilter,
+	NoiseFilter,
+	...PixiFilters
+}
+
+export function findPixiFilter(type: FilterType) {
+	return pixiFilters[type]
+}

package/s/driver/utils/load-decoder-source.ts

@@ -2,10 +2,9 @@ import {BlobSource, UrlSource} from "mediabunny"
 import {DecoderSource} from "../fns/schematic.js"
 
 // only streamable sources
-export async function loadDecoderSource(source: DecoderSource) {
-	if(source instanceof FileSystemFileHandle) {
-		const file = await source.getFile()
-		return new BlobSource(file)
+export async function loadDecoderSource(source: DecoderSource): Promise<UrlSource | BlobSource> {
+	if(source instanceof Blob) {
+		return new BlobSource(source)
 	} else {
 		return new UrlSource(source)
 	}

package/s/features/bg-remover/bg-remover.ts

@@ -0,0 +1,26 @@
+import {queue} from "@e280/stz"
+import {Comrade, LoggerTap, tune} from "@e280/comrade"
+
+import {BgRemoverOptions, BgRemoverSchematic, RemoverOptions} from "./types.js"
+
+
+export async function makeBgRemover({spec, workerUrl, onLoading}: BgRemoverOptions) {
+	const thread = await Comrade.thread<BgRemoverSchematic>({
+		label: "OmnitoolBgRemover",
+		workerUrl,
+		tap: new LoggerTap(),
+		setupHost: () => ({
+			loading: async loading => onLoading(loading),
+		}),
+	})
+
+	await thread.work.prepare(spec)
+
+	return {
+		remove: queue(async(input: RemoverOptions) =>
+			await thread.work.remove[tune]({transfer: [input.frame]})(input.frame)
+		),
+		dispose: () => thread.terminate()
+	}
+}
+

package/s/features/bg-remover/default-spec.ts

@@ -0,0 +1,11 @@
+
+import {PipelineSpec} from "../parts/types.js"
+
+
+export const defaultBgRemoverSpec = (): PipelineSpec => ({
+	model: "Xenova/modnet",
+	dtype: "auto",
+	device: "webgpu"
+})
+
+

package/s/features/bg-remover/types.ts

@@ -0,0 +1,27 @@
+
+import {AsSchematic} from "@e280/comrade"
+import {Loading, PipelineSpec} from "../parts/types.js"
+
+export type BgRemoverSchematic = AsSchematic<{
+	work: {
+		prepare(spec: PipelineSpec): Promise<void>
+		remove(request: VideoFrame): Promise<VideoFrame>
+	},
+
+	host: {
+		loading(load: Loading): Promise<void>
+	}
+}>
+
+export type RemoverOptions = {
+	frame: VideoFrame
+}
+
+export type BgRemoverModels = "onnx-community/ISNet-ONNX" | "Xenova/modnet" | "briaai/RMBG-1.4"
+
+export type BgRemoverOptions = {
+	spec: PipelineSpec
+	workerUrl: URL | string
+	onLoading: (loading: Loading) => void
+}
+

package/s/features/bg-remover/worker.bundle.ts

@@ -0,0 +1,52 @@
+
+import {defer, once} from "@e280/stz"
+import {Comrade, Host} from "@e280/comrade"
+import {BackgroundRemovalPipeline} from "@huggingface/transformers"
+
+import {PipelineSpec} from "../parts/types.js"
+import {BgRemoverSchematic} from "./types.js"
+import {loadPipe} from "../parts/load-pipe.js"
+import {exposeErrors} from "../parts/expose-errors.js"
+
+const deferred = defer<{spec: PipelineSpec, pipe: BackgroundRemovalPipeline}>()
+const makePrepare = (host: Host<BgRemoverSchematic>) => once(async(spec: PipelineSpec) => {
+	deferred.resolve({
+		spec,
+		pipe: await loadPipe({
+			spec,
+			task: "background-removal",
+			onLoading: loading => host.loading(loading),
+		}) as BackgroundRemovalPipeline
+	})
+})
+
+const canvas = new OffscreenCanvas(1920, 1080)
+const ctx = canvas.getContext("2d")
+
+await Comrade.worker<BgRemoverSchematic>(shell => {
+	const prepare = makePrepare(shell.host)
+	return {
+		prepare: exposeErrors(prepare),
+		remove: exposeErrors(async(request) => {
+			const {pipe} = await deferred.promise
+
+			canvas.width = request.displayWidth
+			canvas.height = request.displayHeight
+			ctx?.drawImage(request, 0, 0)
+
+			const output = await pipe(canvas)
+			const mask = output[0]
+
+			const frame = new VideoFrame(mask.toCanvas(), {
+				timestamp: request.timestamp,
+				duration: request.duration ?? undefined,
+			})
+
+			request.close()
+			shell.transfer = [frame]
+			return frame
+		})
+	}
+})
+
+

package/s/features/parts/expose-errors.ts

@@ -0,0 +1,41 @@
+import {ExposedError} from "@e280/renraku"
+
+export function exposeError(error: unknown) {
+	if (error instanceof ExposedError)
+		return error
+
+	const exposed = new ExposedError(errorToString(error))
+
+	if (error instanceof Error)
+		exposed.stack = error.stack
+
+	return exposed
+}
+
+export function exposeErrors<Args extends unknown[], Result>(
+	fn: (...args: Args) => Result | Promise<Result>
+) {
+	return async(...args: Args) => {
+		try {
+			return await fn(...args)
+		}
+		catch (error) {
+			throw exposeError(error)
+		}
+	}
+}
+
+function errorToString(error: unknown) {
+	if (error instanceof Error)
+		return error.message
+
+	if (typeof error === "string")
+		return error
+
+	try {
+		return JSON.stringify(error) ?? String(error)
+	}
+	catch {
+		return String(error)
+	}
+}

package/s/features/parts/load-pipe.ts

@@ -0,0 +1,17 @@
+
+import {pipeline} from "@huggingface/transformers"
+import {PipeOptions} from "./types.js"
+
+export async function loadPipe(options: PipeOptions) {
+	const {spec, onLoading} = options
+
+	const pipe = await pipeline(options.task, spec.model, {
+		device: spec.device,
+		dtype: spec.dtype,
+		progress_callback: (data: any) => {
+			onLoading({total: data.total, progress: data.progress})
+		},
+	})
+
+	return pipe
+}

package/s/features/parts/types.ts

@@ -0,0 +1,19 @@
+
+import {DataType, DeviceType, TaskType} from "@huggingface/transformers"
+
+export type Loading = {
+	total: number
+	progress: number
+}
+
+export type PipelineSpec<Extras extends object = {}> = {
+	model: string
+	dtype: DataType
+	device: DeviceType
+} & Extras
+
+export type PipeOptions = {
+	spec: PipelineSpec
+	task: TaskType
+	onLoading: (loading: Loading) => void
+}

package/s/features/speech/transcribe/default-spec.ts

@@ -0,0 +1,11 @@
+
+import {TranscriberSpec} from "./types.js"
+
+export const defaultTranscriberSpec = (): TranscriberSpec => ({
+	model: "onnx-community/whisper-tiny_timestamped",
+	dtype: "auto",
+	device: "webgpu",
+	chunkLength: 20,
+	strideLength: 3,
+})
+

package/s/features/speech/transcribe/parts/prep-audio.ts

@@ -0,0 +1,23 @@
+
+import {Driver} from "../../../../driver/driver.js"
+
+export async function prepAudio(driver: Driver, source: Blob) {
+	const arrayBuffer = await source.arrayBuffer()
+	const audioCTX = new AudioContext({sampleRate: 16000})
+	const audioData = await audioCTX.decodeAudioData(arrayBuffer)
+	let audio: Float32Array
+	if (audioData.numberOfChannels === 2) {
+		const SCALING_FACTOR = Math.sqrt(2)
+		const left = audioData.getChannelData(0)
+		const right = audioData.getChannelData(1)
+		audio = new Float32Array(left.length)
+		for (let i = 0; i < audioData.length; ++i) {
+			audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
+		}
+	} else {
+		audio = audioData.getChannelData(0)
+	}
+	const duration = await driver.getAudioDuration(source)
+	return {audio, duration}
+}
+

package/s/features/speech/transcribe/parts/transcribe.ts

@@ -0,0 +1,70 @@
+
+import {WhisperTextStreamer} from "@huggingface/transformers"
+import {TranscribeOptions} from "../types.js"
+
+export async function transcribe(options: TranscribeOptions) {
+	const {pipe, spec, request, callbacks} = options
+
+	if (!pipe.processor.feature_extractor)
+		throw new Error("no feature_extractor")
+
+	const timePrecision = (
+		pipe.processor.feature_extractor?.config.chunk_length /
+		// @ts-ignore
+		pipe.model.config.max_source_positions
+	)
+
+	let chunkCount = 0
+	let startTime: number | null = null
+	let tokenCount = 0
+	let tokensPerSecond = 0
+
+	const chunkDuration = spec.chunkLength - spec.strideLength
+
+	const calculateProgress = () => {
+		const audioProgressSeconds = chunkCount * chunkDuration
+		return Math.min(audioProgressSeconds / request.duration, 1)
+	}
+
+	// TODO type error on pipe.tokenizer
+	const tokenizer = pipe.tokenizer as any
+
+	const streamer = new WhisperTextStreamer(tokenizer, {
+		time_precision: timePrecision,
+		token_callback_function: () => {
+			startTime ??= performance.now()
+			if (++tokenCount > 1) {
+				tokensPerSecond = (tokenCount / (performance.now() - startTime)) * 1000
+			}
+		},
+		callback_function: (textChunk: any) => {
+			// TODO
+			callbacks.onTranscription(textChunk)
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+		on_finalize: () => {
+			startTime = null
+			tokenCount = 0
+			chunkCount++
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+	})
+
+	const result = await pipe(new Float32Array(request.audio), {
+		top_k: 0,
+		do_sample: false,
+		chunk_length_s: spec.chunkLength,
+		stride_length_s: spec.strideLength,
+		language: request.language,
+		task: "transcribe",
+		return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+		force_full_sequences: false,
+		streamer,
+	})
+
+	return {
+		text: result.text,
+		chunks: result.chunks
+	}
+}
+

package/s/features/speech/transcribe/transcriber.ts

@@ -0,0 +1,46 @@
+
+import {Comrade} from "@e280/comrade"
+import {coalesce, queue, sub} from "@e280/stz"
+
+import {prepAudio} from "./parts/prep-audio.js"
+import {TranscriberOptions, TranscriberSchematic, TranscriptionOptions, TranscriptionReport} from "./types.js"
+
+export async function makeTranscriber({driver, spec, workerUrl, onLoading}: TranscriberOptions) {
+	const onReport = sub<[report: TranscriptionReport]>()
+	const onTranscription = sub<[transcription: string]>()
+
+	const thread = await Comrade.thread<TranscriberSchematic>({
+		label: "OmnitoolSpeechTranscriber",
+		workerUrl,
+		setupHost: () => ({
+			loading: async loading => onLoading(loading),
+			deliverReport: async report => onReport.pub(report),
+			deliverTranscription: async transcription => onTranscription.pub(transcription),
+		}),
+	})
+
+	await thread.work.prepare(spec)
+
+	return {
+		transcribe: queue(async(info: TranscriptionOptions) => {
+			const {source, language} = info
+			const {audio, duration} = await prepAudio(driver, source)
+
+			const detachCallbacks = coalesce(
+				onReport(info.onReport),
+				onTranscription(info.onTranscription),
+			)
+
+			const result = await thread.work.transcribe({
+				duration,
+				language,
+				audio: audio.buffer,
+			})
+
+			detachCallbacks()
+			return result
+		}),
+		dispose: () => thread.terminate()
+	}
+}
+

package/s/features/speech/transcribe/types.ts

@@ -0,0 +1,76 @@
+
+import {AsSchematic} from "@e280/comrade"
+import {Pipeline} from "@huggingface/transformers"
+
+import {Loading, PipelineSpec} from "../../parts/types.js"
+import {Driver} from "../../../driver/driver.js"
+
+export type TranscriberSchematic = AsSchematic<{
+	work: {
+		prepare(spec: TranscriberSpec): Promise<void>
+		transcribe(request: TranscriptionRequest): Promise<Transcription>
+	},
+
+	host: {
+		loading(load: Loading): Promise<void>
+		deliverReport(report: TranscriptionReport): Promise<void>
+		deliverTranscription(transcription: string): Promise<void>
+	}
+}>
+
+export type TranscribeOptions = {
+	pipe: Pipeline
+	spec: TranscriberSpec
+	request: TranscriptionRequest
+	callbacks: TranscriptionCallbacks
+}
+
+export type SpeechTime = [start: number, end: number]
+
+export type TranscriptWord = {
+	text: string
+	timestamp: SpeechTime
+}
+
+export type TranscriptSegment = TranscriptWord
+
+export type Transcription = {
+	text: string
+	chunks: TranscriptWord[]
+}
+
+export type TranscriberModels = "onnx-community/whisper-tiny_timestamped"
+
+export type TranscriberSpec = PipelineSpec<{
+	chunkLength: number
+	strideLength: number
+}>
+
+export type TranscriptionOptions = {
+	source: Blob
+	language: string | null
+} & TranscriptionCallbacks
+
+export type TranscriptionRequest = {
+	audio: ArrayBufferLike
+	language: string | null
+	duration: number
+}
+
+export type TranscriptionReport = {
+	progress: number
+	tokensPerSecond: number
+}
+
+export type TranscriptionCallbacks = {
+	onReport: (report: TranscriptionReport) => void
+	onTranscription: (transcription: string) => void
+}
+
+export type TranscriberOptions = {
+	driver: Driver
+	spec: TranscriberSpec
+	workerUrl: URL | string
+	onLoading: (loading: Loading) => void
+}
+

package/s/features/speech/transcribe/worker.bundle.ts

@@ -0,0 +1,42 @@
+
+import {defer, once} from "@e280/stz"
+import {Comrade, Host} from "@e280/comrade"
+import {AutomaticSpeechRecognitionPipeline, Pipeline} from "@huggingface/transformers"
+
+import {loadPipe} from "../../parts/load-pipe.js"
+import {transcribe} from "./parts/transcribe.js"
+import {exposeErrors} from "../../parts/expose-errors.js"
+import {TranscriberSchematic, TranscriberSpec} from "./types.js"
+
+const deferred = defer<{pipe: Pipeline, spec: TranscriberSpec}>()
+
+const makePrepare = (host: Host<TranscriberSchematic>) => once(async(spec: TranscriberSpec) => {
+	deferred.resolve({
+		spec,
+		pipe: await loadPipe({
+			spec,
+			task: "automatic-speech-recognition",
+			onLoading: loading => host.loading(loading),
+		}) as AutomaticSpeechRecognitionPipeline
+	})
+})
+
+await Comrade.worker<TranscriberSchematic>(shell => {
+	const prepare = makePrepare(shell.host)
+	return {
+		prepare: exposeErrors(prepare),
+		transcribe: exposeErrors(async(request) => {
+			const {pipe, spec} = await deferred.promise
+			return transcribe({
+				pipe,
+				spec,
+				request,
+				callbacks: {
+					onReport: report => shell.host.deliverReport(report),
+					onTranscription: transcription => shell.host.deliverTranscription(transcription),
+				},
+			})
+		})
+	}
+})
+

package/s/features/transition/parts/fragment.ts

@@ -0,0 +1,24 @@
+export const fragment = (glsl: string) => `
+	precision highp float;
+	varying vec2 vTextureCoord;
+	varying vec2 _uv;
+	uniform sampler2D from, to;
+	uniform float progress, ratio, _fromR, _toR;
+	uniform float customUniform;
+
+	vec4 getFromColor(vec2 uv){
+		return texture2D(from, .5+(uv-.5)*vec2(max(ratio/_fromR,1.), max(_fromR/ratio,1.)));
+	}
+	vec4 getToColor(vec2 uv){
+		return texture2D(to, .5+(uv-.5)*vec2(max(ratio/_toR,1.), max(_toR/ratio,1.)));
+	}
+
+	// gl-transition code here
+	${glsl}
+	// gl-transition code end
+
+	void main(){
+		vec2 uv = vTextureCoord.xy;
+		gl_FragColor = transition(vTextureCoord);
+	}
+`

package/s/features/transition/parts/types.ts

@@ -0,0 +1,27 @@
+import {Renderer} from "pixi.js"
+import {TransitionName} from "../../../timeline/parts/transitions.js"
+
+export interface TransitionOptions {
+	name: TransitionName
+	renderer: Renderer
+}
+
+export interface TransitionRendererOptions {
+	from: VideoFrame
+	to: VideoFrame
+	progress: number
+	width: number
+	height: number
+}
+
+export interface GLTransition {
+	author: string
+	createdAt: string
+	glsl: string
+	license: string
+	name: TransitionName
+	updatedAt: string
+	defaultParams: any
+	paramsTypes: any
+}
+

package/s/features/transition/parts/uniforms.ts

@@ -0,0 +1,29 @@
+import {GLTransition} from "./types.js"
+
+export const uniforms = {
+	custom: (transition: GLTransition) => Object.fromEntries(
+		Object.entries(transition.defaultParams).map(([name, value]) => [
+			name,
+			{
+				value,
+				type: getUniformType(transition.paramsTypes[name])
+			}
+		])
+	),
+	basics: {
+		_fromR: {value: 1, type: "f32"},
+		_toR: {value: 1, type: "f32"},
+		ratio: {value: 1, type: "f32"},
+		progress: {value: 0, type: "f32"},
+		customUniform: {value: 0, type: "f32"},
+	}
+}
+
+const getUniformType = (type: string) => {
+	if(type === "f32" || type === "i32") {
+		return type
+	} else if(type === "float") {
+		return "f32"
+	}
+	else return `${type}<f32>`
+}

package/s/features/transition/parts/vertex.ts

@@ -0,0 +1,31 @@
+export const vertex = `
+	in vec2 aPosition;
+	varying vec2 _uv;                          // gl-transition
+	uniform mat3 projectionMatrix;
+	uniform vec4 uInputSize;
+	uniform vec4 uOutputFrame;
+	out vec2 vTextureCoord;
+	uniform vec4 uOutputTexture;
+
+	vec4 filterVertexPosition( void )
+	{
+			vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+
+			position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+			position.y = position.y * (2.0*uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+
+			return vec4(position, 0.0, 1.0);
+	}
+
+	vec2 filterTextureCoord( void )
+	{
+	return aPosition * (uOutputFrame.zw * uInputSize.zw);
+	}
+
+	void main(void)
+	{
+	gl_Position = filterVertexPosition();
+	vTextureCoord = filterTextureCoord();
+	_uv = vec2(0.5, 0.5) * (aPosition +vec2(1.0, 1.0));    // gl-transition
+	}
+`

package/s/features/transition/transition.ts

@@ -0,0 +1,60 @@
+//@ts-ignore
+import transitions from "gl-transitions"
+import {Filter, GlProgram, Sprite, Texture, ImageSource} from "pixi.js"
+
+import {vertex} from "./parts/vertex.js"
+import {uniforms} from "./parts/uniforms.js"
+import {fragment} from "./parts/fragment.js"
+import {GLTransition, TransitionOptions, TransitionRendererOptions} from "./parts/types.js"
+
+export function makeTransition({name, renderer}: TransitionOptions) {
+	const transition = transitions.find((t: GLTransition) => t.name === name) as GLTransition
+	const transitionSprite = new Sprite()
+	const transitionTexture = new Texture()
+	const sourceFrom = new ImageSource({})
+	const sourceTo = new ImageSource({})
+
+	const filter = new Filter({
+		glProgram: new GlProgram({
+			vertex,
+			fragment: fragment(transition.glsl),
+		}),
+		resources: {
+			from: sourceFrom,
+			to: sourceTo,
+			uniforms: {
+				...uniforms.basics,
+				...uniforms.custom(transition)
+			}
+		}
+	})
+
+	transitionSprite.filters = [filter]
+
+	return {
+		render({width, height, from, to, progress}: TransitionRendererOptions) {
+			if(transitionSprite.width !== width || transitionSprite.height !== height) {
+				transitionSprite.setSize({width, height})
+				transitionTexture.source.resize(width, height)
+			}
+
+			sourceFrom.resource = from
+			sourceTo.resource = to
+			sourceFrom.update()
+			sourceTo.update()
+
+			filter.resources.uniforms.uniforms.progress = progress
+
+			renderer.render({
+				container: transitionSprite,
+				target: transitionTexture,
+				clear: false,
+				width,
+				height
+			})
+
+			return transitionTexture
+		}
+	}
+}
+