@omnimedia/omnitool 1.0.0 → 1.1.0-10

package/x/tools/speech-recognition/whisper/tool.js

@@ -0,0 +1,63 @@
+import { Comrade } from "@e280/comrade";
+import { context } from "../../../context.js";
+import { setupWhisperHost } from "./fns/host.js";
+import { SpeechRecognizer } from "../common/model.js";
+export class Whisper extends SpeechRecognizer {
+    thread;
+    constructor(thread) {
+        super('whisper', "onnx-community/whisper-tiny_timestamped");
+        this.thread = thread;
+    }
+    static async setup() {
+        const thread = await Comrade.thread({
+            label: "OmnitoolDriver",
+            workerUrl: new URL("/tools/speech-recognition/whisper/parts/worker.bundle.min.js", import.meta.url),
+            setupHost: setupWhisperHost({
+                ...this.loaderEvents,
+                ...this.speechRecognizerEvents
+            })
+        });
+        return new this(thread);
+    }
+    async init() {
+        // there should be called loading of the model in worker instead when transcribe is called ..
+    }
+    async #transcribe(source, options) {
+        const arrayBuffer = await source.arrayBuffer();
+        const audioCTX = new AudioContext({ sampleRate: 16000 });
+        const audioData = await audioCTX.decodeAudioData(arrayBuffer);
+        let audio;
+        if (audioData.numberOfChannels === 2) {
+            const SCALING_FACTOR = Math.sqrt(2);
+            const left = audioData.getChannelData(0);
+            const right = audioData.getChannelData(1);
+            audio = new Float32Array(left.length);
+            for (let i = 0; i < audioData.length; ++i) {
+                audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2;
+            }
+        }
+        else {
+            audio = audioData.getChannelData(0);
+        }
+        const driver = await context.driver;
+        const duration = await driver.getAudioDuration(source);
+        return await this.thread.work.transcribe({
+            audio,
+            duration,
+            model: this.model,
+            subtask: this.multilingual ? "transcribe" : null,
+            language: this.multilingual && options?.language !== "auto"
+                ? options?.language ?? "english"
+                : null
+        });
+    }
+    async transcribe(source) {
+        const result = await this.#transcribe(source);
+        const words = result?.chunks.map((chunk) => ({
+            text: chunk.text.trim(),
+            timestamp: chunk.timestamp,
+        }));
+        return words;
+    }
+}
+//# sourceMappingURL=tool.js.map

package/x/tools/speech-recognition/whisper/tool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool.js","sourceRoot":"","sources":["../../../../s/tools/speech-recognition/whisper/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAS,MAAM,eAAe,CAAA;AAG7C,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAC,gBAAgB,EAAC,MAAM,oBAAoB,CAAA;AAGnD,MAAM,OAAO,OAAQ,SAAQ,gBAAgB;IACzB;IAAnB,YAAmB,MAAgC;QAClD,KAAK,CAAC,SAAS,EAAE,yCAAyC,CAAC,CAAA;QADzC,WAAM,GAAN,MAAM,CAA0B;IAEnD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAK;QACjB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAmB;YACrD,KAAK,EAAE,gBAAgB;YACvB,SAAS,EAAE,IAAI,GAAG,CAAC,8DAA8D,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;YACnG,SAAS,EAAE,gBAAgB,CAAC;gBAC3B,GAAG,IAAI,CAAC,YAAY;gBACpB,GAAG,IAAI,CAAC,sBAAsB;aAC9B,CAAC;SACF,CAAC,CAAA;QACF,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,IAAI;QACT,6FAA6F;IAC9F,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAY,EAAE,OAAqD;QACpF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,WAAW,EAAE,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,EAAC,UAAU,EAAE,KAAK,EAAC,CAAC,CAAA;QACtD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QAC7D,IAAI,KAAK,CAAA;QACT,IAAI,SAAS,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,IAAI,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;YACzC,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC3C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACvD,CAAC;QACF,CAAC;aAAM,CAAC;YACP,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;QACpC,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;QACnC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAA;QACtD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YACxC,KAAK;YACL,QAAQ;YACR,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;YAChD,QAAQ,EACP,IAAI,CAAC,YAAY,IAAI,OAAO,EAAE,QAAQ,KAAK,MAAM;gBAChD,CAAC,CAAC,OAAO,EAAE,QAAQ,IAAI,SAAS;gBAChC,CAAC,CAAC,IAAI;SACR,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAY;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QAE7C,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC1B,CAAC,CAAc,CAAA;QAEhB,OAAO,KAAK,CAAA;IACb,CAAC;CACD"}

package/s/parts/compositor.ts

@@ -1,5 +0,0 @@
-export class Compositor {
-	constructor() {}
-
-	crossfade([...clips]) {}
-}

package/s/parts/export.ts

@@ -1,5 +0,0 @@
-export class Export {
-	constructor() {}
-
-	export(clip, {container, codec}) {}
-}

package/s/parts/video-decoder.ts

@@ -1,27 +0,0 @@
-import {Clip, LoadedVideoClip} from "../types.js"
-
-export class OmniVideoDecoder {
-
-	#decoder = new VideoDecoder({
-		output: () => this.#output(),
-		error: () => this.#error()
-	})
-
-	constructor(
-		private getLoadedClip: (clip: Clip) =>
-		LoadedVideoClip | undefined
-	) {}
-
-	#output() {}
-
-	#error() {}
-
-	decodeVideo(clip: Clip) {
-		const loaded = this.getLoadedClip(clip)
-	}
-
-	seek(clip: Clip) {
-		const loaded = this.getLoadedClip(clip)
-		loaded?.demuxer.seek()
-	}
-}

package/s/parts/video-encoder.ts

@@ -1,15 +0,0 @@
-export class OmniVideoEncoder {
-	encoder = new VideoEncoder({
-		output: () => this.#output(),
-		error: () => this.#error()
-	})
-
-	constructor() {
-	}
-
-	#output() {}
-
-	#error() {}
-
-	encode() {}
-}

package/s/tools/generate-id.ts

@@ -1,7 +0,0 @@
-export function generateId(bytes = 32) {
-	const array = new Uint8Array(bytes)
-	const random = crypto.getRandomValues(array)
-	return [...random]
-		.map(b => b.toString(16).padStart(2, "0"))
-		.join("")
-}

package/s/tools/mp4boxjs/LICENSE.md

@@ -1,24 +0,0 @@
-Copyright (c) 2012. Telecom ParisTech/TSI/MM/GPAC Cyril Concolato
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the copyright holder nor the
-      names of its contributors may be used to endorse or promote products
-      derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/s/tools/mp4boxjs/demuxer.ts

@@ -1,106 +0,0 @@
-import MP4Box, {MP4File, MP4Info, MP4MediaTrack} from "./mp4box.adapter.js"
-
-export type Status = "render" | "decode" | "demux" | "fetch"
-type SetStatus = (status: Status, message: string) => void
-type OnChunk = (chunk: EncodedVideoChunk) => void
-type OnConfig = (config: VideoDecoderConfig) => void
-type OnReady = (boolean: boolean) => void
-
-export class MP4FileSink {
-	#setStatus: SetStatus
-	#file: MP4File
-	#offset = 0
-
-	constructor(file: MP4File, setStatus: SetStatus) {
-		this.#file = file
-		this.#setStatus = setStatus
-	}
-
-	write(chunk: ArrayBuffer) {
-		const buffer = new ArrayBuffer(chunk.byteLength)
-		//@ts-ignore
-		new Uint8Array(buffer).set(chunk)
-		//@ts-ignore
-		buffer.fileStart = this.#offset
-		this.#offset += buffer.byteLength
-		this.#setStatus("fetch", (this.#offset / (1024 ** 2)).toFixed(1) + " MiB")!
-		//@ts-ignore
-		this.#file!.appendBuffer(buffer)
-	}
-
-	close() {
-		this.#setStatus("fetch", "Done")
-		this.#file.flush()
-	}
-}
-
-export class MP4Demuxer {
-	#onConfig: OnConfig
-	#onChunk: OnChunk
-	#OnReady: OnReady
-	#setStatus: SetStatus
-	#framesCount: (frames: number) => void
-	#file: MP4File
-
-	constructor(file: File, {onConfig, onChunk, setStatus, framesCount, OnReady}: {onConfig: OnConfig, onChunk: OnChunk, setStatus: SetStatus, framesCount: (frames: number) => void, OnReady: OnReady}) {
-		this.#onConfig = onConfig
-		this.#onChunk = onChunk
-		this.#setStatus = setStatus
-		this.#framesCount = framesCount
-		this.#OnReady = OnReady
-
-		this.#file = MP4Box.createFile()
-		this.#file.onError = error => setStatus("demux", error)
-		this.#file.onReady = this.#onReady.bind(this)
-		this.#file.onSamples = this.#onSamples.bind(this)
-		const fileSink = new MP4FileSink(this.#file, setStatus)
-		file.stream().pipeTo(new WritableStream(fileSink, {highWaterMark: 1}))
-	}
-
-	seek() {
-		console.log("seek", this.#file.seek)
-	}
-
-	#description(track: MP4MediaTrack) {
-		const trak = this.#file.getTrackById(track.id)
-		for (const entry of trak.mdia!.minf!.stbl!.stsd!.entries) {
-		//@ts-ignore
-			const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C
-			if (box) {
-				//@ts-ignore
-				const stream = new MP4Box.DataStream(undefined, 0, MP4Box.DataStream.BIG_ENDIAN)
-				box.write(stream)
-				return new Uint8Array(stream.buffer!, 8)  // Remove the box header.
-			}
-		}
-		throw new Error("avcC, hvcC, vpcC, or av1C box not found")
-	}
-
-	#onReady(info: MP4Info) {
-		this.#OnReady(true)
-		this.#setStatus("demux", "Ready")
-		const track = info.videoTracks[0]
-		this.#onConfig({
-			codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
-			codedHeight: track.video.height,
-			codedWidth: track.video.width,
-			description: this.#description(track),
-		});
-		this.#framesCount(track.nb_samples)
-		this.#file.setExtractionOptions(track.id)
-		// this.#file.start()
-	}
-
-	#onSamples(track_id: number, ref: any, samples: any) {
-		for (const sample of samples) {
-			this.#onChunk(new EncodedVideoChunk({
-				type: sample.is_sync ? "key" : "delta",
-				timestamp: 1e6 * sample.cts / sample.timescale,
-				duration: 1e6 * sample.duration / sample.timescale,
-				data: sample.data
-			}))
-		}
-	}
-}
-
-

package/s/tools/mp4boxjs/mp4box.adapter.ts

@@ -1,148 +0,0 @@
-// @ts-ignore
-import {mp4box} from "./mp4box.js"
-
-export interface MP4MediaTrack {
-	id: number;
-	created: Date;
-	modified: Date;
-	movie_duration: number;
-	layer: number;
-	alternate_group: number;
-	volume: number;
-	track_width: number;
-	track_height: number;
-	timescale: number;
-	duration: number;
-	bitrate: number;
-	codec: string;
-	language: string;
-	nb_samples: number;
-	type: string;
-}
-
-export interface MP4VideoData {
-	width: number;
-	height: number;
-}
-
-export interface MP4VideoTrack extends MP4MediaTrack {
-	video: MP4VideoData;
-}
-export interface MP4AudioTrack extends MP4MediaTrack {
-	audio: MP4AudioData;
-}
-
-export interface MP4Info {
-	duration: number;
-	timescale: number;
-	fragment_duration: number;
-	isFragmented: boolean;
-	isProgressive: boolean;
-	hasIOD: boolean;
-	brands: string[];
-	created: Date;
-	modified: Date;
-	tracks: MP4Track[];
-	audioTracks: MP4AudioTrack[];
-	videoTracks: MP4VideoTrack[];
-}
-
-export interface MP4AudioData {
-	sample_rate: number;
-	channel_count: number;
-	sample_size: number;
-}
-
-export interface Moov {
-	type: string;
-	hdr_size: Number;
-	size: Number;
-	start: Number;
-	traks: any;
-	uuid: String | undefined;
-}
-
-export interface MP4File {
-	onMoovStart?: () => void;
-	onReady?: (info: MP4Info) => void;
-	onError?: (e: string) => void;
-	onSamples?: (track_id: number, _ref: any, samples: any) => void;
-	setExtractionOptions: (id: number) => void;
-
-	moov: Moov;
-
-	appendBuffer(data: MP4ArrayBuffer): number;
-	start(): void;
-	stop(): void;
-	flush(): void;
-	getTrackById: (id: number) => Trak
-}
-
-type MP4Track = MP4VideoTrack | MP4AudioTrack;
-
-export interface MP4Box {
-	createFile(): MP4File;
-	DataStream: new (arrayBuffer: ArrayBuffer, byteOffset: number, endianness: boolean) => XDataStream
-}
-
-export type MP4ArrayBuffer = ArrayBuffer & {fileStart: number};
-
-
-export interface TrackOptions {
-	timescale: number;
-	width: number;
-	height: number;
-	nb_samples: number;
-	avcDecoderConfigRecord: AllowSharedBufferSource | undefined;
-}
-
-export interface SampleOptions {
-	duration: number;
-	dts: number;
-	cts: number;
-	is_sync: boolean;
-}
-
-export interface MP4File {
-	onMoovStart?: () => void;
-	onReady?: (info: MP4Info) => void;
-	onError?: (e: string) => void;
-
-	appendBuffer(data: MP4ArrayBuffer): number;
-	start(): void;
-	stop(): void;
-	flush(): void;
-	addTrack(trackOptions: TrackOptions): MP4Track;
-	addSample(track: MP4Track, data: ArrayBuffer, sampleOptions: SampleOptions):void;
-	save(file: string):void;
-}
-
-export class XDataStream {
-	static BIG_ENDIAN: boolean;
-	static LITTLE_ENDIAN: boolean;
-	buffer?: ArrayBuffer;
-	constructor(arrayBuffer: ArrayBuffer, byteOffset: number, endianness: boolean) {}
-	// TODO: Complete interface
-}
-
-export interface Trak {
-	mdia?: {
-		minf?: {
-			stbl?: {
-				stsd?: {
-					entries: {
-						avcC?: {
-							write: (stream: XDataStream) => void
-						}
-						hvcC?: {
-							write: (stream: XDataStream) => void
-						}
-					}[]
-				}
-			}
-		}
-	}
-	// TODO: Complete interface
-}
-export default mp4box as MP4Box;
-export const DataStreamx = mp4box.DataStream as XDataStream