@omnimedia/omnitool 1.1.0-1 → 1.1.0-10

package/s/tools/speech-recognition/whisper/tool.ts

@@ -0,0 +1,70 @@
+import {Comrade, Thread} from "@e280/comrade"
+
+import {WordGroup} from "./parts/types.js"
+import {context} from "../../../context.js"
+import {setupWhisperHost} from "./fns/host.js"
+import {SpeechRecognizer} from "../common/model.js"
+import {WhisperSchematic} from "./fns/schematic.js"
+
+export class Whisper extends SpeechRecognizer {
+	constructor(public thread: Thread<WhisperSchematic>) {
+		super('whisper', "onnx-community/whisper-tiny_timestamped")
+	}
+
+	static async setup() {
+		const thread = await Comrade.thread<WhisperSchematic>({
+			label: "OmnitoolDriver",
+			workerUrl: new URL("/tools/speech-recognition/whisper/parts/worker.bundle.min.js", import.meta.url),
+			setupHost: setupWhisperHost({
+				...this.loaderEvents,
+				...this.speechRecognizerEvents
+			})
+		})
+		return new this(thread)
+	}
+
+	async init() {
+		// there should be called loading of the model in worker instead when transcribe is called ..
+	}
+
+	async #transcribe(source: Blob, options?: {multilingual?: boolean, language?: string}) {
+		const arrayBuffer = await source.arrayBuffer()
+		const audioCTX = new AudioContext({sampleRate: 16000})
+		const audioData = await audioCTX.decodeAudioData(arrayBuffer)
+		let audio
+		if (audioData.numberOfChannels === 2) {
+			const SCALING_FACTOR = Math.sqrt(2)
+			const left = audioData.getChannelData(0)
+			const right = audioData.getChannelData(1)
+			audio = new Float32Array(left.length)
+			for (let i = 0; i < audioData.length; ++i) {
+				audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
+			}
+		} else {
+			audio = audioData.getChannelData(0)
+		}
+		const driver = await context.driver
+		const duration = await driver.getAudioDuration(source)
+		return await this.thread.work.transcribe({
+			audio,
+			duration,
+			model: this.model,
+			subtask: this.multilingual ? "transcribe" : null,
+			language:
+				this.multilingual && options?.language !== "auto"
+					? options?.language ?? "english"
+					: null
+		})
+	}
+
+	async transcribe(source: Blob): Promise<WordGroup> {
+		const result = await this.#transcribe(source)
+
+		const words = result?.chunks.map((chunk: any) => ({
+			text: chunk.text.trim(),
+			timestamp: chunk.timestamp,
+		})) as WordGroup
+
+		return words
+	}
+}

package/x/context.js

@@ -1,5 +1,5 @@
 import { Driver } from "./driver/driver.js";
-const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url);
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url);
 export const context = {
     driver: Driver.setup({ workerUrl })
 };

package/x/context.js.map

@@ -1 +1 @@
-{"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAA;AAEzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,mCAAmC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAC,CAAC;CACjC,CAAA"}
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAA;AAEzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,uCAAuC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAEnF,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAC,CAAC;CACjC,CAAA"}

package/x/demo/demo.bundle.js

@@ -1,6 +1,7 @@
 import { context } from "../context.js";
 import { waveformTest } from "./routines/waveform-test.js";
 import { filmstripTest } from "./routines/filmstrip-test.js";
+import { transcriberTest } from "./routines/transcriber-test.js";
 import { setupTranscodeTest } from "./routines/transcode-test.js";
 const driver = await context.driver;
 const results = document.querySelector(".results");
@@ -9,6 +10,7 @@ const importButton = document.querySelector(".import");
 fetchButton?.addEventListener("click", startDemoFetch);
 importButton?.addEventListener("click", startDemoImport);
 waveformTest();
+const transcriber = await transcriberTest(driver);
 // hello world test
 {
     await driver.thread.work.hello();
@@ -20,9 +22,11 @@ waveformTest();
 // transcoding tests
 async function startDemoImport() {
     const [fileHandle] = await window.showOpenFilePicker();
-    const transcode = setupTranscodeTest(driver, fileHandle);
-    await filmstripTest(fileHandle);
+    const file = await fileHandle.getFile();
+    const transcode = setupTranscodeTest(driver, file);
+    await filmstripTest(file);
     run(transcode, fileHandle.name);
+    await transcriber.transcribe(file);
 }
 async function startDemoFetch() {
     // which videos to run tests on

package/x/demo/demo.bundle.js.map

@@ -1 +1 @@
-{"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAC1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAE/D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,YAAY,EAAE,CAAA;AAEd,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACxD,MAAM,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/B,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}
+{"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAC1D,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAA;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAE/D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,YAAY,EAAE,CAAA;AACd,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;AAEjD,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,CAAA;IACvC,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAClD,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;IACzB,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}