@omnimedia/omnitool 1.1.0-3 → 1.1.0-30

package/x/features/speech/transcribe/parts/transcribe.js

@@ -0,0 +1,56 @@
+import { WhisperTextStreamer } from "@huggingface/transformers";
+export async function transcribe(options) {
+    const { pipe, spec, request, callbacks } = options;
+    if (!pipe.processor.feature_extractor)
+        throw new Error("no feature_extractor");
+    const timePrecision = (pipe.processor.feature_extractor?.config.chunk_length /
+        // @ts-ignore
+        pipe.model.config.max_source_positions);
+    let chunkCount = 0;
+    let startTime = null;
+    let tokenCount = 0;
+    let tokensPerSecond = 0;
+    const chunkDuration = spec.chunkLength - spec.strideLength;
+    const calculateProgress = () => {
+        const audioProgressSeconds = chunkCount * chunkDuration;
+        return Math.min(audioProgressSeconds / request.duration, 1);
+    };
+    // TODO type error on pipe.tokenizer
+    const tokenizer = pipe.tokenizer;
+    const streamer = new WhisperTextStreamer(tokenizer, {
+        time_precision: timePrecision,
+        token_callback_function: () => {
+            startTime ??= performance.now();
+            if (++tokenCount > 1) {
+                tokensPerSecond = (tokenCount / (performance.now() - startTime)) * 1000;
+            }
+        },
+        callback_function: (textChunk) => {
+            // TODO
+            callbacks.onTranscription(textChunk);
+            callbacks.onReport({ tokensPerSecond, progress: calculateProgress() });
+        },
+        on_finalize: () => {
+            startTime = null;
+            tokenCount = 0;
+            chunkCount++;
+            callbacks.onReport({ tokensPerSecond, progress: calculateProgress() });
+        },
+    });
+    const result = await pipe(new Float32Array(request.audio), {
+        top_k: 0,
+        do_sample: false,
+        chunk_length_s: spec.chunkLength,
+        stride_length_s: spec.strideLength,
+        language: request.language,
+        task: "transcribe",
+        return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+        force_full_sequences: false,
+        streamer,
+    });
+    return {
+        text: result.text,
+        chunks: result.chunks
+    };
+}
+//# sourceMappingURL=transcribe.js.map

package/x/features/speech/transcribe/parts/transcribe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcribe.js","sourceRoot":"","sources":["../../../../../s/features/speech/transcribe/parts/transcribe.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,mBAAmB,EAAC,MAAM,2BAA2B,CAAA;AAG7D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IAC1D,MAAM,EAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAC,GAAG,OAAO,CAAA;IAEhD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB;QACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;IAExC,MAAM,aAAa,GAAG,CACrB,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC,YAAY;QACrD,aAAa;QACb,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,CACtC,CAAA;IAED,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,SAAS,GAAkB,IAAI,CAAA;IACnC,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,eAAe,GAAG,CAAC,CAAA;IAEvB,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;IAE1D,MAAM,iBAAiB,GAAG,GAAG,EAAE;QAC9B,MAAM,oBAAoB,GAAG,UAAU,GAAG,aAAa,CAAA;QACvD,OAAO,IAAI,CAAC,GAAG,CAAC,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAC5D,CAAC,CAAA;IAED,oCAAoC;IACpC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAgB,CAAA;IAEvC,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC,SAAS,EAAE;QACnD,cAAc,EAAE,aAAa;QAC7B,uBAAuB,EAAE,GAAG,EAAE;YAC7B,SAAS,KAAK,WAAW,CAAC,GAAG,EAAE,CAAA;YAC/B,IAAI,EAAE,UAAU,GAAG,CAAC,EAAE,CAAC;gBACtB,eAAe,GAAG,CAAC,UAAU,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,GAAG,IAAI,CAAA;YACxE,CAAC;QACF,CAAC;QACD,iBAAiB,EAAE,CAAC,SAAc,EAAE,EAAE;YACrC,OAAO;YACP,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;YACpC,SAAS,CAAC,QAAQ,CAAC,EAAC,eAAe,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAC,CAAC,CAAA;QACrE,CAAC;QACD,WAAW,EAAE,GAAG,EAAE;YACjB,SAAS,GAAG,IAAI,CAAA;YAChB,UAAU,GAAG,CAAC,CAAA;YACd,UAAU,EAAE,CAAA;YACZ,SAAS,CAAC,QAAQ,CAAC,EAAC,eAAe,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAC,CAAC,CAAA;QACrE,CAAC;KACD,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC1D,KAAK,EAAE,CAAC;QACR,SAAS,EAAE,KAAK;QAChB,cAAc,EAAE,IAAI,CAAC,WAAW;QAChC,eAAe,EAAE,IAAI,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,IAAI,EAAE,YAAY;QAClB,iBAAiB,EAAE,MAAM,EAAE,2HAA2H;QACtJ,oBAAoB,EAAE,KAAK;QAC3B,QAAQ;KACR,CAAC,CAAA;IAEF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;KACrB,CAAA;AACF,CAAC"}

package/x/features/speech/transcribe/transcriber.d.ts

@@ -0,0 +1,5 @@
+import { TranscriberOptions, TranscriptionOptions } from "./types.js";
+export declare function makeTranscriber({ driver, spec, workerUrl, onLoading }: TranscriberOptions): Promise<{
+    transcribe: (info: TranscriptionOptions) => Promise<import("./types.js").Transcription>;
+    dispose: void;
+}>;

package/x/features/speech/transcribe/transcriber.js

@@ -0,0 +1,33 @@
+import { Comrade } from "@e280/comrade";
+import { coalesce, queue, sub } from "@e280/stz";
+import { prepAudio } from "./parts/prep-audio.js";
+export async function makeTranscriber({ driver, spec, workerUrl, onLoading }) {
+    const onReport = sub();
+    const onTranscription = sub();
+    const thread = await Comrade.thread({
+        label: "OmnitoolSpeechTranscriber",
+        workerUrl,
+        setupHost: () => ({
+            loading: async (loading) => onLoading(loading),
+            deliverReport: async (report) => onReport.pub(report),
+            deliverTranscription: async (transcription) => onTranscription.pub(transcription),
+        }),
+    });
+    await thread.work.prepare(spec);
+    return {
+        transcribe: queue(async (info) => {
+            const { source, language } = info;
+            const { audio, duration } = await prepAudio(driver, source);
+            const detachCallbacks = coalesce(onReport(info.onReport), onTranscription(info.onTranscription));
+            const result = await thread.work.transcribe({
+                duration,
+                language,
+                audio: audio.buffer,
+            });
+            detachCallbacks();
+            return result;
+        }),
+        dispose: thread.terminate()
+    };
+}
+//# sourceMappingURL=transcriber.js.map

package/x/features/speech/transcribe/transcriber.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcriber.js","sourceRoot":"","sources":["../../../../s/features/speech/transcribe/transcriber.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAE9C,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAA;AAG/C,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAqB;IAC7F,MAAM,QAAQ,GAAG,GAAG,EAAiC,CAAA;IACrD,MAAM,eAAe,GAAG,GAAG,EAA2B,CAAA;IAEtD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAuB;QACzD,KAAK,EAAE,2BAA2B;QAClC,SAAS;QACT,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YACjB,OAAO,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC;YAC5C,aAAa,EAAE,KAAK,EAAC,MAAM,EAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YACnD,oBAAoB,EAAE,KAAK,EAAC,aAAa,EAAC,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC;SAC/E,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAE/B,OAAO;QACN,UAAU,EAAE,KAAK,CAAC,KAAK,EAAC,IAA0B,EAAE,EAAE;YACrD,MAAM,EAAC,MAAM,EAAE,QAAQ,EAAC,GAAG,IAAI,CAAA;YAC/B,MAAM,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;YAEzD,MAAM,eAAe,GAAG,QAAQ,CAC/B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EACvB,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CACrC,CAAA;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC3C,QAAQ;gBACR,QAAQ;gBACR,KAAK,EAAE,KAAK,CAAC,MAAM;aACnB,CAAC,CAAA;YAEF,eAAe,EAAE,CAAA;YACjB,OAAO,MAAM,CAAA;QACd,CAAC,CAAC;QACF,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE;KAC3B,CAAA;AACF,CAAC"}

package/x/features/speech/transcribe/types.d.ts

@@ -0,0 +1,66 @@
+import { AsSchematic } from "@e280/comrade";
+import { DataType, DeviceType, Pipeline } from "@huggingface/transformers";
+import { Driver } from "../../../driver/driver.js";
+export type TranscriberSchematic = AsSchematic<{
+    work: {
+        prepare(spec: TranscriberSpec): Promise<void>;
+        transcribe(request: TranscriptionRequest): Promise<Transcription>;
+    };
+    host: {
+        loading(load: Loading): Promise<void>;
+        deliverReport(report: TranscriptionReport): Promise<void>;
+        deliverTranscription(transcription: string): Promise<void>;
+    };
+}>;
+export type Loading = {
+    total: number;
+    progress: number;
+};
+export type TranscribeOptions = {
+    pipe: Pipeline;
+    spec: TranscriberSpec;
+    request: TranscriptionRequest;
+    callbacks: TranscriptionCallbacks;
+};
+export type TranscriberPipeOptions = {
+    spec: TranscriberSpec;
+    onLoading: (loading: Loading) => void;
+};
+export type SpeechTime = [start: number, end: number];
+export type Transcription = {
+    text: string;
+    chunks: {
+        text: string;
+        timestamp: SpeechTime;
+    }[];
+};
+export type TranscriberSpec = {
+    model: string;
+    dtype: DataType;
+    device: DeviceType;
+    chunkLength: number;
+    strideLength: number;
+};
+export type TranscriptionOptions = {
+    source: Blob;
+    language: string | null;
+} & TranscriptionCallbacks;
+export type TranscriptionRequest = {
+    audio: ArrayBufferLike;
+    language: string | null;
+    duration: number;
+};
+export type TranscriptionReport = {
+    progress: number;
+    tokensPerSecond: number;
+};
+export type TranscriptionCallbacks = {
+    onReport: (report: TranscriptionReport) => void;
+    onTranscription: (transcription: string) => void;
+};
+export type TranscriberOptions = {
+    driver: Driver;
+    spec: TranscriberSpec;
+    workerUrl: URL | string;
+    onLoading: (loading: Loading) => void;
+};

package/x/features/speech/transcribe/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/x/features/speech/transcribe/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../s/features/speech/transcribe/types.ts"],"names":[],"mappings":""}

package/x/features/speech/transcribe/worker.bundle.d.ts

@@ -0,0 +1 @@
+export {};

package/x/features/speech/transcribe/worker.bundle.js

@@ -0,0 +1,33 @@
+import { defer, once } from "@e280/stz";
+import { Comrade } from "@e280/comrade";
+import { loadPipe } from "./parts/load-pipe.js";
+import { transcribe } from "./parts/transcribe.js";
+const deferred = defer();
+const makePrepare = (host) => once(async (spec) => {
+    deferred.resolve({
+        spec,
+        pipe: await loadPipe({
+            spec,
+            onLoading: loading => host.loading(loading),
+        }),
+    });
+});
+await Comrade.worker(shell => {
+    const prepare = makePrepare(shell.host);
+    return {
+        prepare,
+        async transcribe(request) {
+            const { pipe, spec } = await deferred.promise;
+            return transcribe({
+                pipe,
+                spec,
+                request,
+                callbacks: {
+                    onReport: report => shell.host.deliverReport(report),
+                    onTranscription: transcription => shell.host.deliverTranscription(transcription),
+                },
+            });
+        }
+    };
+});
+//# sourceMappingURL=worker.bundle.js.map

package/x/features/speech/transcribe/worker.bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.bundle.js","sourceRoot":"","sources":["../../../../s/features/speech/transcribe/worker.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,IAAI,EAAC,MAAM,WAAW,CAAA;AACrC,OAAO,EAAC,OAAO,EAAO,MAAM,eAAe,CAAA;AAG3C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAGhD,MAAM,QAAQ,GAAG,KAAK,EAA2C,CAAA;AAEjE,MAAM,WAAW,GAAG,CAAC,IAAgC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAC,IAAqB,EAAE,EAAE;IAC7F,QAAQ,CAAC,OAAO,CAAC;QAChB,IAAI;QACJ,IAAI,EAAE,MAAM,QAAQ,CAAC;YACpB,IAAI;YACJ,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3C,CAAC;KACF,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,MAAM,OAAO,CAAC,MAAM,CAAuB,KAAK,CAAC,EAAE;IAClD,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,OAAO;QACN,OAAO;QACP,KAAK,CAAC,UAAU,CAAC,OAAO;YACvB,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAA;YAC3C,OAAO,UAAU,CAAC;gBACjB,IAAI;gBACJ,IAAI;gBACJ,OAAO;gBACP,SAAS,EAAE;oBACV,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;oBACpD,eAAe,EAAE,aAAa,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;iBAChF;aACD,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC,CAAC,CAAA"}