@omnimedia/omnitool 1.1.0-4 → 1.1.0-41

package/x/timeline/sugar/omni.js.map

@@ -1 +1 @@
-{"version":3,"file":"omni.js","sourceRoot":"","sources":["../../../s/timeline/sugar/omni.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,CAAC,EAAC,MAAM,QAAQ,CAAA;AAKxB,OAAO,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAA;AAEtD,MAAM,OAAO,IAAI;IAChB,SAAS,GAAG,IAAI,YAAY,EAAE,CAAA;IAE9B,IAAI,GAAG,KAAK,EAA8C,IAAO,EAAE,EAAE;QACpE,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CACnE,KAAK,EAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CACrE,CAAC,CAA4B,CAAA;IAC/B,CAAC,CAAA;IAED,QAAQ,GAAG,CAAC,EAA2B,EAAgB,EAAE;QACxD,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAA;QACjB,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;QACtB,OAAO;YACN,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,uBAAuB;YAC7B,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC1B,KAAK,EAAE,CAAC,CAAC,KAAK;SACd,CAAA;IACF,CAAC,CAAA;CACD"}
+{"version":3,"file":"omni.js","sourceRoot":"","sources":["../../../s/timeline/sugar/omni.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,CAAC,EAAC,MAAM,QAAQ,CAAA;AAMxB,OAAO,EAAC,MAAM,EAAC,MAAM,+BAA+B,CAAA;AACpD,OAAO,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAA;AAGtD,MAAM,OAAO,IAAI;IAII;IAHpB,SAAS,GAAG,IAAI,YAAY,EAAE,CAAA;IAC9B,OAAO,CAAQ;IAEf,YAAoB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QACjC,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,IAAI,GAAG,KAAK,EAA8C,IAAO,EAAE,EAAE;QACpE,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CACnE,KAAK,EAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CACrE,CAAC,CAA4B,CAAA;IAC/B,CAAC,CAAA;IAED,QAAQ,GAAG,CAAC,EAAsB,EAAgB,EAAE;QACnD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;YACf,OAAO,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,uBAAuB;gBAC7B,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,CAAC;aACT;SACD,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;QAChC,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAA;IACvB,CAAC,CAAA;IAED,MAAM,GAAG,KAAK,EAAE,QAAsB,EAAE,MAAoB,EAAE,EAAE;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACpC,CAAC,CAAA;CACD"}

package/x/timeline/types.d.ts

@@ -0,0 +1,24 @@
+export type Interpolation = "linear" | "catmullRom";
+export type Keyframe<Value = number> = [time: number, value: Value];
+export type Keyframes<Value = number> = Keyframe<Value>[];
+export type Vec2 = [x: number, y: number];
+export type Transform = [position: Vec2, scale: Vec2, rotation: number];
+export type TrackVec2 = {
+    x: Keyframes;
+    y: Keyframes;
+};
+export type Anim<T> = {
+    terp: Interpolation;
+    track: T;
+};
+export type Animations = Anim<TrackTransform>;
+export type TrackTransform = {
+    position: TrackVec2;
+    scale: TrackVec2;
+    rotation: Keyframes;
+};
+export type TransformOptions = {
+    position?: Vec2;
+    scale?: Vec2;
+    rotation?: number;
+};

package/x/timeline/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/x/timeline/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../s/timeline/types.ts"],"names":[],"mappings":""}

package/x/timeline/utils/audio-stream.d.ts

@@ -0,0 +1,6 @@
+export declare class AudioStream {
+    private reader;
+    constructor(reader: ReadableStreamDefaultReader<AudioData>);
+    stream(): AsyncGenerator<AudioData>;
+    cancel: () => Promise<void>;
+}

package/x/timeline/utils/audio-stream.js

@@ -0,0 +1,17 @@
+export class AudioStream {
+    reader;
+    constructor(reader) {
+        this.reader = reader;
+    }
+    async *stream() {
+        while (true) {
+            const { done, value: hit } = await this.reader.read();
+            if (done) {
+                break;
+            }
+            yield hit;
+        }
+    }
+    cancel = async () => await this.reader.cancel();
+}
+//# sourceMappingURL=audio-stream.js.map

package/x/timeline/utils/audio-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-stream.js","sourceRoot":"","sources":["../../../s/timeline/utils/audio-stream.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,WAAW;IACH;IAApB,YAAoB,MAA8C;QAA9C,WAAM,GAAN,MAAM,CAAwC;IAAG,CAAC;IAEtE,KAAK,CAAC,CAAC,MAAM;QACZ,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YACnD,IAAI,IAAI,EAAE,CAAC;gBACV,MAAK;YACN,CAAC;YACD,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;IAED,MAAM,GAAG,KAAK,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;CAC/C"}

package/x/timeline/utils/matrix.d.ts

@@ -0,0 +1,8 @@
+import { Matrix } from "pixi.js";
+import { Transform } from "../types.js";
+export declare const transformToMat6: (t: Transform) => Mat6;
+export declare const mat6ToMatrix: ([a, b, c, d, tx, ty]: Mat6) => Matrix;
+export declare const transformToMatrix: (t: Transform) => Matrix;
+export declare const mul6: (local: Mat6, parent: Mat6) => Mat6;
+export declare const I6: Mat6;
+export type Mat6 = [a: number, b: number, c: number, d: number, tx: number, ty: number];

package/x/timeline/utils/matrix.js

@@ -0,0 +1,26 @@
+import { Matrix } from "pixi.js";
+export const transformToMat6 = (t) => {
+    const [pos, scl, rotDeg] = t;
+    const [x, y] = pos;
+    const [sx, sy] = scl;
+    const r = rotDeg * Math.PI / 180;
+    const cos = Math.cos(r);
+    const sin = Math.sin(r);
+    return [cos * sx, sin * sx, -sin * sy, cos * sy, x, y];
+};
+export const mat6ToMatrix = ([a, b, c, d, tx, ty]) => new Matrix(a, b, c, d, tx, ty);
+export const transformToMatrix = (t) => mat6ToMatrix(transformToMat6(t));
+export const mul6 = (local, parent) => {
+    const [a1, b1, c1, d1, tx1, ty1] = local;
+    const [a2, b2, c2, d2, tx2, ty2] = parent;
+    return [
+        a1 * a2 + c1 * b2,
+        b1 * a2 + d1 * b2,
+        a1 * c2 + c1 * d2,
+        b1 * c2 + d1 * d2,
+        a1 * tx2 + c1 * ty2 + tx1,
+        b1 * tx2 + d1 * ty2 + ty1
+    ];
+};
+export const I6 = [1, 0, 0, 1, 0, 0];
+//# sourceMappingURL=matrix.js.map

package/x/timeline/utils/matrix.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matrix.js","sourceRoot":"","sources":["../../../s/timeline/utils/matrix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,SAAS,CAAA;AAG9B,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAY,EAAQ,EAAE;IACrD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;IAC5B,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAA;IAClB,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAA;IACpB,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,CAAA;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IACvB,OAAO,CAAC,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;AACvD,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAO,EAAU,EAAE,CAClE,IAAI,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;AAE/B,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAA;AAEnF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAW,EAAE,MAAY,EAAQ,EAAE;IACvD,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK,CAAA;IACxC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAA;IACzC,OAAO;QACN,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;QACjB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;QACjB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;QACjB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;QACjB,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,GAAG,GAAG;QACzB,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,GAAG,GAAG;KACzB,CAAA;AACF,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,EAAE,GAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA"}

package/x/timeline/utils/video-cursor.d.ts

@@ -0,0 +1,10 @@
+/**
+ * A stateful, forward-only frame cursor for a single clip instance.
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
+ */
+export declare class VideoCursor {
+    private reader;
+    constructor(reader: ReadableStreamDefaultReader<VideoFrame>);
+    atOrNear(targetUs: number): Promise<VideoFrame | undefined>;
+    cancel: () => Promise<void>;
+}

package/x/timeline/utils/video-cursor.js

@@ -0,0 +1,36 @@
+/**
+ * A stateful, forward-only frame cursor for a single clip instance.
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
+ */
+export class VideoCursor {
+    reader;
+    constructor(reader) {
+        this.reader = reader;
+    }
+    async atOrNear(targetUs) {
+        let prev = null;
+        while (true) {
+            const { done, value: hit } = await this.reader.read();
+            if (done) {
+                const out = prev ? new VideoFrame(prev) : undefined;
+                prev?.close();
+                return out;
+            }
+            const hitUs = hit.timestamp ?? 0;
+            if (hitUs >= targetUs) {
+                const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY;
+                const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs);
+                const chosen = usePrev ? prev : hit;
+                const other = usePrev ? hit : prev;
+                const copy = new VideoFrame(chosen);
+                chosen.close();
+                other?.close();
+                return copy;
+            }
+            prev?.close();
+            prev = hit;
+        }
+    }
+    cancel = async () => await this.reader.cancel();
+}
+//# sourceMappingURL=video-cursor.js.map

package/x/timeline/utils/video-cursor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-cursor.js","sourceRoot":"","sources":["../../../s/timeline/utils/video-cursor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,OAAO,WAAW;IACF;IAApB,YAAoB,MAA+C;QAA/C,WAAM,GAAN,MAAM,CAAyC;IAAG,CAAC;IAEvE,KAAK,CAAC,QAAQ,CAAC,QAAgB;QAC7B,IAAI,IAAI,GAAsB,IAAI,CAAA;QAClC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YAEnD,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;gBACnD,IAAI,EAAE,KAAK,EAAE,CAAA;gBACb,OAAO,GAAG,CAAA;YACZ,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAChC,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,EAAE,SAAS,IAAI,MAAM,CAAC,iBAAiB,CAAA;gBAC1D,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAA;gBAElF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAK,CAAC,CAAC,CAAC,GAAG,CAAA;gBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;gBAElC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;gBACnC,MAAM,CAAC,KAAK,EAAE,CAAA;gBACd,KAAK,EAAE,KAAK,EAAE,CAAA;gBACd,OAAO,IAAI,CAAA;YACb,CAAC;YAED,IAAI,EAAE,KAAK,EAAE,CAAA;YACb,IAAI,GAAG,GAAG,CAAA;QACZ,CAAC;IACH,CAAC;IAED,MAAM,GAAG,KAAK,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;CAChD"}

package/s/tools/common/loader.ts

@@ -1,26 +0,0 @@
-import {pub, Pub} from "@e280/stz"
-import {ProgressItem} from "../speech-recognition/whisper/parts/types.js"
-
-export interface LoaderEvents {
-	onModelLoadProgress: Pub<ProgressItem[]>
-	onTpsUpdate: Pub<[number]>
-}
-
-export abstract class Loader {
-	tps = 0
-
-	static loaderEvents = {
-		onModelLoadProgress: pub<ProgressItem[]>(),
-		onTpsUpdate: pub<[number]>()
-	}
-
-	constructor(public readonly name: string, public model: string) {}
-
-	abstract init(): Promise<void>
-
-	abstract setModel(model: string): void
-
-	setTps(value: number) {
-		this.tps = value
-	}
-}

package/s/tools/common/transformer-pipeline.ts

@@ -1,26 +0,0 @@
-//@ts-ignore
-import {pipeline} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
-
-import {ProgressCallback} from "../speech-recognition/whisper/parts/types.js"
-
-export class PipelineFactory {
-	instance: any = null
-	model: string | null = null
-
-	constructor(public task: string) {}
-
-	async createInstance(model: string, progressCallback?: ProgressCallback) {
-		this.model = model
-		return this.instance = await pipeline(this.task, this.model, {
-			dtype: {
-				encoder_model:
-					this.model === "onnx-community/whisper-large-v3-turbo"
-						? "fp16"
-						: "fp32",
-				decoder_model_merged: "q4",
-			},
-			device: "webgpu",
-			progress_callback: progressCallback,
-		})
-	}
-}

package/s/tools/speech-recognition/common/model.ts

@@ -1,26 +0,0 @@
-import {pub} from "@e280/stz"
-
-import {Loader} from "../../common/loader.js"
-import {DecoderSource} from "../../../driver/fns/schematic.js"
-import {SpeechRecognizerModels, Word, WordGroup} from "../whisper/parts/types.js"
-
-export abstract class SpeechRecognizer extends Loader {
-	multilingual = true
-
-	static speechRecognizerEvents = {
-		onTranscriptionChunk: pub<Word[]>(),
-		onTranscribeProgress: pub<[number]>()
-	}
-
-	abstract transcribe(input: DecoderSource): Promise<WordGroup>
-
-	setMultilingual(value: boolean) {
-		this.multilingual = value
-	}
-
-	detectLanguage?(input: Blob | AudioBuffer): Promise<string>
-
-	setModel(value: SpeechRecognizerModels) {
-		this.model = value
-	}
-}

package/s/tools/speech-recognition/whisper/fns/host.ts

@@ -1,25 +0,0 @@
-
-import {Comrade} from "@e280/comrade"
-import {ProgressItem} from "../parts/types.js"
-import {SpeechRecognizerHostEvents, WhisperSchematic} from "./schematic.js"
-
-export const setupWhisperHost = (events: SpeechRecognizerHostEvents) => (
-	Comrade.host<WhisperSchematic>(_shell => ({
-		async updateModelLoadProgress(item) {
-			events.onModelLoadProgress.pub(item)
-		},
-		async deliverTranscriptionChunk(chunk) {
-			events.onTranscriptionChunk.pub({
-				text: chunk.text,
-				timestamp: chunk.timestamp
-			})
-		},
-		async updateTps(value) {
-			events.onTpsUpdate.pub(value)
-		},
-		async updateTranscribeProgress(value) {
-			events.onTranscribeProgress(value)
-		}
-	}))
-)
-

package/s/tools/speech-recognition/whisper/fns/schematic.ts

@@ -1,23 +0,0 @@
-import {Pub} from "@e280/stz"
-import {AsSchematic} from "@e280/comrade"
-
-import {LoaderEvents} from "../../../common/loader.js"
-import {ProgressItem, TranscriptionChunk, TranscriptionMessage, TranscriptionResult, Word} from "../parts/types.js"
-
-export type WhisperSchematic = AsSchematic<{
-	work: {
-		transcribe(input: TranscriptionMessage): Promise<TranscriptionResult | null>
-	},
-
-	host: {
-		updateModelLoadProgress(item: ProgressItem): Promise<void>
-		deliverTranscriptionChunk(chunk: TranscriptionChunk): Promise<void>
-		updateTps(value: number): Promise<void>
-		updateTranscribeProgress(value: number): Promise<void>
-	}
-}>
-
-export interface SpeechRecognizerHostEvents extends LoaderEvents {
-	onTranscriptionChunk: Pub<Word[]>
-	onTranscribeProgress: Pub<[number]>
-}

package/s/tools/speech-recognition/whisper/fns/work.ts

@@ -1,91 +0,0 @@
-import {Comrade} from "@e280/comrade"
-//@ts-ignore
-import {pipeline, WhisperTextStreamer} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
-
-import {WhisperSchematic} from "./schematic.js"
-import {TranscriptionChunk} from "../parts/types.js"
-import {PipelineFactory} from "../../../common/transformer-pipeline.js"
-
-// TODO suspicious globals, probably bad
-const pipeline = new PipelineFactory("automatic-speech-recognition")
-let transcriber: any
-
-export const setupWhisperWork = Comrade.work<WhisperSchematic>(shell => ({
-	async transcribe({audio, model, language, duration}) {
-		const isDistil = model.startsWith("distil-whisper/")
-
-		if(!pipeline.model || pipeline.model !== model) {
-			pipeline.instance?.dispose()?.()
-			pipeline.instance = null
-			transcriber = await pipeline.createInstance(
-				model,
-				(data) => {
-					if(data.progress)
-						shell.host.updateModelLoadProgress({
-							id: data.file,
-							progress: data.progress
-						})
-				}
-			)
-		}
-
-		const timePrecision =
-			transcriber.processor.feature_extractor.config.chunk_length /
-			transcriber.model.config.max_source_positions
-
-		const chunkLength = isDistil ? 20 : 30
-		const strideLength = isDistil ? 3 : 5
-
-		let chunkCount = 0
-		let startTime: number | null = null
-		let tokenCount = 0
-		let tps = 0
-
-		const chunkDuration = chunkLength - strideLength
-
-		const estimateProgress = () => {
-			const audioProgressSeconds = chunkCount * chunkDuration
-			return Math.min(audioProgressSeconds / duration, 1)
-		}
-
-		const streamer = new WhisperTextStreamer(transcriber.tokenizer, {
-			time_precision: timePrecision,
-			token_callback_function: () => {
-				startTime ??= performance.now()
-				if (++tokenCount > 1) {
-					tps = (tokenCount / (performance.now() - startTime)) * 1000
-					shell.host.updateTps(tps)
-				}
-			},
-			callback_function: (textChunk: any) => {
-				shell.host.deliverTranscriptionChunk(textChunk)
-			},
-			on_finalize: () => {
-				startTime = null
-				tokenCount = 0
-				chunkCount++
-				const progress = estimateProgress()
-				shell.host.updateTranscribeProgress(progress)
-			},
-		})
-
-		const output = await transcriber(audio, {
-			top_k: 0,
-			do_sample: false,
-			chunk_length_s: chunkLength,
-			stride_length_s: strideLength,
-			language,
-			task: "transcribe",
-			return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
-			force_full_sequences: false,
-			streamer,
-		})
-
-		if (!output) return null
-
-		return {
-			tps,
-			...output,
-		}
-	}
-}))

package/s/tools/speech-recognition/whisper/parts/types.ts

@@ -1,38 +0,0 @@
-export interface ProgressItem {
-	id: string
-	progress: number
-}
-
-export type Word = {
-	text: string
-	timestamp: [start: number, end: number]
-}
-
-export type WordGroup = Word[]
-export type Transcript = WordGroup[]
-
-export interface TranscriptionChunk {
-	text: string
-	offset: number
-	timestamp: [number, number]
-	finalised: boolean
-}
-
-export interface TranscriptionMessage {
-	audio: Float32Array
-	model: string
-	subtask: string | null
-	language: string | null
-	duration: number
-}
-
-export interface TranscriptionResult {
-	text: string
-	chunks: TranscriptionChunk[]
-	tps: number
-}
-
-export type ProgressCallback = (data: any) => void
-
-export type SpeechRecognizerModels = "onnx-community/whisper-tiny_timestamped"
-export type SpeechRecognizerSubtasks = "transcribe"

package/s/tools/speech-recognition/whisper/parts/worker.bundle.ts

@@ -1,7 +0,0 @@
-import {Comrade} from "@e280/comrade"
-
-import {setupWhisperWork} from "../fns/work.js"
-import {WhisperSchematic} from "../fns/schematic.js"
-
-await Comrade.worker<WhisperSchematic>(setupWhisperWork)
-

package/s/tools/speech-recognition/whisper/tool.ts

@@ -1,70 +0,0 @@
-import {Comrade, Thread} from "@e280/comrade"
-
-import {WordGroup} from "./parts/types.js"
-import {context} from "../../../context.js"
-import {setupWhisperHost} from "./fns/host.js"
-import {SpeechRecognizer} from "../common/model.js"
-import {WhisperSchematic} from "./fns/schematic.js"
-
-export class Whisper extends SpeechRecognizer {
-	constructor(public thread: Thread<WhisperSchematic>) {
-		super('whisper', "onnx-community/whisper-tiny_timestamped")
-	}
-
-	static async setup() {
-		const thread = await Comrade.thread<WhisperSchematic>({
-			label: "OmnitoolDriver",
-			workerUrl: new URL("/tools/speech-recognition/whisper/parts/worker.bundle.min.js", import.meta.url),
-			setupHost: setupWhisperHost({
-				...this.loaderEvents,
-				...this.speechRecognizerEvents
-			})
-		})
-		return new this(thread)
-	}
-
-	async init() {
-		// there should be called loading of the model in worker instead when transcribe is called ..
-	}
-
-	async #transcribe(source: Blob, options?: {multilingual?: boolean, language?: string}) {
-		const arrayBuffer = await source.arrayBuffer()
-		const audioCTX = new AudioContext({sampleRate: 16000})
-		const audioData = await audioCTX.decodeAudioData(arrayBuffer)
-		let audio
-		if (audioData.numberOfChannels === 2) {
-			const SCALING_FACTOR = Math.sqrt(2)
-			const left = audioData.getChannelData(0)
-			const right = audioData.getChannelData(1)
-			audio = new Float32Array(left.length)
-			for (let i = 0; i < audioData.length; ++i) {
-				audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
-			}
-		} else {
-			audio = audioData.getChannelData(0)
-		}
-		const driver = await context.driver
-		const duration = await driver.getAudioDuration(source)
-		return await this.thread.work.transcribe({
-			audio,
-			duration,
-			model: this.model,
-			subtask: this.multilingual ? "transcribe" : null,
-			language:
-				this.multilingual && options?.language !== "auto"
-					? options?.language ?? "english"
-					: null
-		})
-	}
-
-	async transcribe(source: Blob): Promise<WordGroup> {
-		const result = await this.#transcribe(source)
-
-		const words = result?.chunks.map((chunk: any) => ({
-			text: chunk.text.trim(),
-			timestamp: chunk.timestamp,
-		})) as WordGroup
-
-		return words
-	}
-}

package/x/tools/common/loader.d.ts

@@ -1,19 +0,0 @@
-import { Pub } from "@e280/stz";
-import { ProgressItem } from "../speech-recognition/whisper/parts/types.js";
-export interface LoaderEvents {
-    onModelLoadProgress: Pub<ProgressItem[]>;
-    onTpsUpdate: Pub<[number]>;
-}
-export declare abstract class Loader {
-    readonly name: string;
-    model: string;
-    tps: number;
-    static loaderEvents: {
-        onModelLoadProgress: Pub<ProgressItem[]>;
-        onTpsUpdate: Pub<[number]>;
-    };
-    constructor(name: string, model: string);
-    abstract init(): Promise<void>;
-    abstract setModel(model: string): void;
-    setTps(value: number): void;
-}

package/x/tools/common/loader.js

@@ -1,18 +0,0 @@
-import { pub } from "@e280/stz";
-export class Loader {
-    name;
-    model;
-    tps = 0;
-    static loaderEvents = {
-        onModelLoadProgress: pub(),
-        onTpsUpdate: pub()
-    };
-    constructor(name, model) {
-        this.name = name;
-        this.model = model;
-    }
-    setTps(value) {
-        this.tps = value;
-    }
-}
-//# sourceMappingURL=loader.js.map

package/x/tools/common/loader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../../s/tools/common/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,GAAG,EAAM,MAAM,WAAW,CAAA;AAQlC,MAAM,OAAgB,MAAM;IAQC;IAAqB;IAPjD,GAAG,GAAG,CAAC,CAAA;IAEP,MAAM,CAAC,YAAY,GAAG;QACrB,mBAAmB,EAAE,GAAG,EAAkB;QAC1C,WAAW,EAAE,GAAG,EAAY;KAC5B,CAAA;IAED,YAA4B,IAAY,EAAS,KAAa;QAAlC,SAAI,GAAJ,IAAI,CAAQ;QAAS,UAAK,GAAL,KAAK,CAAQ;IAAG,CAAC;IAMlE,MAAM,CAAC,KAAa;QACnB,IAAI,CAAC,GAAG,GAAG,KAAK,CAAA;IACjB,CAAC"}

package/x/tools/common/transformer-pipeline.d.ts

@@ -1,8 +0,0 @@
-import { ProgressCallback } from "../speech-recognition/whisper/parts/types.js";
-export declare class PipelineFactory {
-    task: string;
-    instance: any;
-    model: string | null;
-    constructor(task: string);
-    createInstance(model: string, progressCallback?: ProgressCallback): Promise<any>;
-}

package/x/tools/common/transformer-pipeline.js

@@ -1,24 +0,0 @@
-//@ts-ignore
-import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js";
-export class PipelineFactory {
-    task;
-    instance = null;
-    model = null;
-    constructor(task) {
-        this.task = task;
-    }
-    async createInstance(model, progressCallback) {
-        this.model = model;
-        return this.instance = await pipeline(this.task, this.model, {
-            dtype: {
-                encoder_model: this.model === "onnx-community/whisper-large-v3-turbo"
-                    ? "fp16"
-                    : "fp32",
-                decoder_model_merged: "q4",
-            },
-            device: "webgpu",
-            progress_callback: progressCallback,
-        });
-    }
-}
-//# sourceMappingURL=transformer-pipeline.js.map

package/x/tools/common/transformer-pipeline.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformer-pipeline.js","sourceRoot":"","sources":["../../../s/tools/common/transformer-pipeline.ts"],"names":[],"mappings":"AAAA,YAAY;AACZ,OAAO,EAAC,QAAQ,EAAC,MAAM,uFAAuF,CAAA;AAI9G,MAAM,OAAO,eAAe;IAIR;IAHnB,QAAQ,GAAQ,IAAI,CAAA;IACpB,KAAK,GAAkB,IAAI,CAAA;IAE3B,YAAmB,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAEnC,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,gBAAmC;QACtE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,OAAO,IAAI,CAAC,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE;YAC5D,KAAK,EAAE;gBACN,aAAa,EACZ,IAAI,CAAC,KAAK,KAAK,uCAAuC;oBACrD,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,MAAM;gBACV,oBAAoB,EAAE,IAAI;aAC1B;YACD,MAAM,EAAE,QAAQ;YAChB,iBAAiB,EAAE,gBAAgB;SACnC,CAAC,CAAA;IACH,CAAC;CACD"}

package/x/tools/speech-recognition/common/model.d.ts

@@ -1,14 +0,0 @@
-import { Loader } from "../../common/loader.js";
-import { DecoderSource } from "../../../driver/fns/schematic.js";
-import { SpeechRecognizerModels, Word, WordGroup } from "../whisper/parts/types.js";
-export declare abstract class SpeechRecognizer extends Loader {
-    multilingual: boolean;
-    static speechRecognizerEvents: {
-        onTranscriptionChunk: import("@e280/stz").Pub<Word[]>;
-        onTranscribeProgress: import("@e280/stz").Pub<[number]>;
-    };
-    abstract transcribe(input: DecoderSource): Promise<WordGroup>;
-    setMultilingual(value: boolean): void;
-    detectLanguage?(input: Blob | AudioBuffer): Promise<string>;
-    setModel(value: SpeechRecognizerModels): void;
-}

package/x/tools/speech-recognition/common/model.js

@@ -1,16 +0,0 @@
-import { pub } from "@e280/stz";
-import { Loader } from "../../common/loader.js";
-export class SpeechRecognizer extends Loader {
-    multilingual = true;
-    static speechRecognizerEvents = {
-        onTranscriptionChunk: pub(),
-        onTranscribeProgress: pub()
-    };
-    setMultilingual(value) {
-        this.multilingual = value;
-    }
-    setModel(value) {
-        this.model = value;
-    }
-}
-//# sourceMappingURL=model.js.map

package/x/tools/speech-recognition/common/model.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model.js","sourceRoot":"","sources":["../../../../s/tools/speech-recognition/common/model.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAE7B,OAAO,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAA;AAI7C,MAAM,OAAgB,gBAAiB,SAAQ,MAAM;IACpD,YAAY,GAAG,IAAI,CAAA;IAEnB,MAAM,CAAC,sBAAsB,GAAG;QAC/B,oBAAoB,EAAE,GAAG,EAAU;QACnC,oBAAoB,EAAE,GAAG,EAAY;KACrC,CAAA;IAID,eAAe,CAAC,KAAc;QAC7B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAA;IAC1B,CAAC;IAID,QAAQ,CAAC,KAA6B;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC"}

package/x/tools/speech-recognition/whisper/fns/host.d.ts

@@ -1,13 +0,0 @@
-import { ProgressItem } from "../parts/types.js";
-import { SpeechRecognizerHostEvents } from "./schematic.js";
-export declare const setupWhisperHost: (events: SpeechRecognizerHostEvents) => import("@e280/comrade").SetupHost<{
-    work: {
-        transcribe(input: import("../parts/types.js").TranscriptionMessage): Promise<import("../parts/types.js").TranscriptionResult | null>;
-    };
-    host: {
-        updateModelLoadProgress(item: ProgressItem): Promise<void>;
-        deliverTranscriptionChunk(chunk: import("../parts/types.js").TranscriptionChunk): Promise<void>;
-        updateTps(value: number): Promise<void>;
-        updateTranscribeProgress(value: number): Promise<void>;
-    };
-}>;