@omnimedia/omnitool 1.1.0-7 → 1.1.0-71

package/s/timeline/parts/waveform.ts

@@ -1,62 +0,0 @@
-import WaveSurfer from "wavesurfer.js"
-
-import {context} from "../../context.js"
-import {DecoderSource} from "../../driver/fns/schematic.js"
-
-export class Waveform {
-	wavesurfer: WaveSurfer
-
-	constructor(peaks: number[], container: HTMLElement, duration: number) {
-		this.wavesurfer = WaveSurfer.create({
-  		container,
-  		waveColor: 'rgb(200, 0, 200)',
-  		progressColor: 'rgb(100, 0, 100)',
-  		barWidth: 10,
-  		barRadius: 10,
-  		barGap: 2,
-  		peaks: [peaks],
-  		duration
-		})
-	}
-
-	static async init(source: DecoderSource, container: HTMLElement) {
-		const driver = await context.driver
-		const reader = driver.decodeAudio({source}).getReader()
-
-		const peaks: number[] = []
-		let buffer: number[] = []
-		const samplesPerPeak = 1024
-		const duration = await driver.getAudioDuration(source)
-
-		while (true) {
-			const {done, value: audioData} = await reader.read()
-			if (done) break
-
-			const frames = audioData.numberOfFrames
-			const plane = new Float32Array(frames)
-			audioData.copyTo(plane, {planeIndex: 0}) // Use left channel only
-
-			for (let i = 0; i < plane.length; i++) {
-				buffer.push(plane[i])
-				if (buffer.length >= samplesPerPeak) {
-					const chunk = buffer.splice(0, samplesPerPeak)
-					const min = Math.min(...chunk)
-					const max = Math.max(...chunk)
-					peaks.push(min, max)
-				}
-			}
-
-			audioData.close()
-		}
-
-		return new Waveform(peaks, container, duration ?? 0)
-	}
-
-	// set zoom(value: number) {
-	// 	this.wavesurfer.zoom(value)
-	// }
-
-	set width(value: number) {
-		this.wavesurfer.setOptions({width: value})
-	}
-}

package/s/timeline/sugar/builders.ts

@@ -1,102 +0,0 @@
-import {O} from "./o.js"
-import {Id} from "../parts/basics.js"
-import {Item} from "../parts/item.js"
-
-export class TimelineItem {
-	public readonly id: Id
-
-	constructor(public item: Item.Any) {
-		this.id = item.id
-	}
-
-	toJSON() {
-		return {
-			...this.item
-		}
-	}
-}
-
-abstract class VisualItem extends TimelineItem {
-  abstract spatial(spatial: Spatial): TimelineItem
-}
-
-export class Stack extends VisualItem {
-  constructor(private o: O, public item: Item.Stack) {
-  	super(item)
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-
-  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
-  	const result = fn(this.o)
-  	const items = Array.isArray(result) ? result : [result]
-    this.item.childrenIds.push(...items.map(c => c.item.id))
-    return this
-  }
-}
-
-export class Spatial extends TimelineItem {
-	constructor(public item: Item.Spatial) {super(item)}
-}
-
-export class Gap extends TimelineItem {
-	constructor(public item: Item.Gap) {super(item)}
-}
-
-export class Audio extends TimelineItem {
-	constructor(public item: Item.Audio) {super(item)}
-}
-
-export class Video extends VisualItem {
-  constructor(public item: Item.Video) {
-  	super(item)
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-}
-
-export class Text extends VisualItem {
-  constructor(public item: Item.Text) {
-  	super(item)
-  }
-
-  color(color: string) {
-  	this.item.color = color
-		return this
-  }
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-}
-
-export class Sequence extends VisualItem {
-	constructor(private o: O, public item: Item.Sequence) {
-		super(item)
-	}
-
-  spatial(spatial: Spatial) {
-  	this.item.spatialId = spatial.item.id
-    return this
-  }
-
-  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
-  	const result = fn(this.o)
-  	const items = Array.isArray(result) ? result : [result]
-    this.item.childrenIds.push(...items.map(c => c.item.id))
-    return this
-  }
-}
-
-export class Transition extends TimelineItem {
-	constructor(public item: Item.Transition) {
-		super(item)
-	}
-}

package/s/timeline/sugar/omni-test.ts

@@ -1,38 +0,0 @@
-
-import {Omni} from "./omni.js"
-import {dummyData} from "../utils/dummy-data.js"
-
-//
-// create an omni context
-//
-
-const omni = new Omni()
-
-//
-// load in some media resources
-//
-
-const {mediaA, mediaB} = await omni.load({
-	mediaA: dummyData(),
-	mediaB: dummyData(),
-})
-
-//
-// create a timeline
-//
-
-const timeline = omni.timeline(o => o.sequence(
-	o.video(mediaA),
-	o.transition.crossfade(600),
-	o.stack(
-		o.video(mediaB),
-		o.text("hello world"),
-	),
-))
-
-//
-// log the timeline
-//
-
-console.log(JSON.stringify(timeline, undefined, "  "))
-

package/s/timeline/timeline.ts

@@ -1,22 +0,0 @@
-import {TimelineItem} from "./sugar/builders.js"
-import {Id, TimelineFile} from "./parts/basics.js"
-
-export class Timeline {
-  constructor(public root: TimelineItem, private items: Map<Id, TimelineItem>) {}
-
-  require<T extends TimelineItem>(id: Id): T {
-    const item = this.items.get(id)
-    return item as T
-  }
-
-  toJSON(): TimelineFile {
-    return {
-			format: "timeline",
-			info: "https://omniclip.app/",
-			version: 0,
-			rootId: this.root.item.id,
-			items: Array.from(this.items.values()).map(item => item.toJSON())
-    }
-  }
-}
-

package/s/timeline/utils/audio-stream.ts

@@ -1,15 +0,0 @@
-export class AudioStream {
-	constructor(private reader: ReadableStreamDefaultReader<AudioData>) {}
-
-	async *stream(): AsyncGenerator<AudioData> {
-		while (true) {
-			const {done, value: hit} = await this.reader.read()
-			if (done) {
-				break
-			}
-			yield hit
-		}
-	}
-
-	cancel = async () => await this.reader.cancel()
-}

package/s/timeline/utils/video-cursor.ts

@@ -1,40 +0,0 @@
-/**
- * A stateful, forward-only frame cursor for a single clip instance.
- * It efficiently reads a video stream to find the frame nearest to a target timestamp.
- */
-
-export class VideoCursor {
-  constructor(private reader: ReadableStreamDefaultReader<VideoFrame>) {}
-
-  async atOrNear(targetUs: number): Promise<VideoFrame | undefined> {
-    let prev: VideoFrame | null = null
-    while (true) {
-      const {done, value: hit} = await this.reader.read()
-
-      if (done) {
-        const out = prev ? new VideoFrame(prev) : undefined
-        prev?.close()
-        return out
-      }
-
-      const hitUs = hit.timestamp ?? 0
-      if (hitUs >= targetUs) {
-        const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
-        const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
-
-        const chosen = usePrev ? prev! : hit
-        const other = usePrev ? hit : prev
-
-        const copy = new VideoFrame(chosen)
-        chosen.close()
-        other?.close()
-        return copy
-      }
-
-      prev?.close()
-      prev = hit
-    }
-  }
-
-  cancel = async () => await this.reader.cancel()
-}

package/s/tools/common/loader.ts

@@ -1,26 +0,0 @@
-import {pub, Pub} from "@e280/stz"
-import {ProgressItem} from "../speech-recognition/whisper/parts/types.js"
-
-export interface LoaderEvents {
-	onModelLoadProgress: Pub<ProgressItem[]>
-	onTpsUpdate: Pub<[number]>
-}
-
-export abstract class Loader {
-	tps = 0
-
-	static loaderEvents = {
-		onModelLoadProgress: pub<ProgressItem[]>(),
-		onTpsUpdate: pub<[number]>()
-	}
-
-	constructor(public readonly name: string, public model: string) {}
-
-	abstract init(): Promise<void>
-
-	abstract setModel(model: string): void
-
-	setTps(value: number) {
-		this.tps = value
-	}
-}

package/s/tools/common/transformer-pipeline.ts

@@ -1,26 +0,0 @@
-//@ts-ignore
-import {pipeline} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
-
-import {ProgressCallback} from "../speech-recognition/whisper/parts/types.js"
-
-export class PipelineFactory {
-	instance: any = null
-	model: string | null = null
-
-	constructor(public task: string) {}
-
-	async createInstance(model: string, progressCallback?: ProgressCallback) {
-		this.model = model
-		return this.instance = await pipeline(this.task, this.model, {
-			dtype: {
-				encoder_model:
-					this.model === "onnx-community/whisper-large-v3-turbo"
-						? "fp16"
-						: "fp32",
-				decoder_model_merged: "q4",
-			},
-			device: "webgpu",
-			progress_callback: progressCallback,
-		})
-	}
-}

package/s/tools/speech-recognition/common/model.ts

@@ -1,26 +0,0 @@
-import {pub} from "@e280/stz"
-
-import {Loader} from "../../common/loader.js"
-import {DecoderSource} from "../../../driver/fns/schematic.js"
-import {SpeechRecognizerModels, Word, WordGroup} from "../whisper/parts/types.js"
-
-export abstract class SpeechRecognizer extends Loader {
-	multilingual = true
-
-	static speechRecognizerEvents = {
-		onTranscriptionChunk: pub<Word[]>(),
-		onTranscribeProgress: pub<[number]>()
-	}
-
-	abstract transcribe(input: DecoderSource): Promise<WordGroup>
-
-	setMultilingual(value: boolean) {
-		this.multilingual = value
-	}
-
-	detectLanguage?(input: Blob | AudioBuffer): Promise<string>
-
-	setModel(value: SpeechRecognizerModels) {
-		this.model = value
-	}
-}

package/s/tools/speech-recognition/whisper/fns/host.ts

@@ -1,25 +0,0 @@
-
-import {Comrade} from "@e280/comrade"
-import {ProgressItem} from "../parts/types.js"
-import {SpeechRecognizerHostEvents, WhisperSchematic} from "./schematic.js"
-
-export const setupWhisperHost = (events: SpeechRecognizerHostEvents) => (
-	Comrade.host<WhisperSchematic>(_shell => ({
-		async updateModelLoadProgress(item) {
-			events.onModelLoadProgress.pub(item)
-		},
-		async deliverTranscriptionChunk(chunk) {
-			events.onTranscriptionChunk.pub({
-				text: chunk.text,
-				timestamp: chunk.timestamp
-			})
-		},
-		async updateTps(value) {
-			events.onTpsUpdate.pub(value)
-		},
-		async updateTranscribeProgress(value) {
-			events.onTranscribeProgress(value)
-		}
-	}))
-)
-

package/s/tools/speech-recognition/whisper/fns/schematic.ts

@@ -1,23 +0,0 @@
-import {Pub} from "@e280/stz"
-import {AsSchematic} from "@e280/comrade"
-
-import {LoaderEvents} from "../../../common/loader.js"
-import {ProgressItem, TranscriptionChunk, TranscriptionMessage, TranscriptionResult, Word} from "../parts/types.js"
-
-export type WhisperSchematic = AsSchematic<{
-	work: {
-		transcribe(input: TranscriptionMessage): Promise<TranscriptionResult | null>
-	},
-
-	host: {
-		updateModelLoadProgress(item: ProgressItem): Promise<void>
-		deliverTranscriptionChunk(chunk: TranscriptionChunk): Promise<void>
-		updateTps(value: number): Promise<void>
-		updateTranscribeProgress(value: number): Promise<void>
-	}
-}>
-
-export interface SpeechRecognizerHostEvents extends LoaderEvents {
-	onTranscriptionChunk: Pub<Word[]>
-	onTranscribeProgress: Pub<[number]>
-}

package/s/tools/speech-recognition/whisper/fns/work.ts

@@ -1,91 +0,0 @@
-import {Comrade} from "@e280/comrade"
-//@ts-ignore
-import {pipeline, WhisperTextStreamer} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
-
-import {WhisperSchematic} from "./schematic.js"
-import {TranscriptionChunk} from "../parts/types.js"
-import {PipelineFactory} from "../../../common/transformer-pipeline.js"
-
-// TODO suspicious globals, probably bad
-const pipeline = new PipelineFactory("automatic-speech-recognition")
-let transcriber: any
-
-export const setupWhisperWork = Comrade.work<WhisperSchematic>(shell => ({
-	async transcribe({audio, model, language, duration}) {
-		const isDistil = model.startsWith("distil-whisper/")
-
-		if(!pipeline.model || pipeline.model !== model) {
-			pipeline.instance?.dispose()?.()
-			pipeline.instance = null
-			transcriber = await pipeline.createInstance(
-				model,
-				(data) => {
-					if(data.progress)
-						shell.host.updateModelLoadProgress({
-							id: data.file,
-							progress: data.progress
-						})
-				}
-			)
-		}
-
-		const timePrecision =
-			transcriber.processor.feature_extractor.config.chunk_length /
-			transcriber.model.config.max_source_positions
-
-		const chunkLength = isDistil ? 20 : 30
-		const strideLength = isDistil ? 3 : 5
-
-		let chunkCount = 0
-		let startTime: number | null = null
-		let tokenCount = 0
-		let tps = 0
-
-		const chunkDuration = chunkLength - strideLength
-
-		const estimateProgress = () => {
-			const audioProgressSeconds = chunkCount * chunkDuration
-			return Math.min(audioProgressSeconds / duration, 1)
-		}
-
-		const streamer = new WhisperTextStreamer(transcriber.tokenizer, {
-			time_precision: timePrecision,
-			token_callback_function: () => {
-				startTime ??= performance.now()
-				if (++tokenCount > 1) {
-					tps = (tokenCount / (performance.now() - startTime)) * 1000
-					shell.host.updateTps(tps)
-				}
-			},
-			callback_function: (textChunk: any) => {
-				shell.host.deliverTranscriptionChunk(textChunk)
-			},
-			on_finalize: () => {
-				startTime = null
-				tokenCount = 0
-				chunkCount++
-				const progress = estimateProgress()
-				shell.host.updateTranscribeProgress(progress)
-			},
-		})
-
-		const output = await transcriber(audio, {
-			top_k: 0,
-			do_sample: false,
-			chunk_length_s: chunkLength,
-			stride_length_s: strideLength,
-			language,
-			task: "transcribe",
-			return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
-			force_full_sequences: false,
-			streamer,
-		})
-
-		if (!output) return null
-
-		return {
-			tps,
-			...output,
-		}
-	}
-}))

package/s/tools/speech-recognition/whisper/parts/types.ts

@@ -1,38 +0,0 @@
-export interface ProgressItem {
-	id: string
-	progress: number
-}
-
-export type Word = {
-	text: string
-	timestamp: [start: number, end: number]
-}
-
-export type WordGroup = Word[]
-export type Transcript = WordGroup[]
-
-export interface TranscriptionChunk {
-	text: string
-	offset: number
-	timestamp: [number, number]
-	finalised: boolean
-}
-
-export interface TranscriptionMessage {
-	audio: Float32Array
-	model: string
-	subtask: string | null
-	language: string | null
-	duration: number
-}
-
-export interface TranscriptionResult {
-	text: string
-	chunks: TranscriptionChunk[]
-	tps: number
-}
-
-export type ProgressCallback = (data: any) => void
-
-export type SpeechRecognizerModels = "onnx-community/whisper-tiny_timestamped"
-export type SpeechRecognizerSubtasks = "transcribe"

package/s/tools/speech-recognition/whisper/parts/worker.bundle.ts

@@ -1,7 +0,0 @@
-import {Comrade} from "@e280/comrade"
-
-import {setupWhisperWork} from "../fns/work.js"
-import {WhisperSchematic} from "../fns/schematic.js"
-
-await Comrade.worker<WhisperSchematic>(setupWhisperWork)
-

package/s/tools/speech-recognition/whisper/tool.ts

@@ -1,70 +0,0 @@
-import {Comrade, Thread} from "@e280/comrade"
-
-import {WordGroup} from "./parts/types.js"
-import {context} from "../../../context.js"
-import {setupWhisperHost} from "./fns/host.js"
-import {SpeechRecognizer} from "../common/model.js"
-import {WhisperSchematic} from "./fns/schematic.js"
-
-export class Whisper extends SpeechRecognizer {
-	constructor(public thread: Thread<WhisperSchematic>) {
-		super('whisper', "onnx-community/whisper-tiny_timestamped")
-	}
-
-	static async setup() {
-		const thread = await Comrade.thread<WhisperSchematic>({
-			label: "OmnitoolDriver",
-			workerUrl: new URL("/tools/speech-recognition/whisper/parts/worker.bundle.min.js", import.meta.url),
-			setupHost: setupWhisperHost({
-				...this.loaderEvents,
-				...this.speechRecognizerEvents
-			})
-		})
-		return new this(thread)
-	}
-
-	async init() {
-		// there should be called loading of the model in worker instead when transcribe is called ..
-	}
-
-	async #transcribe(source: Blob, options?: {multilingual?: boolean, language?: string}) {
-		const arrayBuffer = await source.arrayBuffer()
-		const audioCTX = new AudioContext({sampleRate: 16000})
-		const audioData = await audioCTX.decodeAudioData(arrayBuffer)
-		let audio
-		if (audioData.numberOfChannels === 2) {
-			const SCALING_FACTOR = Math.sqrt(2)
-			const left = audioData.getChannelData(0)
-			const right = audioData.getChannelData(1)
-			audio = new Float32Array(left.length)
-			for (let i = 0; i < audioData.length; ++i) {
-				audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2
-			}
-		} else {
-			audio = audioData.getChannelData(0)
-		}
-		const driver = await context.driver
-		const duration = await driver.getAudioDuration(source)
-		return await this.thread.work.transcribe({
-			audio,
-			duration,
-			model: this.model,
-			subtask: this.multilingual ? "transcribe" : null,
-			language:
-				this.multilingual && options?.language !== "auto"
-					? options?.language ?? "english"
-					: null
-		})
-	}
-
-	async transcribe(source: Blob): Promise<WordGroup> {
-		const result = await this.#transcribe(source)
-
-		const words = result?.chunks.map((chunk: any) => ({
-			text: chunk.text.trim(),
-			timestamp: chunk.timestamp,
-		})) as WordGroup
-
-		return words
-	}
-}

package/x/tests.test.js

@@ -1,6 +0,0 @@
-import { Science } from "@e280/science";
-import driver from "./driver/driver.test.js";
-await Science.run({
-    driver,
-});
-//# sourceMappingURL=tests.test.js.map

package/x/tests.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tests.test.js","sourceRoot":"","sources":["../s/tests.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,MAAM,MAAM,yBAAyB,CAAA;AAE5C,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,MAAM;CACN,CAAC,CAAA"}

package/x/timeline/parts/compositor/export.d.ts

@@ -1,9 +0,0 @@
-import { TimelineFile } from "../basics.js";
-import { DecoderSource } from "../../../driver/fns/schematic.js";
-export declare class Export {
-    #private;
-    private framerate;
-    private resolveMedia;
-    constructor(framerate?: number, resolveMedia?: (hash: string) => DecoderSource);
-    render(timeline: TimelineFile): Promise<void>;
-}