@omnimedia/omnitool 1.1.0-1 → 1.1.0-10

package/s/timeline/sugar/o.ts

@@ -1,51 +1,119 @@
 
-import {MapG} from "@e280/stz"
-import {Id} from "../parts/basics.js"
 import {Media} from "../parts/media.js"
+import {Id, TimelineFile} from "../parts/basics.js"
 import {Effect, Item, Kind} from "../parts/item.js"
+import {Transform, TransformOptions, Vec2} from "../types.js"
 
 export class O {
 	#nextId = 0
-	#items = new MapG<Id, Item.Any>()
+
+	constructor(public state: {project: TimelineFile}) {}
+
+  require<T extends Item.Any>(id: Id): T {
+    const item = this.state.project.items.find(item => item.id === id)
+    return item as T
+  }
 
 	#getId() {
 		return this.#nextId++
 	}
 
-	register(item: Item.Any) {
-		if (!this.#items.has(item.id))
-			this.#items.set(item.id, item)
-		return item.id
+  #mutate(fn: (project: TimelineFile) => TimelineFile) {
+    this.state.project = fn(this.state.project)
+  }
+
+  spatial = (transform: Transform): Item.Spatial => {
+  	const item: Item.Spatial = {
+  		id: this.#getId(),
+  		kind: Kind.Spatial,
+  		transform
+  	}
+  	return item
+  }
+
+	sequence = (...items: Item.Any[]): Item.Any => {
+		const item =  {
+			id: this.#getId(),
+			kind: Kind.Sequence,
+			childrenIds: items.map(item => item.id)
+		} as Item.Sequence
+
+		this.#mutate(state => {
+			state.items.push(item, ...items)
+			return state
+		})
+
+		return item
 	}
 
-	get items() {
-		return [...this.#items.values()]
+	stack = (...items: Item.Any[]): Item.Any => {
+		const item = {
+			kind: Kind.Stack,
+			id: this.#getId(),
+			childrenIds: items.map(item => item.id)
+		} as Item.Stack
+
+		this.#mutate(state => {
+			state.items.push(item, ...items)
+			return state
+		})
+
+		return item
 	}
 
-	sequence = (...items: Item.Any[]): Item.Sequence => ({
-		id: this.#getId(),
-		kind: Kind.Sequence,
-		children: items.map(item => this.register(item)),
-	})
+	video = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Video => {
 
-	stack = (...items: Item.Any[]): Item.Stack => ({
-		id: this.#getId(),
-		kind: Kind.Stack,
-		children: items.map(item => this.register(item)),
-	})
+		if(!media.hasVideo)
+			throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
 
-	clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
-		id: this.#getId(),
-		kind: Kind.Clip,
-		mediaHash: media.datafile.checksum.hash,
-		start: start ?? 0,
-		duration: duration ?? media.duration,
-	})
+		const item: Item.Video = {
+			kind: Kind.Video,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return item
+	}
+
+	audio = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Audio => {
+
+		if(!media.hasAudio)
+			throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
+
+		const item: Item.Audio = {
+			kind: Kind.Audio,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return item
+	}
 
 	text = (content: string): Item.Text => ({
 		id: this.#getId(),
-		kind: Kind.Text,
 		content,
+		kind: Kind.Text,
+		color: "#FFFFF"
+	})
+
+	gap = (duration: number): Item.Gap => ({
+		id: this.#getId(),
+		kind: Kind.Gap,
+		duration
 	})
 
 	transition = {
@@ -56,5 +124,27 @@ export class O {
 			duration,
 		}),
 	}
+
+  transform = (options?: TransformOptions): Transform => {
+    const position: Vec2 = [
+    	options?.position?.[0] ?? 0,
+    	options?.position?.[1] ?? 0
+    ]
+    const scale: Vec2 = [
+    	options?.scale?.[0] ?? 1,
+    	options?.scale?.[1] ?? 1
+    ]
+    const rotation = options?.rotation ?? 0
+    return [position, scale, rotation]
+  }
+
+  addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
+		this.#mutate(state => {
+			const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
+			parentItem.childrenIds.push(...items.map(item => item.id))
+			state.items.push(...items)
+			return state
+		})
+  }
 }
 

package/s/timeline/sugar/omni-test.ts

@@ -22,10 +22,10 @@ const {mediaA, mediaB} = await omni.load({
 //
 
 const timeline = omni.timeline(o => o.sequence(
-	o.clip(mediaA),
+	o.video(mediaA),
 	o.transition.crossfade(600),
 	o.stack(
-		o.clip(mediaB),
+		o.video(mediaB),
 		o.text("hello world"),
 	),
 ))

package/s/timeline/sugar/omni.ts

@@ -2,12 +2,15 @@
 import {O} from "./o.js"
 import {Item} from "../parts/item.js"
 import {Media} from "../parts/media.js"
-import {TimelineFile} from "../parts/basics.js"
 import {Datafile} from "../utils/datafile.js"
+import {TimelineFile} from "../parts/basics.js"
+import {Export} from "../parts/compositor/export.js"
 import {ResourcePool} from "../parts/resource-pool.js"
+import {RenderConfig} from "../../driver/fns/schematic.js"
 
 export class Omni {
 	resources = new ResourcePool()
+	#export = new Export()
 
 	load = async<S extends Record<string, Promise<Datafile>>>(spec: S) => {
 		return Object.fromEntries(await Promise.all(Object.entries(spec).map(
@@ -15,16 +18,23 @@ export class Omni {
 		))) as {[K in keyof S]: Media}
 	}
 
-	timeline = (fn: (o: O) => Item.Sequence): TimelineFile => {
-		const o = new O()
-		const sequence = fn(o)
-		return {
-			format: "timeline",
-			info: "https://omniclip.app/",
-			version: 0,
-			root: o.register(sequence),
-			items: o.items,
-		}
+	timeline = (fn: (o: O) => Item.Any): TimelineFile => {
+		const o = new O({
+			project: {
+				format: "timeline",
+				info: "https://omniclip.app/",
+				version: 0,
+				items: [],
+				rootId: 0
+			}
+		})
+		const root = fn(o)
+		o.state.project.rootId = root.id
+		return o.state.project
+	}
+
+	render = async (timeline: TimelineFile, config: RenderConfig) => {
+		await this.#export.render(timeline)
 	}
 }
 

package/s/timeline/types.ts

@@ -0,0 +1,29 @@
+export type Interpolation = "linear" | "catmullRom"
+export type Keyframe<Value = number> = [time: number, value: Value]
+export type Keyframes<Value = number> = Keyframe<Value>[]
+export type Vec2 = [x: number, y: number]
+export type Transform = [position: Vec2, scale: Vec2, rotation: number]
+
+export type TrackVec2 = {
+	x: Keyframes
+	y: Keyframes
+}
+
+export type Anim<T> = {
+  terp: Interpolation
+  track: T
+}
+
+export type Animations = Anim<TrackTransform>
+
+export type TrackTransform = {
+	position: TrackVec2
+	scale: TrackVec2
+	rotation: Keyframes
+}
+
+export type TransformOptions = {
+  position?: Vec2
+  scale?: Vec2
+  rotation?: number
+}

package/s/timeline/utils/audio-stream.ts

@@ -0,0 +1,15 @@
+export class AudioStream {
+	constructor(private reader: ReadableStreamDefaultReader<AudioData>) {}
+
+	async *stream(): AsyncGenerator<AudioData> {
+		while (true) {
+			const {done, value: hit} = await this.reader.read()
+			if (done) {
+				break
+			}
+			yield hit
+		}
+	}
+
+	cancel = async () => await this.reader.cancel()
+}

package/s/timeline/utils/checksum.ts

@@ -10,7 +10,8 @@ export class Checksum {
 	) {}
 
 	static async make(data: Uint8Array) {
-		const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data))
+		const data2 = new Uint8Array(data)
+		const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data2))
 		const hash = Hex.fromBytes(bytes)
 		const nickname = Thumbprint.sigil.fromBytes(bytes)
 		return new this(data, bytes, hash, nickname)

package/s/timeline/utils/matrix.ts

@@ -0,0 +1,33 @@
+import {Matrix} from "pixi.js"
+import {Transform} from "../types.js"
+
+export const transformToMat6 = (t: Transform): Mat6 => {
+	const [pos, scl, rotDeg] = t
+	const [x, y] = pos
+	const [sx, sy] = scl
+	const r = rotDeg * Math.PI / 180
+	const cos = Math.cos(r)
+	const sin = Math.sin(r)
+	return [cos * sx, sin * sx, -sin * sy, cos * sy, x, y]
+}
+
+export const mat6ToMatrix = ([a, b, c, d, tx, ty]: Mat6): Matrix =>
+	new Matrix(a, b, c, d, tx, ty)
+
+export const transformToMatrix = (t: Transform) => mat6ToMatrix(transformToMat6(t))
+
+export const mul6 = (local: Mat6, parent: Mat6): Mat6 => {
+	const [a1, b1, c1, d1, tx1, ty1] = local
+	const [a2, b2, c2, d2, tx2, ty2] = parent
+	return [
+		a1 * a2 + c1 * b2,
+		b1 * a2 + d1 * b2,
+		a1 * c2 + c1 * d2,
+		b1 * c2 + d1 * d2,
+		a1 * tx2 + c1 * ty2 + tx1,
+		b1 * tx2 + d1 * ty2 + ty1
+	]
+}
+
+export const I6: Mat6 = [1, 0, 0, 1, 0, 0]
+export type Mat6 = [a: number, b: number, c: number, d: number, tx: number, ty: number]

package/s/timeline/utils/video-cursor.ts

@@ -0,0 +1,40 @@
+/**
+ * A stateful, forward-only frame cursor for a single clip instance.
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
+ */
+
+export class VideoCursor {
+  constructor(private reader: ReadableStreamDefaultReader<VideoFrame>) {}
+
+  async atOrNear(targetUs: number): Promise<VideoFrame | undefined> {
+    let prev: VideoFrame | null = null
+    while (true) {
+      const {done, value: hit} = await this.reader.read()
+
+      if (done) {
+        const out = prev ? new VideoFrame(prev) : undefined
+        prev?.close()
+        return out
+      }
+
+      const hitUs = hit.timestamp ?? 0
+      if (hitUs >= targetUs) {
+        const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
+        const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
+
+        const chosen = usePrev ? prev! : hit
+        const other = usePrev ? hit : prev
+
+        const copy = new VideoFrame(chosen)
+        chosen.close()
+        other?.close()
+        return copy
+      }
+
+      prev?.close()
+      prev = hit
+    }
+  }
+
+  cancel = async () => await this.reader.cancel()
+}

package/s/tools/common/loader.ts

@@ -0,0 +1,26 @@
+import {pub, Pub} from "@e280/stz"
+import {ProgressItem} from "../speech-recognition/whisper/parts/types.js"
+
+export interface LoaderEvents {
+	onModelLoadProgress: Pub<ProgressItem[]>
+	onTpsUpdate: Pub<[number]>
+}
+
+export abstract class Loader {
+	tps = 0
+
+	static loaderEvents = {
+		onModelLoadProgress: pub<ProgressItem[]>(),
+		onTpsUpdate: pub<[number]>()
+	}
+
+	constructor(public readonly name: string, public model: string) {}
+
+	abstract init(): Promise<void>
+
+	abstract setModel(model: string): void
+
+	setTps(value: number) {
+		this.tps = value
+	}
+}

package/s/tools/common/transformer-pipeline.ts

@@ -0,0 +1,26 @@
+//@ts-ignore
+import {pipeline} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
+
+import {ProgressCallback} from "../speech-recognition/whisper/parts/types.js"
+
+export class PipelineFactory {
+	instance: any = null
+	model: string | null = null
+
+	constructor(public task: string) {}
+
+	async createInstance(model: string, progressCallback?: ProgressCallback) {
+		this.model = model
+		return this.instance = await pipeline(this.task, this.model, {
+			dtype: {
+				encoder_model:
+					this.model === "onnx-community/whisper-large-v3-turbo"
+						? "fp16"
+						: "fp32",
+				decoder_model_merged: "q4",
+			},
+			device: "webgpu",
+			progress_callback: progressCallback,
+		})
+	}
+}

package/s/tools/speech-recognition/common/model.ts

@@ -0,0 +1,26 @@
+import {pub} from "@e280/stz"
+
+import {Loader} from "../../common/loader.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {SpeechRecognizerModels, Word, WordGroup} from "../whisper/parts/types.js"
+
+export abstract class SpeechRecognizer extends Loader {
+	multilingual = true
+
+	static speechRecognizerEvents = {
+		onTranscriptionChunk: pub<Word[]>(),
+		onTranscribeProgress: pub<[number]>()
+	}
+
+	abstract transcribe(input: DecoderSource): Promise<WordGroup>
+
+	setMultilingual(value: boolean) {
+		this.multilingual = value
+	}
+
+	detectLanguage?(input: Blob | AudioBuffer): Promise<string>
+
+	setModel(value: SpeechRecognizerModels) {
+		this.model = value
+	}
+}

package/s/tools/speech-recognition/whisper/fns/host.ts

@@ -0,0 +1,25 @@
+
+import {Comrade} from "@e280/comrade"
+import {ProgressItem} from "../parts/types.js"
+import {SpeechRecognizerHostEvents, WhisperSchematic} from "./schematic.js"
+
+export const setupWhisperHost = (events: SpeechRecognizerHostEvents) => (
+	Comrade.host<WhisperSchematic>(_shell => ({
+		async updateModelLoadProgress(item) {
+			events.onModelLoadProgress.pub(item)
+		},
+		async deliverTranscriptionChunk(chunk) {
+			events.onTranscriptionChunk.pub({
+				text: chunk.text,
+				timestamp: chunk.timestamp
+			})
+		},
+		async updateTps(value) {
+			events.onTpsUpdate.pub(value)
+		},
+		async updateTranscribeProgress(value) {
+			events.onTranscribeProgress(value)
+		}
+	}))
+)
+

package/s/tools/speech-recognition/whisper/fns/schematic.ts

@@ -0,0 +1,23 @@
+import {Pub} from "@e280/stz"
+import {AsSchematic} from "@e280/comrade"
+
+import {LoaderEvents} from "../../../common/loader.js"
+import {ProgressItem, TranscriptionChunk, TranscriptionMessage, TranscriptionResult, Word} from "../parts/types.js"
+
+export type WhisperSchematic = AsSchematic<{
+	work: {
+		transcribe(input: TranscriptionMessage): Promise<TranscriptionResult | null>
+	},
+
+	host: {
+		updateModelLoadProgress(item: ProgressItem): Promise<void>
+		deliverTranscriptionChunk(chunk: TranscriptionChunk): Promise<void>
+		updateTps(value: number): Promise<void>
+		updateTranscribeProgress(value: number): Promise<void>
+	}
+}>
+
+export interface SpeechRecognizerHostEvents extends LoaderEvents {
+	onTranscriptionChunk: Pub<Word[]>
+	onTranscribeProgress: Pub<[number]>
+}

package/s/tools/speech-recognition/whisper/fns/work.ts

@@ -0,0 +1,91 @@
+import {Comrade} from "@e280/comrade"
+//@ts-ignore
+import {pipeline, WhisperTextStreamer} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.0/dist/transformers.min.js"
+
+import {WhisperSchematic} from "./schematic.js"
+import {TranscriptionChunk} from "../parts/types.js"
+import {PipelineFactory} from "../../../common/transformer-pipeline.js"
+
+// TODO suspicious globals, probably bad
+const pipeline = new PipelineFactory("automatic-speech-recognition")
+let transcriber: any
+
+export const setupWhisperWork = Comrade.work<WhisperSchematic>(shell => ({
+	async transcribe({audio, model, language, duration}) {
+		const isDistil = model.startsWith("distil-whisper/")
+
+		if(!pipeline.model || pipeline.model !== model) {
+			pipeline.instance?.dispose()?.()
+			pipeline.instance = null
+			transcriber = await pipeline.createInstance(
+				model,
+				(data) => {
+					if(data.progress)
+						shell.host.updateModelLoadProgress({
+							id: data.file,
+							progress: data.progress
+						})
+				}
+			)
+		}
+
+		const timePrecision =
+			transcriber.processor.feature_extractor.config.chunk_length /
+			transcriber.model.config.max_source_positions
+
+		const chunkLength = isDistil ? 20 : 30
+		const strideLength = isDistil ? 3 : 5
+
+		let chunkCount = 0
+		let startTime: number | null = null
+		let tokenCount = 0
+		let tps = 0
+
+		const chunkDuration = chunkLength - strideLength
+
+		const estimateProgress = () => {
+			const audioProgressSeconds = chunkCount * chunkDuration
+			return Math.min(audioProgressSeconds / duration, 1)
+		}
+
+		const streamer = new WhisperTextStreamer(transcriber.tokenizer, {
+			time_precision: timePrecision,
+			token_callback_function: () => {
+				startTime ??= performance.now()
+				if (++tokenCount > 1) {
+					tps = (tokenCount / (performance.now() - startTime)) * 1000
+					shell.host.updateTps(tps)
+				}
+			},
+			callback_function: (textChunk: any) => {
+				shell.host.deliverTranscriptionChunk(textChunk)
+			},
+			on_finalize: () => {
+				startTime = null
+				tokenCount = 0
+				chunkCount++
+				const progress = estimateProgress()
+				shell.host.updateTranscribeProgress(progress)
+			},
+		})
+
+		const output = await transcriber(audio, {
+			top_k: 0,
+			do_sample: false,
+			chunk_length_s: chunkLength,
+			stride_length_s: strideLength,
+			language,
+			task: "transcribe",
+			return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+			force_full_sequences: false,
+			streamer,
+		})
+
+		if (!output) return null
+
+		return {
+			tps,
+			...output,
+		}
+	}
+}))

package/s/tools/speech-recognition/whisper/parts/types.ts

@@ -0,0 +1,38 @@
+export interface ProgressItem {
+	id: string
+	progress: number
+}
+
+export type Word = {
+	text: string
+	timestamp: [start: number, end: number]
+}
+
+export type WordGroup = Word[]
+export type Transcript = WordGroup[]
+
+export interface TranscriptionChunk {
+	text: string
+	offset: number
+	timestamp: [number, number]
+	finalised: boolean
+}
+
+export interface TranscriptionMessage {
+	audio: Float32Array
+	model: string
+	subtask: string | null
+	language: string | null
+	duration: number
+}
+
+export interface TranscriptionResult {
+	text: string
+	chunks: TranscriptionChunk[]
+	tps: number
+}
+
+export type ProgressCallback = (data: any) => void
+
+export type SpeechRecognizerModels = "onnx-community/whisper-tiny_timestamped"
+export type SpeechRecognizerSubtasks = "transcribe"

package/s/tools/speech-recognition/whisper/parts/worker.bundle.ts

@@ -0,0 +1,7 @@
+import {Comrade} from "@e280/comrade"
+
+import {setupWhisperWork} from "../fns/work.js"
+import {WhisperSchematic} from "../fns/schematic.js"
+
+await Comrade.worker<WhisperSchematic>(setupWhisperWork)
+