@omnimedia/omnitool 1.1.0-1 → 1.1.0-10

package/README.md

@@ -26,7 +26,7 @@ Omni Tools is a collection of composable utilities for working with Omniclip tim
 ### Install
 
 ```bash
-npm install @omni/tools
+npm i @omnimedia/omnitool
 ```
 
 ---

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-1",
+	"version": "1.1.0-10",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -23,20 +23,23 @@
 		"test-debug": "node inspect x/tests.test.js"
 	},
 	"devDependencies": {
-		"@e280/science": "^0.0.5",
-		"@e280/scute": "^0.0.0-6",
-		"@types/node": "^24.0.14",
+		"@e280/science": "^0.0.6",
+		"@e280/scute": "^0.0.0-7",
+		"@types/node": "^24.2.0",
 		"http-server": "^14.1.1",
 		"npm-run-all": "^4.1.5",
-		"typescript": "^5.8.3"
+		"typescript": "^5.9.2"
 	},
 	"dependencies": {
 		"@benev/slate": "^0.3.9",
-		"@e280/comrade": "^0.0.0-18",
-		"@e280/renraku": "^0.5.0-19",
-		"@e280/stz": "^0.0.0-22",
+		"@e280/comrade": "^0.0.0-23",
+		"@e280/renraku": "^0.5.0-29",
+		"@e280/stz": "^0.0.0-34",
+		"@huggingface/transformers": "^3.7.1",
 		"comrade": "^0.0.3",
-		"mediabunny": "^1.1.1",
+		"gl-transitions": "^1.43.0",
+		"gsap": "^3.13.0",
+		"mediabunny": "^1.14.3",
 		"mp4-muxer": "^5.2.1",
 		"pixi.js": "^8.10.1",
 		"wavesurfer.js": "^7.10.0",

package/s/context.ts

@@ -1,6 +1,6 @@
 import {Driver} from "./driver/driver.js"
 
-const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url)
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
 
 export const context = {
 	driver: Driver.setup({workerUrl})

package/s/demo/demo.bundle.ts

@@ -2,6 +2,7 @@
 import {context} from "../context.js"
 import {waveformTest} from "./routines/waveform-test.js"
 import {filmstripTest} from "./routines/filmstrip-test.js"
+import {transcriberTest} from "./routines/transcriber-test.js"
 import {setupTranscodeTest} from "./routines/transcode-test.js"
 
 const driver = await context.driver
@@ -14,6 +15,7 @@ fetchButton?.addEventListener("click", startDemoFetch)
 importButton?.addEventListener("click", startDemoImport)
 
 waveformTest()
+const transcriber = await transcriberTest(driver)
 
 // hello world test
 {
@@ -26,9 +28,11 @@ waveformTest()
 async function startDemoImport()
 {
 	const [fileHandle] = await window.showOpenFilePicker()
-	const transcode = setupTranscodeTest(driver, fileHandle)
-	await filmstripTest(fileHandle)
+	const file = await fileHandle.getFile()
+	const transcode = setupTranscodeTest(driver, file)
+	await filmstripTest(file)
 	run(transcode, fileHandle.name)
+	await transcriber.transcribe(file)
 }
 
 async function startDemoFetch()

package/s/demo/demo.css

@@ -46,6 +46,11 @@
 		}
 	}
 
+	canvas {
+		width: 500px;
+		height: 300px;
+	}
+
 	#filmstrip {
 		display: flex;
 		overflow-x: scroll;

package/s/demo/routines/filmstrip-test.ts

@@ -1,6 +1,6 @@
 import {Filmstrip} from "../../timeline/parts/filmstrip.js"
 
-export async function filmstripTest(fileHandle: FileSystemFileHandle) {
+export async function filmstripTest(file: File) {
 	const rangeSlider = document.querySelector(".range") as HTMLInputElement
 	const rangeView = document.querySelector(".range-view")!
 	const rangeSizeSlider = document.querySelector(".range-size")! as HTMLInputElement
@@ -10,7 +10,7 @@ export async function filmstripTest(fileHandle: FileSystemFileHandle) {
 	const FPS_10 = 1000/10 / 1000
 	let rangeSize = 0.5
 	const filmstrip = await Filmstrip.init(
-			fileHandle,
+			file,
 			{
 				onChange(tiles) {
 					// Sort by time (optional, for clean ordering)

package/s/demo/routines/transcode-test.ts

@@ -10,7 +10,7 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 	const ctx = canvas.getContext("2d")
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const composed = await driver.composite([
@@ -30,9 +30,11 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 				return composed
 			}
 		})
+		const audio = driver.decodeAudio({source})
 
 		await driver.encode({
-			readables,
+			video,
+			audio,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transcriber-test.ts

@@ -0,0 +1,34 @@
+import {Driver} from "../../driver/driver.js"
+import {makeTranscriber} from "../../features/speech/transcribe/transcriber.js"
+
+export async function transcriberTest(driver: Driver) {
+	const transcriber = await makeTranscriber({
+		driver,
+		spec: {
+			model: "onnx-community/whisper-tiny_timestamped",
+			device: "webgpu",
+			strideLength: 5,
+			chunkLength: 30,
+			dtype: "fp32"
+		},
+		workerUrl: new URL("/features/speech/transcribe/worker.bundle.min.js", import.meta.url),
+		onLoading({progress, total}) {
+	  	console.log(progress, total, "total")
+		},
+	})
+	return {
+		transcribe: async (file: File) => {
+			const result = await transcriber.transcribe({
+				source: file,
+				language: "english",
+				onReport(report) {
+	  			console.log("report", report)
+				},
+				onTranscription(transcription) {
+	  			console.log("transcript", transcription)
+				}
+			})
+			console.log(result, "transcript result")
+		}
+	}
+}

package/s/demo/routines/transitions-test.ts

@@ -0,0 +1,43 @@
+import {Application, Sprite} from "pixi.js"
+
+import {Driver} from "../../driver/driver.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export async function setupTransitionsTest(driver: Driver, source: DecoderSource) {
+	const app = new Application()
+	await app.init({width: 300, height: 300, preference: "webgl"})
+	const sprite = new Sprite({width: 300, height: 300})
+
+	app.stage.addChild(sprite)
+
+	document.body.appendChild(app.canvas)
+	const transition = makeTransition({name: "circle", renderer: app.renderer})
+
+	async function run() {
+		const video = driver.decodeVideo({
+			source,
+			async onFrame(frame) {
+				const texture = transition.render({
+					from: frame,
+					to: frame,
+					progress: 0.7,
+					width: app.canvas.width,
+					height: app.canvas.height
+				})
+				sprite.texture = texture
+				return frame
+			}
+		})
+
+		await driver.encode({
+			video,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000}
+			}
+		})
+	}
+
+	return {run}
+}

package/s/driver/driver.ts

@@ -50,7 +50,7 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
@@ -61,19 +61,27 @@ export class Driver {
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
 			source: input.source,
 			video: videoTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return videoTransform.readable
+	}
+
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
 			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
-		}
+		return audioTransform.readable
 	}
 
-	async encode({readables, config}: EncoderInput) {
+	async encode({video, audio, config}: EncoderInput) {
 		const handle = await window.showSaveFilePicker()
 		const writable = await handle.createWritable()
 		// making bridge because file picker writable is not transferable
@@ -85,7 +93,7 @@ export class Driver {
 				await writable.close()
 			}
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
 	}
 
 	async composite(

package/s/driver/fns/host.ts

@@ -3,10 +3,11 @@ import {Comrade} from "@e280/comrade"
 import {Machina} from "../parts/machina.js"
 import {DriverSchematic} from "./schematic.js"
 
-export const setupDriverHost = (machina: Machina) => Comrade.host<DriverSchematic>(({work}, rig) => ({
-
-	async world() {
-		machina.count++
-	}
-}))
+export const setupDriverHost = (machina: Machina) => (
+	Comrade.host<DriverSchematic>(_shell => ({
+		async world() {
+			machina.count++
+		},
+	}))
+)
 

package/s/driver/fns/schematic.ts

@@ -2,16 +2,26 @@
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Mat6} from "../../timeline/utils/matrix.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
 		}): Promise<void>
 
 		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
@@ -26,20 +36,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
-export type DecoderSource = FileSystemFileHandle | string
+export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +71,36 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
-
 export type TextLayer = {
 	kind: 'text'
 	content: string
 	fontSize?: number
 	color?: string
-} & Transform
+	matrix?: Mat6
+}
 
 export type ImageLayer = {
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	kind: 'gap'
+}
+
+export type Audio = {
+	kind: "audio"
+	data: AudioData
+}
 
-export type Layer = TextLayer | ImageLayer
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer