@omnimedia/omnitool 1.1.0-4 → 1.1.0-40

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-4",
+	"version": "1.1.0-40",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -31,13 +31,17 @@
 		"typescript": "^5.9.2"
 	},
 	"dependencies": {
-		"@benev/slate": "^0.3.9",
 		"@e280/comrade": "^0.0.0-23",
 		"@e280/renraku": "^0.5.0-29",
-		"@e280/stz": "^0.0.0-34",
+		"@e280/sly": "^0.2.0-32",
+		"@e280/strata": "^0.2.1",
+		"@e280/stz": "^0.2.13",
 		"@huggingface/transformers": "^3.7.1",
 		"comrade": "^0.0.3",
-		"mediabunny": "^1.4.4",
+		"gl-transitions": "^1.43.0",
+		"gsap": "^3.13.0",
+		"lit": "^3.3.1",
+		"mediabunny": "^1.14.3",
 		"mp4-muxer": "^5.2.1",
 		"pixi.js": "^8.10.1",
 		"wavesurfer.js": "^7.10.0",

package/s/context.ts

@@ -1,7 +0,0 @@
-import {Driver} from "./driver/driver.js"
-
-const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
-
-export const context = {
-	driver: Driver.setup({workerUrl})
-}

package/s/demo/demo.bundle.ts

@@ -1,21 +1,51 @@
 
-import {context} from "../context.js"
+import {Driver} from "../driver/driver.js"
 import {waveformTest} from "./routines/waveform-test.js"
 import {filmstripTest} from "./routines/filmstrip-test.js"
 import {transcriberTest} from "./routines/transcriber-test.js"
 import {setupTranscodeTest} from "./routines/transcode-test.js"
+import {Datafile, Omni, VideoPlayer} from "../timeline/index.js"
 
-const driver = await context.driver
+const driver = await Driver.setup({workerUrl: new URL("../driver/driver.worker.bundle.min.js", import.meta.url)})
 const results = document.querySelector(".results")!
 
 const fetchButton = document.querySelector(".fetch")
 const importButton = document.querySelector(".import") as HTMLButtonElement
 
+const playButton = document.querySelector(".play") as HTMLButtonElement
+const stopButton = document.querySelector(".stop") as HTMLButtonElement
+const seekButton = document.querySelector(".seek") as HTMLButtonElement
+
 fetchButton?.addEventListener("click", startDemoFetch)
 importButton?.addEventListener("click", startDemoImport)
 
-waveformTest()
-const transcriber = await transcriberTest(driver)
+const omni = new Omni(driver)
+const file = await fetch("/assets/temp/gl.mp4")
+const buffer = await file.arrayBuffer()
+const uint = new Uint8Array(buffer)
+
+const {videoA} = await omni.load({videoA: Datafile.make(uint)})
+const timeline =	omni.timeline(o =>
+	o.sequence(
+	o.stack(
+		o.video(videoA, {duration: 5000}),
+		o.audio(videoA, {duration: 8000})
+	),
+	o.video(videoA, {duration: 7000})
+))
+
+const player = await VideoPlayer.create(driver, timeline)
+document.body.appendChild(player.canvas)
+
+playButton.addEventListener("click", () => player.play())
+stopButton.addEventListener("click", () => player.pause())
+seekButton.addEventListener("change", async (e: Event) => {
+	const target = e.target as HTMLInputElement
+	await player.seek(+target.value)
+})
+
+waveformTest(driver)
+// const transcriber = await transcriberTest(driver)
 
 // hello world test
 {
@@ -32,7 +62,7 @@ async function startDemoImport()
 	const transcode = setupTranscodeTest(driver, file)
 	await filmstripTest(file)
 	run(transcode, fileHandle.name)
-	await transcriber.transcribe(file)
+	// await transcriber.transcribe(file)
 }
 
 async function startDemoFetch()

package/s/demo/demo.css

@@ -46,6 +46,11 @@
 		}
 	}
 
+	canvas {
+		width: 500px;
+		height: 300px;
+	}
+
 	#filmstrip {
 		display: flex;
 		overflow-x: scroll;

package/s/demo/routines/transcode-test.ts

@@ -10,7 +10,7 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 	const ctx = canvas.getContext("2d")
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const composed = await driver.composite([
@@ -21,8 +21,10 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 					{
 						kind: "text",
 						content: "omnitool",
-						fontSize: 50,
-						color: "green"
+						style: {
+							fontSize: 50,
+							fill: "green"
+						}
 					}
 				])
 				frame.close()
@@ -30,9 +32,11 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 				return composed
 			}
 		})
+		const audio = driver.decodeAudio({source})
 
 		await driver.encode({
-			readables,
+			video,
+			audio,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transitions-test.ts

@@ -0,0 +1,43 @@
+import {Application, Sprite} from "pixi.js"
+
+import {Driver} from "../../driver/driver.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export async function setupTransitionsTest(driver: Driver, source: DecoderSource) {
+	const app = new Application()
+	await app.init({width: 300, height: 300, preference: "webgl"})
+	const sprite = new Sprite({width: 300, height: 300})
+
+	app.stage.addChild(sprite)
+
+	document.body.appendChild(app.canvas)
+	const transition = makeTransition({name: "circle", renderer: app.renderer})
+
+	async function run() {
+		const video = driver.decodeVideo({
+			source,
+			async onFrame(frame) {
+				const texture = transition.render({
+					from: frame,
+					to: frame,
+					progress: 0.7,
+					width: app.canvas.width,
+					height: app.canvas.height
+				})
+				sprite.texture = texture
+				return frame
+			}
+		})
+
+		await driver.encode({
+			video,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000}
+			}
+		})
+	}
+
+	return {run}
+}

package/s/demo/routines/waveform-test.ts

@@ -1,9 +1,10 @@
+import {Driver} from "../../driver/driver.js"
 import {Waveform} from "../../timeline/parts/waveform.js"
 
-export async function waveformTest() {
+export async function waveformTest(driver: Driver) {
 	const container = document.querySelector(".waveform-demo") as HTMLElement
 	const widthSlider = document.querySelector(".width") as HTMLInputElement
-	const waveform = await Waveform.init("/assets/temp/gl.mp4", container)
+	const waveform = await Waveform.init(driver, "/assets/temp/gl.mp4", container)
 
 	widthSlider.addEventListener("change", () => {
 		const width = +widthSlider.value

package/s/driver/driver.ts

@@ -11,11 +11,11 @@ export type DriverOptions = {
 }
 
 export class Driver {
-	static async setup(options: DriverOptions) {
+	static async setup(options?: DriverOptions) {
 		const machina = new Machina()
 		const thread = await Comrade.thread<DriverSchematic>({
 			label: "OmnitoolDriver",
-			workerUrl: options.workerUrl,
+			workerUrl: options?.workerUrl ?? "/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js",
 			setupHost: setupDriverHost(machina),
 		})
 		return new this(machina, thread)
@@ -50,7 +50,7 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
@@ -61,19 +61,27 @@ export class Driver {
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
 			source: input.source,
 			video: videoTransform.writable,
+			start: input.start,
+			end: input.end
+		})
+		return videoTransform.readable
+	}
+
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+			source: input.source,
 			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
-		}
+		return audioTransform.readable
 	}
 
-	async encode({readables, config}: EncoderInput) {
+	async encode({video, audio, config}: EncoderInput) {
 		const handle = await window.showSaveFilePicker()
 		const writable = await handle.createWritable()
 		// making bridge because file picker writable is not transferable
@@ -85,7 +93,7 @@ export class Driver {
 				await writable.close()
 			}
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return await this.thread.work.encode[tune]({transfer: [audio ?? [], video ?? [], bridge]})({video, audio, config, bridge})
 	}
 
 	async composite(

package/s/driver/fns/schematic.ts

@@ -1,17 +1,28 @@
 
+import {TextStyleOptions} from "pixi.js"
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Mat6} from "../../timeline/utils/matrix.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
+		}): Promise<void>
+
+		decodeVideo(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
 		}): Promise<void>
 
 		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
@@ -26,20 +37,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
 export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +72,35 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
-
 export type TextLayer = {
 	kind: 'text'
 	content: string
-	fontSize?: number
-	color?: string
-} & Transform
+	style?: TextStyleOptions
+	matrix?: Mat6
+}
 
 export type ImageLayer = {
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+}
+
+export type TransitionLayer = {
+  kind: 'transition'
+  name: string
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	kind: 'gap'
+}
+
+export type Audio = {
+	kind: "audio"
+	data: AudioData
+}
 
-export type Layer = TextLayer | ImageLayer
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer