@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/demo/routines/timeline-setup.ts

@@ -0,0 +1,81 @@
+
+import {Driver} from "../../driver/driver.js"
+import {Datafile, defaultTranscriberSpec, Item, makeTranscriber, Omni} from "../../timeline/index.js"
+
+export async function TimelineSchemaTest(driver: Driver, file: File) {
+	const transcriber = await makeTranscriber({
+		driver,
+		spec: defaultTranscriberSpec(),
+		workerUrl: new URL("/features/speech/transcribe/worker.bundle.min.js", import.meta.url),
+		onLoading: loading => console.log("transcriber loading", loading),
+	})
+
+	const omni = new Omni(driver)
+	const {videoA} = await omni.load({videoA: Datafile.make(file)})
+
+	const transcript = await transcriber.transcribe({
+		source: file,
+		language: "english",
+		onReport: report => console.log("transcriber report", report),
+		onTranscription: text => console.log("transcribing", text),
+	})
+
+	const timeline = omni.timeline(o => {
+		const text = o.text("content", {duration: 3000})
+		const fade = o.animate.opacity.make("easeIn", [
+			[0, 0],
+			[700, 1],
+			[2200, 1],
+			[3000, 0.35],
+		])
+		const style = o.textStyle({fill: "green", fontSize: 100})
+		const videoSpatial = o.spatial(
+			o.transform({
+				position: [240, 160],
+				scale: [1.4, 1.4],
+				rotation: 0.08,
+			}),
+			[0.15, 0.1, 0.05, 0.2],
+		)
+		const textSpatial = o.spatial()
+		const textMotion = o.animate.transform.make("linear", [
+				[0, o.transform({position: [-320, 80], scale: [0.7, 0.7]})],
+				[1000, o.transform({position: [120, 80], scale: [1, 1]})],
+				[2000, o.transform({position: [200, 40], scale: [1.35, 1.35], rotation: 8})],
+				[3000, o.transform({position: [320, 0], scale: [1.15, 1.15], rotation: 0})],
+		])
+
+		const video = o.video(videoA, {duration: 6000, start: 3000})
+		o.set<Item.Text>(text.id, {styleId: style.id, spatialId: textSpatial.id, animationIds: [fade.id, textMotion.id]})
+		o.set<Item.Video>(video.id, {spatialId: videoSpatial.id})
+
+		const visual = o.sequence(
+			o.stack(
+				text,
+				o.captions(video, transcript, {
+					maxChars: 34,
+					maxDuration: 2800,
+					maxSilence: 450,
+					styles: {
+						fontFamily: "Arial",
+						fontSize: 64,
+						fill: "#ffffff",
+						align: "center",
+						wordWrap: true,
+						wordWrapWidth: 1280,
+						stroke: {color: "#000000", width: 6},
+					},
+				})
+			),
+			o.transition.circle(1500),
+			o.video(videoA, {duration: 7000, start: 5000})
+		)
+
+		return o.stack(
+			visual,
+			o.audio(videoA, {start: 3000, duration: video.duration})
+		)
+	})
+
+	return {timeline, omni}
+}

package/s/demo/routines/transcode-test.ts

@@ -1,3 +1,4 @@
+
 import {Driver} from "../../driver/driver.js"
 import {DecoderSource} from "../../driver/fns/schematic.js"
 
@@ -10,19 +11,23 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 	const ctx = canvas.getContext("2d")
 
 	async function run() {
-		const readables = driver.decode({
+		const video = driver.decodeVideo({
 			source,
 			async onFrame(frame) {
 				const composed = await driver.composite([
 					{
+						id: 0,
 						kind: "image",
 						frame
 					},
 					{
+						id: 1,
 						kind: "text",
 						content: "omnitool",
-						fontSize: 50,
-						color: "green"
+						style: {
+							fontSize: 50,
+							fill: "green"
+						}
 					}
 				])
 				frame.close()
@@ -30,15 +35,26 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 				return composed
 			}
 		})
+		const audio = driver.decodeAudio({source})
 
-		await driver.encode({
-			readables,
+		const {readable, done} = driver.encode({
+			video: video.readable,
+			audio: audio.readable,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}
 			}
 		})
+
+		const handle = await window.showSaveFilePicker()
+		const writable = await handle.createWritable()
+
+		await Promise.all([
+			readable.pipeTo(writable),
+			done
+		])
 	}
 
 	return {canvas, run}
 }
+

package/s/demo/routines/transcriber-test.ts

@@ -0,0 +1,34 @@
+import {Driver} from "../../driver/driver.js"
+import {makeTranscriber} from "../../features/speech/transcribe/transcriber.js"
+
+export async function transcriberTest(driver: Driver) {
+	const transcriber = await makeTranscriber({
+		driver,
+		spec: {
+			model: "onnx-community/whisper-tiny_timestamped",
+			device: "webgpu",
+			strideLength: 5,
+			chunkLength: 30,
+			dtype: "fp32"
+		},
+		workerUrl: new URL("/features/speech/transcribe/worker.bundle.min.js", import.meta.url),
+		onLoading({progress, total}) {
+	  	console.log(progress, total, "total")
+		},
+	})
+	return {
+		transcribe: async (file: File) => {
+			const result = await transcriber.transcribe({
+				source: file,
+				language: "english",
+				onReport(report) {
+	  			console.log("report", report)
+				},
+				onTranscription(transcription) {
+	  			console.log("transcript", transcription)
+				}
+			})
+			console.log(result, "transcript result")
+		}
+	}
+}

package/s/demo/routines/transitions-test.ts

@@ -0,0 +1,43 @@
+import {Application, Sprite} from "pixi.js"
+
+import {Driver} from "../../driver/driver.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export async function setupTransitionsTest(driver: Driver, source: DecoderSource) {
+	const app = new Application()
+	await app.init({width: 300, height: 300, preference: "webgl"})
+	const sprite = new Sprite({width: 300, height: 300})
+
+	app.stage.addChild(sprite)
+
+	document.body.appendChild(app.canvas)
+	const transition = makeTransition({name: "circle", renderer: app.renderer})
+
+	async function run() {
+		const video = driver.decodeVideo({
+			source,
+			async onFrame(frame) {
+				const texture = transition.render({
+					from: frame,
+					to: frame,
+					progress: 0.7,
+					width: app.canvas.width,
+					height: app.canvas.height
+				})
+				sprite.texture = texture
+				return frame
+			}
+		})
+
+		driver.encode({
+			video: video.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000}
+			}
+		})
+	}
+
+	return {run}
+}

package/s/demo/routines/waveform-test.ts

@@ -1,12 +1,39 @@
-import {Waveform} from "../../timeline/parts/waveform.js"
 
-export async function waveformTest() {
-	const container = document.querySelector(".waveform-demo") as HTMLElement
-	const widthSlider = document.querySelector(".width") as HTMLInputElement
-	const waveform = await Waveform.init("/assets/temp/gl.mp4", container)
+import {Driver} from "../../driver/driver.js"
+import {Waveform} from "../../timeline/parts/waveform/waveform.js"
 
-	widthSlider.addEventListener("change", () => {
-		const width = +widthSlider.value
-		waveform.width = width
+export async function waveformTest(driver: Driver, source: File, root: HTMLElement) {
+	const container = root.querySelector(".waveform-canvas") as HTMLDivElement
+	const widthSlider = root.querySelector(".width") as HTMLInputElement
+
+	container.replaceChildren()
+	container.style.position = "relative"
+	container.style.height = "96px"
+	container.style.overflow = "hidden"
+
+	const waveform = await Waveform.init(driver, source, {
+		tileHeight: 96,
+		onChange: () => renderTiles()
 	})
+
+	const renderTiles = () => {
+		const width = +widthSlider.value
+		const zoom = pixelsPerSecond(width, waveform.duration)
+		waveform.zoom = zoom
+		container.style.width = `${width}px`
+		container.replaceChildren(...[...waveform.getTiles().values()].map(tile => {
+			tile.canvas.style.position = "absolute"
+			tile.canvas.style.top = "0"
+			tile.canvas.style.left = `${tile.startTime * pixelsPerSecond(width, waveform.duration)}px`
+			tile.canvas.style.height = "100%"
+			return tile.canvas
+		}))
+	}
+
+	widthSlider.oninput = renderTiles
+	waveform.range = [0, waveform.duration]
+}
+
+function pixelsPerSecond(width: number, duration: number) {
+	return duration > 0 ? width / duration : 0
 }

package/s/driver/driver-worker.ts

@@ -0,0 +1,9 @@
+
+import {Comrade} from "@e280/comrade"
+import {setupDriverWork} from "./fns/work.js"
+import {DriverSchematic} from "./fns/schematic.js"
+
+export async function driverWorker() {
+	await Comrade.worker<DriverSchematic>(setupDriverWork)
+}
+

package/s/driver/driver.test.ts

@@ -5,7 +5,7 @@ import {Science, test, expect} from "@e280/science"
 const workerUrl = new URL("./driver.worker.bundle.js", import.meta.url)
 
 export default Science.suite({
-	"driver hello world": test(async() => {
+	"driver hello world": test.skip(async() => {
 		const driver = await Driver.setup({workerUrl})
 		expect(driver.machina.count).is(0)
 		await driver.thread.work.hello()

package/s/driver/driver.ts

@@ -1,7 +1,10 @@
+
+import {is} from "@e280/stz"
 import {Comrade, tune, Thread} from "@e280/comrade"
 import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
 
 import {Machina} from "./parts/machina.js"
+import {Compositor} from "./parts/compositor.js"
 import {setupDriverHost} from "./fns/host.js"
 import {loadDecoderSource} from "./utils/load-decoder-source.js"
 import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
@@ -11,19 +14,21 @@ export type DriverOptions = {
 }
 
 export class Driver {
-	static async setup(options: DriverOptions) {
+	static async setup(options?: DriverOptions) {
 		const machina = new Machina()
 		const thread = await Comrade.thread<DriverSchematic>({
 			label: "OmnitoolDriver",
-			workerUrl: options.workerUrl,
+			workerUrl: options?.workerUrl ?? "/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js",
 			setupHost: setupDriverHost(machina),
 		})
-		return new this(machina, thread)
+		const compositor = await Compositor.setup()
+		return new this(machina, thread, compositor)
 	}
 
 	constructor(
 		public machina: Machina,
-		public thread: Thread<DriverSchematic>
+		public thread: Thread<DriverSchematic>,
+		public compositor: Compositor
 	) {}
 
 	async hello() {
@@ -37,7 +42,8 @@ export class Driver {
 		})
 
 		const audioTrack = await input.getPrimaryAudioTrack()
-		return await audioTrack?.computeDuration()
+		if (!audioTrack) throw new Error("primary audio track not found")
+		return await audioTrack.computeDuration()
 	}
 
 	async getVideoDuration(source: DecoderSource) {
@@ -50,67 +56,70 @@ export class Driver {
 		return await videoTrack?.computeDuration()
 	}
 
-	decode(input: DecoderInput) {
+	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
+		const {port1, port2} = new MessageChannel()
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
 				const frame = await input.onFrame?.(chunk) ?? chunk
-				// below code is to prevent mem leaks and hardware accelerated decoder stall
 				lastFrame?.close()
 				controller.enqueue(frame)
 				lastFrame = frame
 			}
 		})
-		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			video: videoTransform.writable,
-			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
 		return {
-			audio: audioTransform.readable,
-			video: videoTransform.readable
+			readable: videoTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
 		}
 	}
 
-	async encode({readables, config}: EncoderInput) {
-		const handle = await window.showSaveFilePicker()
-		const writable = await handle.createWritable()
-		// making bridge because file picker writable is not transferable
-		const bridge = new WritableStream<StreamTargetChunk>({
-			async write(chunk) {
-				await writable.write(chunk)
-			},
-			async close() {
-				await writable.close()
-			}
+	decodeAudio(input: DecoderInput) {
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		const {port1, port2} = new MessageChannel()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable, port2]})({
+			source: input.source,
+			cancel: port2,
+			audio: audioTransform.writable,
+			start: input.start,
+			end: input.end
 		})
-		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+		return {
+			readable: audioTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
+	}
+
+	encode({audio, video, config}: EncoderInput) {
+		const {readable, writable} = new TransformStream<StreamTargetChunk, StreamTargetChunk>()
+		const transfer = [audio, video, writable].filter(is.happy)
+		const done = this.thread.work.encode[tune]({transfer})({audio, video, config, writable})
+		return {readable, done}
 	}
 
 	async composite(
 		composition: Composition,
 	) {
-		const transfer = this.#collectTransferablesFromComposition(composition)
-		return await this.thread.work.composite[tune]({transfer})(composition)
+		return await this.compositor.composite(composition)
 	}
 
-	#collectTransferablesFromComposition(composition: Composition) {
-		const transferables: Transferable[] = []
-
-		const visit = (node: Composition) => {
-			if (Array.isArray(node)) {
-				for (const child of node)
-					visit(child)
-			}
-			else if (node && typeof node === 'object' && 'kind' in node) {
-				if (node.kind === 'image' && node.frame instanceof VideoFrame)
-					transferables.push(node.frame)
-			}
-		}
-
-		visit(composition)
-		return transferables
-	}
 }
 

package/s/driver/fns/host.ts

@@ -3,10 +3,11 @@ import {Comrade} from "@e280/comrade"
 import {Machina} from "../parts/machina.js"
 import {DriverSchematic} from "./schematic.js"
 
-export const setupDriverHost = (machina: Machina) => Comrade.host<DriverSchematic>(({work}, rig) => ({
-
-	async world() {
-		machina.count++
-	}
-}))
+export const setupDriverHost = (machina: Machina) => (
+	Comrade.host<DriverSchematic>(_shell => ({
+		async world() {
+			machina.count++
+		},
+	}))
+)
 

package/s/driver/fns/schematic.ts

@@ -1,22 +1,37 @@
 
+import {TextStyleOptions} from "pixi.js"
 import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
+import {Id} from "../../timeline/index.js"
+import {Crop} from "../../timeline/parts/item.js"
+import {Mat6} from "../../timeline/utils/matrix.js"
+import {FilterParams, FilterType} from "../../timeline/parts/filters.js"
+import {TransitionName} from "../../timeline/parts/transitions.js"
+
 export type DriverSchematic = AsSchematic<{
 
 	// happens on the web worker
 	work: {
 		hello(): Promise<void>
 
-		decode(input: {
+		decodeAudio(input: {
 			source: DecoderSource
-			video: WritableStream<VideoFrame>
+			cancel: MessagePort
 			audio: WritableStream<AudioData>
+			start?: number
+			end?: number
 		}): Promise<void>
 
-		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
+		decodeVideo(input: {
+			source: DecoderSource
+			cancel: MessagePort
+			video: WritableStream<VideoFrame>
+			start?: number
+			end?: number
+		}): Promise<void>
 
-		composite(input: Composition): Promise<VideoFrame>
+		encode(input: EncoderInput & {writable: WritableStream<StreamTargetChunk>}): Promise<void>
 	}
 
 	// happens on the main thread
@@ -26,20 +41,22 @@ export type DriverSchematic = AsSchematic<{
 }>
 
 export interface EncoderInput {
-	readables: {
-		video: ReadableStream<VideoFrame>
-		audio: ReadableStream<AudioData>
-	},
-	config: {
-		video: VideoEncodingConfig
-		audio: AudioEncodingConfig
-	}
+	video?: ReadableStream<VideoFrame>
+	audio?: ReadableStream<AudioData>
+	config: RenderConfig
+}
+
+export interface RenderConfig {
+	video: VideoEncodingConfig
+	audio: AudioEncodingConfig
 }
 
-export type DecoderSource = FileSystemFileHandle | string
+export type DecoderSource = Blob | string | URL
 
 export interface DecoderInput {
 	source: DecoderSource
+	start?: number
+	end?: number
 	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
 }
 
@@ -59,25 +76,52 @@ export interface MuxOpts {
 
 export type Composition = Layer | (Layer | Composition)[]
 
-export type Transform = {
-	x?: number
-	y?: number
-	scale?: number
-	opacity?: number
-	anchor?: number
-}
+export type FilterSpec = {
+	[TFilter in FilterType]: {
+		type: TFilter
+		params?: FilterParams<TFilter>
+	}
+}[FilterType]
 
 export type TextLayer = {
+	id: Id
 	kind: 'text'
 	content: string
-	fontSize?: number
-	color?: string
-} & Transform
+	style?: TextStyleOptions
+	matrix?: Mat6
+	alpha?: number
+	crop?: Crop
+	filters?: FilterSpec[]
+}
 
 export type ImageLayer = {
+	id: Id
 	kind: 'image'
 	frame: VideoFrame
-} & Transform
+	matrix?: Mat6
+	alpha?: number
+	crop?: Crop
+	filters?: FilterSpec[]
+}
 
-export type Layer = TextLayer | ImageLayer
+export type TransitionLayer = {
+	id: Id
+  kind: 'transition'
+  name: TransitionName
+  progress: number
+  from: VideoFrame
+  to: VideoFrame
+}
+
+export type GapLayer = {
+	id: Id
+	kind: 'gap'
+}
+
+export type Audio = {
+	id: Id
+	kind: "audio"
+	data: AudioData
+}
 
+export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer