@omnimedia/omnitool 1.1.0-43 → 1.1.0-44

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-43",
+	"version": "1.1.0-44",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",
@@ -23,29 +23,28 @@
 		"test-debug": "node inspect x/tests.test.js"
 	},
 	"devDependencies": {
-		"@e280/science": "^0.0.6",
-		"@e280/scute": "^0.0.0-7",
-		"@types/node": "^24.2.0",
+		"@e280/science": "^0.1.4",
+		"@e280/scute": "^0.1.2",
+		"@types/node": "^25.0.3",
 		"http-server": "^14.1.1",
 		"npm-run-all": "^4.1.5",
-		"typescript": "^5.9.2"
+		"typescript": "^5.9.3"
 	},
 	"dependencies": {
-		"@e280/comrade": "^0.0.0-23",
-		"@e280/renraku": "^0.5.0-29",
-		"@e280/sly": "^0.2.0-32",
-		"@e280/strata": "^0.2.1",
-		"@e280/stz": "^0.2.13",
-		"@huggingface/transformers": "^3.7.1",
+		"@e280/comrade": "^0.1.0",
+		"@e280/renraku": "^0.5.3",
+		"@e280/sly": "^0.2.5",
+		"@e280/strata": "^0.2.5",
+		"@e280/stz": "^0.2.15",
+		"@huggingface/transformers": "^3.8.1",
 		"comrade": "^0.0.3",
 		"gl-transitions": "^1.43.0",
-		"gsap": "^3.13.0",
-		"lit": "^3.3.1",
-		"mediabunny": "^1.14.3",
-		"mp4-muxer": "^5.2.1",
-		"pixi.js": "^8.10.1",
-		"wavesurfer.js": "^7.10.0",
-		"web-demuxer": "^2.3.8"
+		"gsap": "^3.14.2",
+		"lit": "^3.3.2",
+		"mediabunny": "^1.27.3",
+		"mp4-muxer": "^5.2.2",
+		"pixi.js": "^8.14.3",
+		"wavesurfer.js": "^7.12.1"
 	},
 	"homepage": "https://github.com/omni-media/omnitool#readme",
 	"repository": {

package/s/demo/demo.bundle.ts

@@ -1,48 +1,20 @@
 
 import {Driver} from "../driver/driver.js"
+import {exportTest} from "./routines/export-test.js"
+import {playbackTest} from "./routines/playback-test.js"
 import {waveformTest} from "./routines/waveform-test.js"
+import {TimelineSchemaTest } from "./routines/timeline-setup.js"
 import {filmstripTest} from "./routines/filmstrip-test.js"
-import {transcriberTest} from "./routines/transcriber-test.js"
 import {setupTranscodeTest} from "./routines/transcode-test.js"
-import {Datafile, Omni, VideoPlayer} from "../timeline/index.js"
 
 const driver = await Driver.setup({workerUrl: new URL("../driver/driver.worker.bundle.min.js", import.meta.url)})
 const results = document.querySelector(".results")!
 
 const fetchButton = document.querySelector(".fetch")
-const importButton = document.querySelector(".import") as HTMLButtonElement
-
-const playButton = document.querySelector(".play") as HTMLButtonElement
-const stopButton = document.querySelector(".stop") as HTMLButtonElement
-const seekButton = document.querySelector(".seek") as HTMLButtonElement
+const fileInput = document.querySelector(".file-input") as HTMLInputElement
 
 fetchButton?.addEventListener("click", startDemoFetch)
-importButton?.addEventListener("click", startDemoImport)
-
-const omni = new Omni(driver)
-const file = await fetch("/assets/temp/gl.mp4")
-const buffer = await file.arrayBuffer()
-const uint = new Uint8Array(buffer)
-
-const {videoA} = await omni.load({videoA: Datafile.make(uint)})
-const timeline =	omni.timeline(o =>
-	o.sequence(
-	o.stack(
-		o.video(videoA, {duration: 5000}),
-		o.audio(videoA, {duration: 8000})
-	),
-	o.video(videoA, {duration: 7000})
-))
-
-const player = await VideoPlayer.create(driver, timeline)
-document.body.appendChild(player.canvas)
-
-playButton.addEventListener("click", () => player.play())
-stopButton.addEventListener("click", () => player.pause())
-seekButton.addEventListener("change", async (e: Event) => {
-	const target = e.target as HTMLInputElement
-	await player.seek(+target.value)
-})
+fileInput?.addEventListener("input", startDemoImport)
 
 waveformTest(driver)
 // const transcriber = await transcriberTest(driver)
@@ -55,14 +27,22 @@ waveformTest(driver)
 }
 
 // transcoding tests
-async function startDemoImport()
+async function startDemoImport(e: Event)
 {
-	const [fileHandle] = await window.showOpenFilePicker()
-	const file = await fileHandle.getFile()
-	const transcode = setupTranscodeTest(driver, file)
-	await filmstripTest(file)
-	run(transcode, fileHandle.name)
-	// await transcriber.transcribe(file)
+	const file = fileInput.files?.[0]
+	if(file) {
+		const transcode = setupTranscodeTest(driver, file)
+		await filmstripTest(file)
+		run(transcode, file.name)
+		// await transcriber.transcribe(file)
+
+		const {timeline, omni} = await TimelineSchemaTest(driver, file)
+
+		playbackTest(driver, timeline, omni)
+		exportTest(omni, timeline)
+	}
+	// const [fileHandle] = await window.showOpenFilePicker()
+	// const file = await fileHandle.getFile()
 }
 
 async function startDemoFetch()

package/s/demo/routines/export-test.ts

@@ -0,0 +1,8 @@
+import {Omni, TimelineFile} from "../../timeline/index.js"
+
+export function exportTest(omni: Omni, timeline: TimelineFile) {
+	const exportButton = document.querySelector(".export") as HTMLButtonElement
+	exportButton!.addEventListener("click", () => {
+		omni.render(timeline).then(() => console.log("done"))
+	})
+}

package/s/demo/routines/playback-test.ts

@@ -0,0 +1,21 @@
+
+import {Driver} from "../../driver/driver.js"
+import {O, Omni, TimelineFile, VideoPlayer} from "../../timeline/index.js"
+
+export async function playbackTest(driver: Driver, timeline: TimelineFile, omni: Omni) {
+	const playButton = document.querySelector(".play") as HTMLButtonElement
+	const stopButton = document.querySelector(".stop") as HTMLButtonElement
+	const seekButton = document.querySelector(".seek") as HTMLButtonElement
+	const o = new O({project: timeline})
+	const player = await VideoPlayer.create(driver, timeline, (hash) => omni.resources.require(hash).url)
+	document.body.appendChild(player.canvas)
+
+	playButton.addEventListener("click", () => player.play())
+	stopButton.addEventListener("click", () => player.pause())
+	seekButton.addEventListener("change", async (e: Event) => {
+		const target = e.target as HTMLInputElement
+		await player.seek(+target.value)
+	})
+
+	player.update(o.state.project)
+}

package/s/demo/routines/timeline-setup.ts

@@ -0,0 +1,24 @@
+
+import {Driver} from "../../driver/driver.js"
+import {Datafile, Item, Omni} from "../../timeline/index.js"
+
+export async function TimelineSchemaTest(driver: Driver, file: File) {
+	const omni = new Omni(driver)
+	const {videoA} = await omni.load({videoA: Datafile.make(file)})
+	const timeline = omni.timeline(o => {
+	const text = o.text("content", {duration: 1000})
+	const style = o.textStyle({fill: "green", fontSize: 100})
+	o.set<Item.Text>(text.id, {styleId: style.id})
+
+	return o.sequence(
+		o.stack(
+			text,
+			o.video(videoA, {duration: 3000, start: 3000}),
+			o.audio(videoA, {duration: 1000, start: 3000})
+		),
+		o.gap(500),
+		o.video(videoA, {duration: 7000, start: 5000})
+	)})
+
+	return {timeline, omni}
+}

package/s/demo/routines/transcode-test.ts

@@ -15,10 +15,12 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 			async onFrame(frame) {
 				const composed = await driver.composite([
 					{
+						id: 0,
 						kind: "image",
 						frame
 					},
 					{
+						id: 1,
 						kind: "text",
 						content: "omnitool",
 						style: {

package/s/driver/driver-worker.ts

@@ -0,0 +1,9 @@
+
+import {Comrade} from "@e280/comrade"
+import {setupDriverWork} from "./fns/work.js"
+import {DriverSchematic} from "./fns/schematic.js"
+
+export async function driverWorker() {
+	await Comrade.worker<DriverSchematic>(setupDriverWork)
+}
+

package/s/driver/driver.test.ts

@@ -5,7 +5,7 @@ import {Science, test, expect} from "@e280/science"
 const workerUrl = new URL("./driver.worker.bundle.js", import.meta.url)
 
 export default Science.suite({
-	"driver hello world": test(async() => {
+	"driver hello world": test.skip(async() => {
 		const driver = await Driver.setup({workerUrl})
 		expect(driver.machina.count).is(0)
 		await driver.thread.work.hello()

package/s/driver/driver.ts

@@ -2,6 +2,7 @@ import {Comrade, tune, Thread} from "@e280/comrade"
 import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
 
 import {Machina} from "./parts/machina.js"
+import {Compositor} from "./parts/compositor.js"
 import {setupDriverHost} from "./fns/host.js"
 import {loadDecoderSource} from "./utils/load-decoder-source.js"
 import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
@@ -18,12 +19,14 @@ export class Driver {
 			workerUrl: options?.workerUrl ?? "/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js",
 			setupHost: setupDriverHost(machina),
 		})
-		return new this(machina, thread)
+		const compositor = await Compositor.setup()
+		return new this(machina, thread, compositor)
 	}
 
 	constructor(
 		public machina: Machina,
-		public thread: Thread<DriverSchematic>
+		public thread: Thread<DriverSchematic>,
+		public compositor: Compositor
 	) {}
 
 	async hello() {
@@ -37,7 +40,8 @@ export class Driver {
 		})
 
 		const audioTrack = await input.getPrimaryAudioTrack()
-		return await audioTrack?.computeDuration()
+		if (!audioTrack) throw new Error("primary audio track not found")
+		return await audioTrack.computeDuration()
 	}
 
 	async getVideoDuration(source: DecoderSource) {
@@ -99,26 +103,8 @@ export class Driver {
 	async composite(
 		composition: Composition,
 	) {
-		const transfer = this.#collectTransferablesFromComposition(composition)
-		return await this.thread.work.composite[tune]({transfer})(composition)
+		return await this.compositor.composite(composition)
 	}
 
-	#collectTransferablesFromComposition(composition: Composition) {
-		const transferables: Transferable[] = []
-
-		const visit = (node: Composition) => {
-			if (Array.isArray(node)) {
-				for (const child of node)
-					visit(child)
-			}
-			else if (node && typeof node === 'object' && 'kind' in node) {
-				if (node.kind === 'image' && node.frame instanceof VideoFrame)
-					transferables.push(node.frame)
-			}
-		}
-
-		visit(composition)
-		return transferables
-	}
 }
 

package/s/driver/fns/schematic.ts

@@ -4,6 +4,7 @@ import {AsSchematic} from "@e280/comrade"
 import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
 
 import {Mat6} from "../../timeline/utils/matrix.js"
+import {Id} from "../../timeline/index.js"
 
 export type DriverSchematic = AsSchematic<{
 
@@ -26,8 +27,6 @@ export type DriverSchematic = AsSchematic<{
 		}): Promise<void>
 
 		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
-
-		composite(input: Composition): Promise<VideoFrame>
 	}
 
 	// happens on the main thread
@@ -73,6 +72,7 @@ export interface MuxOpts {
 export type Composition = Layer | (Layer | Composition)[]
 
 export type TextLayer = {
+	id: Id
 	kind: 'text'
 	content: string
 	style?: TextStyleOptions
@@ -80,12 +80,14 @@ export type TextLayer = {
 }
 
 export type ImageLayer = {
+	id: Id
 	kind: 'image'
 	frame: VideoFrame
 	matrix?: Mat6
 }
 
 export type TransitionLayer = {
+	id: Id
   kind: 'transition'
   name: string
   progress: number
@@ -94,10 +96,12 @@ export type TransitionLayer = {
 }
 
 export type GapLayer = {
+	id: Id
 	kind: 'gap'
 }
 
 export type Audio = {
+	id: Id
 	kind: "audio"
 	data: AudioData
 }

package/s/driver/fns/work.ts

@@ -1,10 +1,8 @@
 import {Comrade} from "@e280/comrade"
-import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter, Matrix} from "pixi.js"
+import {Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
 import {Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample, AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource} from "mediabunny"
 
-import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
-import {makeTransition} from "../../features/transition/transition.js"
-import {Composition, DecoderSource, DriverSchematic, Layer} from "./schematic.js"
+import {DecoderSource, DriverSchematic} from "./schematic.js"
 
 DOMAdapter.set(WebWorkerAdapter)
 
@@ -111,136 +109,7 @@ export const setupDriverWork = (
 			await Promise.all(promises)
 			await output.finalize()
 		},
-
-		async composite(composition) {
-			const {stage, renderer} = await renderPIXI(1920, 1080)
-			stage.removeChildren()
-
-			const {dispose} = await renderLayer(composition, stage)
-			renderer.render(stage)
-
-			// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
-			// if its very slow on eg edge try chrome
-			const frame = new VideoFrame(renderer.canvas, {
-				timestamp: 0,
-				duration: 0,
-			})
-
-			renderer.clear()
-			dispose()
-
-			shell.transfer = [frame]
-			return frame
-		}
 	}))
 )
 
-// TODO suspicious global, probably bad
-let pixi: {
-	renderer: Renderer
-	stage: Container
-} | null = null
-
-async function renderPIXI(width: number, height: number) {
-	if (pixi)
-		return pixi
-
-	const renderer = await autoDetectRenderer({
-		width,
-		height,
-		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
-		background: "black",
-		preferWebGLVersion: 2
-	})
-
-	const stage = new Container()
-	pixi = {renderer, stage}
-
-	return pixi
-}
-
-const transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
-
 type RenderableObject = Sprite | Text | Texture
-
-async function renderLayer(
-	layer: Layer | Composition,
-	parent: Container,
-) {
-	if (Array.isArray(layer)) {
-		layer.reverse()
-		const disposers: (() => void)[] = []
-		for (const child of layer) {
-			const result = await renderLayer(child, parent)
-			disposers.push(result.dispose)
-		}
-		return {dispose: () => disposers.forEach(d => d())}
-	}
-
-	switch (layer.kind) {
-		case 'text':
-			return renderTextLayer(layer, parent)
-		case 'image':
-			return renderImageLayer(layer, parent)
-		case 'transition':
-			return renderTransitionLayer(layer, parent)
-		case 'gap': {
-			pixi?.renderer.clear()
-			return {dispose: () => {}}
-		}
-		default:
-			console.warn('Unknown layer kind', (layer as any).kind)
-			return {dispose: () => {}}
-	}
-}
-
-function renderTextLayer(
-	layer: Extract<Layer, {kind: 'text'}>,
-	parent: Container,
-) {
-	const text = new Text({
-		text: layer.content,
-		style: layer.style
-	})
-	applyTransform(text, layer.matrix)
-	parent.addChild(text)
-	return {dispose: () => text.destroy(true)}
-}
-
-function renderImageLayer(
-	layer: Extract<Layer, {kind: 'image'}>,
-	parent: Container,
-) {
-	const texture = Texture.from(layer.frame)
-	const sprite = new Sprite(texture)
-	applyTransform(sprite, layer.matrix)
-	parent.addChild(sprite)
-	return {dispose: () => {
-		sprite.destroy(true)
-		texture.destroy(true)
-		layer.frame.close()
-	}}
-}
-
-function renderTransitionLayer(
-	{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
-	parent: Container,
-) {
-	const transition = transitions.get(name) ??
-		(transitions.set(name, makeTransition({
-			name: "circle",
-			renderer: pixi!.renderer
-		})),
-	  transitions.get(name)!
-	)
-	const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
-	const sprite = new Sprite(texture)
-	parent.addChild(sprite)
-	return {dispose: () => sprite.destroy(false)}
-}
-
-function applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
-  if (!worldMatrix) return
-	const mx = mat6ToMatrix(worldMatrix)
-  target.setFromMatrix(mx)
-}

package/s/driver/parts/compositor.ts

@@ -0,0 +1,178 @@
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture} from "pixi.js"
+
+import {Composition, Layer} from "../fns/schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export class Compositor {
+
+	static async setup() {
+		const renderer = await autoDetectRenderer({
+			width: 1920,
+			height: 1080,
+			preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+			background: "black",
+			preferWebGLVersion: 2
+		})
+		const stage = new Container()
+		stage.interactive = true
+		return new this({renderer, stage})
+	}
+
+	constructor(public pixi: {renderer: Renderer, stage: Container}) {}
+
+	#transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+	#objects = new Map<number, Container>()
+
+	async composite(
+		composition: Composition,
+	) {
+		const {stage, renderer} = this.pixi
+
+		this.#cleanup(this.#collectIds(composition))
+		const {dispose} = await this.#renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: 0,
+			duration: 0,
+		})
+
+		dispose()
+
+		return frame
+	}
+
+	async #renderLayer(
+		layer: Layer | Composition,
+		parent: Container,
+	) {
+		if (Array.isArray(layer)) {
+			layer.reverse()
+			const disposers: (() => void)[] = []
+			for (const child of layer) {
+				const result = await this.#renderLayer(child, parent)
+				disposers.push(result.dispose)
+			}
+			return {dispose: () => disposers.forEach(d => d())}
+		}
+
+		switch (layer.kind) {
+			case 'text':
+				return this.#renderTextLayer(layer, parent)
+			case 'image':
+				return this.#renderImageLayer(layer, parent)
+			case 'transition':
+				return this.#renderTransitionLayer(layer, parent)
+			case 'gap': {
+				this.pixi?.renderer.clear()
+				return {dispose: () => {}}
+			}
+			default:
+				console.warn('Unknown layer kind', (layer as any).kind)
+				return {dispose: () => {}}
+		}
+	}
+
+	#renderTextLayer(
+		layer: Extract<Layer, {kind: 'text'}>,
+		parent: Container,
+	) {
+		const text = this.#findOrCreate<Text>(layer)!
+		this.#applyTransform(text, layer.matrix)
+		parent.addChild(text)
+		return {
+			dispose: () => {}
+		}
+	}
+
+	#renderImageLayer(
+		layer: Extract<Layer, {kind: 'image'}>,
+		parent: Container,
+	) {
+		const texture = Texture.from(layer.frame)
+		const sprite = this.#findOrCreate<Sprite>(layer)!
+		sprite.texture = texture
+		this.#applyTransform(sprite, layer.matrix)
+		parent.addChild(sprite)
+		return {
+			dispose: () => {
+				texture.destroy(true)
+				layer.frame.close()
+			}
+		}
+	}
+
+	#renderTransitionLayer(
+		{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+		parent: Container,
+	) {
+		const transition = this.#transitions.get(name) ??
+			(this.#transitions.set(name, makeTransition({
+				name: "circle",
+				renderer: this.pixi.renderer
+			})),
+	  	this.#transitions.get(name)!
+		)
+		const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+		const sprite = new Sprite(texture)
+		parent.addChild(sprite)
+		return {dispose: () => sprite.destroy(false)}
+	}
+
+	#applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  	if (!worldMatrix) return
+		const mx = mat6ToMatrix(worldMatrix)
+  	target.setFromMatrix(mx)
+	}
+
+	#findOrCreate<T = Container>(layer: Layer) {
+		const object = this.#objects.get(layer.id)
+		if(!object) {
+			switch (layer.kind) {
+				case 'text': {
+					const text = new Text({
+						text: layer.content,
+						style: layer.style
+					})
+					text.onmouseenter = () => console.log("enter text")
+					return this.#objects
+						.set(layer.id, text)
+						.get(layer.id) as T
+				}
+				case 'image': {
+					const sprite = new Sprite()
+					sprite.onmouseenter = () => console.log("enter")
+					return this.#objects
+						.set(layer.id, sprite)
+						.get(layer.id) as T
+				}
+			}
+		} else return object as T
+	}
+
+	#collectIds(layers: Layer | Composition): Set<number> {
+		const result = new Set<number>()
+		const traverse = (node: Layer | Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node) traverse(child)
+			} else {
+				result.add(node.id)
+			}
+		}
+		traverse(layers)
+		return result
+	}
+
+	#cleanup(activeIds: Set<number>) {
+		for (const id of this.#objects.keys()) {
+			if (!activeIds.has(id)) {
+				const obj = this.#objects.get(id)!
+				obj.destroy(true)
+				this.#objects.delete(id)
+			}
+		}
+	}
+}