@omnimedia/omnitool 1.1.0-5 → 1.1.0-51

package/s/driver/parts/compositor.ts

@@ -0,0 +1,178 @@
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture} from "pixi.js"
+
+import {Composition, Layer} from "../fns/schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export class Compositor {
+
+	static async setup() {
+		const renderer = await autoDetectRenderer({
+			width: 1920,
+			height: 1080,
+			preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+			background: "black",
+			preferWebGLVersion: 2
+		})
+		const stage = new Container()
+		stage.interactive = true
+		return new this({renderer, stage})
+	}
+
+	constructor(public pixi: {renderer: Renderer, stage: Container}) {}
+
+	#transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+	#objects = new Map<number, Container>()
+
+	async composite(
+		composition: Composition,
+	) {
+		const {stage, renderer} = this.pixi
+
+		this.#cleanup(this.#collectIds(composition))
+		const {dispose} = await this.#renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: 0,
+			duration: 0,
+		})
+
+		dispose()
+
+		return frame
+	}
+
+	async #renderLayer(
+		layer: Layer | Composition,
+		parent: Container,
+	) {
+		if (Array.isArray(layer)) {
+			layer.reverse()
+			const disposers: (() => void)[] = []
+			for (const child of layer) {
+				const result = await this.#renderLayer(child, parent)
+				disposers.push(result.dispose)
+			}
+			return {dispose: () => disposers.forEach(d => d())}
+		}
+
+		switch (layer.kind) {
+			case 'text':
+				return this.#renderTextLayer(layer, parent)
+			case 'image':
+				return this.#renderImageLayer(layer, parent)
+			case 'transition':
+				return this.#renderTransitionLayer(layer, parent)
+			case 'gap': {
+				this.pixi?.renderer.clear()
+				return {dispose: () => {}}
+			}
+			default:
+				console.warn('Unknown layer kind', (layer as any).kind)
+				return {dispose: () => {}}
+		}
+	}
+
+	#renderTextLayer(
+		layer: Extract<Layer, {kind: 'text'}>,
+		parent: Container,
+	) {
+		const text = this.#findOrCreate<Text>(layer)!
+		this.#applyTransform(text, layer.matrix)
+		parent.addChild(text)
+		return {
+			dispose: () => {}
+		}
+	}
+
+	#renderImageLayer(
+		layer: Extract<Layer, {kind: 'image'}>,
+		parent: Container,
+	) {
+		const texture = Texture.from(layer.frame)
+		const sprite = this.#findOrCreate<Sprite>(layer)!
+		sprite.texture = texture
+		this.#applyTransform(sprite, layer.matrix)
+		parent.addChild(sprite)
+		return {
+			dispose: () => {
+				texture.destroy(true)
+				layer.frame.close()
+			}
+		}
+	}
+
+	#renderTransitionLayer(
+		{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+		parent: Container,
+	) {
+		const transition = this.#transitions.get(name) ??
+			(this.#transitions.set(name, makeTransition({
+				name: "circle",
+				renderer: this.pixi.renderer
+			})),
+	  	this.#transitions.get(name)!
+		)
+		const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+		const sprite = new Sprite(texture)
+		parent.addChild(sprite)
+		return {dispose: () => sprite.destroy(false)}
+	}
+
+	#applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  	if (!worldMatrix) return
+		const mx = mat6ToMatrix(worldMatrix)
+  	target.setFromMatrix(mx)
+	}
+
+	#findOrCreate<T = Container>(layer: Layer) {
+		const object = this.#objects.get(layer.id)
+		if(!object) {
+			switch (layer.kind) {
+				case 'text': {
+					const text = new Text({
+						text: layer.content,
+						style: layer.style
+					})
+					text.onmouseenter = () => console.log("enter text")
+					return this.#objects
+						.set(layer.id, text)
+						.get(layer.id) as T
+				}
+				case 'image': {
+					const sprite = new Sprite()
+					sprite.onmouseenter = () => console.log("enter")
+					return this.#objects
+						.set(layer.id, sprite)
+						.get(layer.id) as T
+				}
+			}
+		} else return object as T
+	}
+
+	#collectIds(layers: Layer | Composition): Set<number> {
+		const result = new Set<number>()
+		const traverse = (node: Layer | Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node) traverse(child)
+			} else {
+				result.add(node.id)
+			}
+		}
+		traverse(layers)
+		return result
+	}
+
+	#cleanup(activeIds: Set<number>) {
+		for (const id of this.#objects.keys()) {
+			if (!activeIds.has(id)) {
+				const obj = this.#objects.get(id)!
+				obj.destroy(true)
+				this.#objects.delete(id)
+			}
+		}
+	}
+}

package/s/driver/parts/machina.ts

@@ -1,27 +1,26 @@
-import {WebMediaInfo} from "web-demuxer"
 
-type Events =
-	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
-	| {type: "info", data: WebMediaInfo}
-	| {type: "encoderQueueSize", size: number}
-
-type Handler = (event: Events) => void
+// type Events =
+// 	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+// 	| {type: "info", data: WebMediaInfo}
+// 	| {type: "encoderQueueSize", size: number}
+//
+// type Handler = (event: Events) => void
 
 export class Machina {
 	count = 0
 
-	#handlers = new Map<number, Handler>()
-
-	register(id: number, handler: Handler) {
-		this.#handlers.set(id, handler)
-	}
-
-	unregister(id: number) {
-		this.#handlers.delete(id)
-	}
-
-	dispatch(id: number, event: Events) {
-		this.#handlers.get(id)?.(event)
-	}
+	// #handlers = new Map<number, Handler>()
+	//
+	// register(id: number, handler: Handler) {
+	// 	this.#handlers.set(id, handler)
+	// }
+	//
+	// unregister(id: number) {
+	// 	this.#handlers.delete(id)
+	// }
+	//
+	// dispatch(id: number, event: Events) {
+	// 	this.#handlers.get(id)?.(event)
+	// }
 }
 

package/s/index.html.ts

@@ -15,7 +15,7 @@ export default ssg.page(import.meta.url, async orb => ({
 	head: html`
 		<link rel="preconnect" href="https://fonts.googleapis.com">
 		<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&display=swap" rel="stylesheet">
+		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&family=Space+Grotesk:wght@400;600&display=swap" rel="stylesheet">
 	`,
 
 	socialCard: {
@@ -28,25 +28,109 @@ export default ssg.page(import.meta.url, async orb => ({
 
 	body: html`
 		<section>
-			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
-			<button class=fetch>fetch</button>
-			<button class="import">import</button>
-			<div class=results></div>
-			<div class=filmstrip-demo>
-				<label for="viewable-range">viewable range:</label>
-				<input type="range" min="0" max="100" step="1" value="10" class="range" id="viewable-range" name="viewable-range">
-				<div class="range-view"></div>
-				<label for="range-size">viewable range size:</label>
-				<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5" id="range-size" name="range-size">
-				<label for="frequency">frequency:</label>
-				<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10" id="frequency" name="frequency">
-				<div class="frequency-view">10 (fps)</div>
-				<div id=filmstrip></div>
-			</div>
-			<div class=waveform-demo>
-				<label for="width">width:</label>
-				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
+			<header class="hero">
+				<div>
+					<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
+					<p>Developer demos for decoding, preview, waveform, playback, and export.</p>
+				</div>
+			</header>
+
+			<div class="demo-grid">
+				<article class="demo-card" data-demo="transcode">
+					<header>
+						<h2>Transcode Test</h2>
+						<p>Full decode + composite + encode pipeline.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Run Transcode</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
+
+				<article class="demo-card" data-demo="filmstrip">
+					<header>
+						<h2>Filmstrip</h2>
+						<p>Thumbnail sampling across a time range.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Generate Filmstrip</button>
+					</div>
+					<div class="filmstrip-controls">
+						<label>viewable range</label>
+						<input type="range" min="0" max="100" step="1" value="10" class="range">
+						<div class="range-view"></div>
+						<label>range size</label>
+						<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5">
+						<label>frequency</label>
+						<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10">
+						<div class="frequency-view">10 (fps)</div>
+					</div>
+					<div class="filmstrip"></div>
+				</article>
+
+				<article class="demo-card" data-demo="waveform">
+					<header>
+						<h2>Waveform</h2>
+						<p>Audio peak rendering with adjustable width.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="audio/*,video/*" />
+						<button data-action="run">Build Waveform</button>
+					</div>
+					<div class="waveform-controls">
+						<label>width</label>
+						<input class="width" type="range" min="100" max="1000000" value="1000" />
+					</div>
+					<div class="waveform-canvas"></div>
+				</article>
+
+				<article class="demo-card" data-demo="playback">
+					<header>
+						<h2>Timeline Playback</h2>
+						<p>Build timeline and run the playback engine.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+					</div>
+					<div class="player-canvas"></div>
+					<div class="player">
+						<div class="timeline">
+							<div class="track">
+								<div class="playhead"></div>
+							</div>
+							<input class="scrub" type="range" min="0" max="0" step="1" value="0">
+						</div>
+						<div class="timecode">0:00.000 / 0:00.000</div>
+						<div class="controls">
+							<button class=play disabled>play</button>
+							<button class=stop disabled>stop</button>
+						</div>
+					</div>
+				</article>
+
+				<article class="demo-card" data-demo="export">
+					<header>
+						<h2>Timeline Export</h2>
+						<p>Build timeline and export a render.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+						<button data-action="export" disabled>Export</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
 			</div>
+
 		</section>
 	`,
 }))

package/s/index.ts

@@ -1,4 +1,5 @@
 
 export * from "./driver/driver.js"
+export * from "./driver/driver-worker.js"
 export * from "./timeline/index.js"
 

package/s/tests.bundle.ts

@@ -0,0 +1,11 @@
+
+import {Science} from "@e280/science"
+import omniTest from "./timeline/sugar/omni.test.js"
+import renderersTest from "./timeline/renderers/renderers.test.js"
+
+await Science.run
+({
+	omniTest,
+	renderersTest
+})
+

package/s/tests.html.ts

@@ -0,0 +1,28 @@
+
+import {ssg, html} from "@e280/scute"
+
+const title = "omnitool"
+const domain = "omnitool.omniclip.app"
+const favicon = "/assets/favicon.png"
+
+export default ssg.page(import.meta.url, async orb => ({
+	title,
+	dark: true,
+	css: "demo/demo.css",
+	js: "tests.bundle.min.js",
+
+	head: html`
+	`,
+
+	socialCard: {
+		title,
+		description: "video processing toolkit",
+		themeColor: "#3cff9c",
+		siteName: domain,
+		image: `https://${domain}${favicon}`,
+	},
+
+	body: html`
+	`,
+}))
+

package/s/timeline/index.ts

@@ -5,8 +5,11 @@ export * from "./parts/media.js"
 export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
+export * from "./parts/waveform.js"
+export * from "./renderers/player/player.js"
 
 export * from "./sugar/o.js"
+export * from "./sugar/helpers.js"
 export * from "./sugar/omni.js"
 
 export * from "./utils/checksum.js"

package/s/timeline/parts/basics.ts

@@ -11,7 +11,7 @@ export type TimelineFile = {
 	info: "https://omniclip.app/"
 	format: "timeline"
 	version: number
-	root: Id
+	rootId: Id
 	items: Item.Any[]
 }
 

package/s/timeline/parts/filmstrip.ts

@@ -17,7 +17,7 @@ export class Filmstrip {
 
 	private constructor(
 		private videoTrack: InputVideoTrack,
-		private options: Required<FilmstripOptions>
+		private options: FilmstripOptions
 	) {
 		this.#sink = new CanvasSink(videoTrack, options.canvasSinkOptions)
 	}
@@ -33,7 +33,8 @@ export class Filmstrip {
 				videoTrack, {
 					frequency: options.frequency ?? 1,
 					canvasSinkOptions: options.canvasSinkOptions ?? {width: 80, height: 50, fit: "fill"},
-					onChange: options.onChange
+					onChange: options.onChange,
+					onPlaceholders: options.onPlaceholders
 			})
 		else throw new Error("Source has no video track")
 	}
@@ -54,9 +55,28 @@ export class Filmstrip {
 		return this.options.frequency
 	}
 
-	#computeActiveRange([start, end]: TimeRange): TimeRange {
+	#computeActiveRange([start, end]: TimeRange, margin = 1): TimeRange {
 		const tileSize = end - start
-		return [start - tileSize, end + tileSize]
+		return [start - tileSize * margin, end + tileSize * margin]
+	}
+
+	async #generatePlaceholders() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededTimestamps = new Set<number>()
+
+		// duration should be computed but with trim etc also
+		const duration = await this.videoTrack.computeDuration()
+		for (
+			let timestamp = Math.max(0, rangeStart);
+			timestamp <= rangeEnd;
+			timestamp += this.options.frequency
+		) {
+			// Clamp to valid time range
+			if (timestamp >= 0 && timestamp <= duration)
+				neededTimestamps.add(timestamp)
+		}
+
+		this.options.onPlaceholders?.([...neededTimestamps])
 	}
 
 	async #generateTiles() {
@@ -66,13 +86,13 @@ export class Filmstrip {
 		// duration should be computed but with trim etc also
 		const duration = await this.videoTrack.computeDuration()
 		for (
-			let timestamp = rangeStart;
+			let timestamp = Math.max(0, rangeStart);
 			timestamp <= rangeEnd;
 			timestamp += this.options.frequency
 		) {
 			// Clamp to valid time range
 			if (timestamp >= 0 && timestamp <= duration)
-				neededTimestamps.add(+timestamp.toFixed(3))
+				neededTimestamps.add(timestamp)
 		}
 
 		const missingTimestamps = [...neededTimestamps]
@@ -106,15 +126,20 @@ export class Filmstrip {
  	* @param visibleRange - The current timeline viewport as a [start, end] tuple in seconds.
  	*/
 	set range(visibleRange: TimeRange) {
-		const newRange = this.#computeActiveRange(visibleRange)
-		// Avoid redundant updates
-		if (
-			this.#activeRange[0] === newRange[0] &&
-			this.#activeRange[1] === newRange[1]
-		)
-			return
+		const [visStart, visEnd] = visibleRange
+		const visibleSize = visEnd - visStart
+		const [actStart, actEnd] = this.#activeRange
+
+		// trigger when we're 1x visible width away from margin edges
+		const leftTrigger = actStart + visibleSize
+		const rightTrigger = actEnd - visibleSize
+
+		const nearLeftEdge = visStart < leftTrigger
+		const nearRightEdge = visEnd > rightTrigger
+
+		if (!nearLeftEdge && !nearRightEdge) return
 
-		this.#activeRange = newRange
+		this.#activeRange = this.#computeActiveRange(visibleRange, 2)
 		this.#update()
 	}
 
@@ -122,6 +147,7 @@ export class Filmstrip {
 	#shouldRunAgain = false
 
 	async #update() {
+		this.#generatePlaceholders()
 		// Perform update immediately. If multiple updates are requested while updating,
 		// only the latest one will run after the current finishes (skips intermediate ones).
 		if(this.#updating) {
@@ -147,11 +173,12 @@ export class Filmstrip {
 	}
 }
 
-type TimeRange = [number, number]
+export type TimeRange = [start: number, end: number]
 
 interface FilmstripOptions {
-	frequency?: number
+	frequency: number
 	canvasSinkOptions?: CanvasSinkOptions
+	onPlaceholders?: (timestamps: number[]) => void
 	onChange: (tiles: {
 		time: number
 		canvas: WrappedCanvas

package/s/timeline/parts/item.ts

@@ -1,12 +1,20 @@
 
+import {TextStyleOptions} from "pixi.js"
+
 import {Id, Hash} from "./basics.js"
+import {Transform} from "../types.js"
+import {Ms} from "../../units/ms.js"
 
 export enum Kind {
 	Sequence,
 	Stack,
-	Clip,
+	Video,
+	Audio,
 	Text,
+	Gap,
+	Spatial,
 	Transition,
+	TextStyle
 }
 
 export enum Effect {
@@ -14,30 +22,64 @@ export enum Effect {
 }
 
 export namespace Item {
+	export type TextStyle = {
+		id: Id
+		kind: Kind.TextStyle
+		style: TextStyleOptions
+	}
+
+	export type Spatial = {
+		id: Id
+		kind: Kind.Spatial
+		transform: Transform
+		enabled: boolean
+	}
+
+	export type Gap = {
+		id: Id
+		kind: Kind.Gap
+		duration: number
+	}
+
 	export type Sequence = {
 		id: Id
 		kind: Kind.Sequence
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
 	}
 
 	export type Stack = {
 		id: Id
 		kind: Kind.Stack
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
+	}
+
+	export type Video = {
+		id: Id
+		kind: Kind.Video
+		mediaHash: Hash
+		start: number
+		duration: number
+		spatialId?: Id
 	}
 
-	export type Clip = {
+	export type Audio = {
 		id: Id
-		kind: Kind.Clip
+		kind: Kind.Audio
 		mediaHash: Hash
 		start: number
 		duration: number
+		gain?: number
 	}
 
 	export type Text = {
 		id: Id
 		kind: Kind.Text
 		content: string
+		duration: number
+		spatialId?: Id
+		styleId?: Id
 	}
 
 	export type Transition = {
@@ -50,9 +92,20 @@ export namespace Item {
 	export type Any = (
 		| Sequence
 		| Stack
-		| Clip
+		| Video
+		| Audio
 		| Text
+		| Gap
 		| Transition
+		| Spatial
+		| TextStyle
 	)
 }
 
+export type ContainerItem = Item.Sequence | Item.Stack
+export type NonContainerItem = Exclude<Item.Any, ContainerItem>
+
+export type PlayableItem = Item.Any & {
+	start: Ms
+	duration: Ms
+}