@omnimedia/omnitool 1.1.0-7 → 1.1.0-70

package/s/driver/parts/compositor.ts

@@ -0,0 +1,187 @@
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture} from "pixi.js"
+
+import {Id} from "../../timeline/index.js"
+import {Composition, Layer} from "../fns/schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export class Compositor {
+
+	static async setup() {
+		const renderer = await autoDetectRenderer({
+			width: 1920,
+			height: 1080,
+			preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+			background: "black",
+			preferWebGLVersion: 2
+		})
+		const stage = new Container()
+		stage.interactive = true
+		return new this({renderer, stage})
+	}
+
+	constructor(public pixi: {renderer: Renderer, stage: Container}) {}
+
+	#transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+	// objects rendered for current Composition
+	#activeObjects = new Map<number, Container>()
+
+	async composite(
+		composition: Composition,
+	) {
+		const {stage, renderer} = this.pixi
+
+		this.#cleanup(this.#collectIds(composition))
+		const {dispose} = await this.#renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: 0,
+			duration: 0,
+		})
+
+		dispose()
+
+		return frame
+	}
+
+	/**
+	 * get object for current Composition
+	 * */
+	getActiveObject(id: Id) {
+		return this.#activeObjects.get(id)
+	}
+
+	async #renderLayer(
+		layer: Layer | Composition,
+		parent: Container,
+	) {
+		if (Array.isArray(layer)) {
+			layer.reverse()
+			const disposers: (() => void)[] = []
+			for (const child of layer) {
+				const result = await this.#renderLayer(child, parent)
+				disposers.push(result.dispose)
+			}
+			return {dispose: () => disposers.forEach(d => d())}
+		}
+
+		switch (layer.kind) {
+			case 'text':
+				return this.#renderTextLayer(layer, parent)
+			case 'image':
+				return this.#renderImageLayer(layer, parent)
+			case 'transition':
+				return this.#renderTransitionLayer(layer, parent)
+			case 'gap': {
+				this.pixi?.renderer.clear()
+				return {dispose: () => {}}
+			}
+			default:
+				console.warn('Unknown layer kind', (layer as any).kind)
+				return {dispose: () => {}}
+		}
+	}
+
+	#renderTextLayer(
+		layer: Extract<Layer, {kind: 'text'}>,
+		parent: Container,
+	) {
+		const text = this.#findOrCreate<Text>(layer)!
+		this.#applyTransform(text, layer.matrix)
+		parent.addChild(text)
+		return {
+			dispose: () => {}
+		}
+	}
+
+	#renderImageLayer(
+		layer: Extract<Layer, {kind: 'image'}>,
+		parent: Container,
+	) {
+		const texture = Texture.from(layer.frame)
+		const sprite = this.#findOrCreate<Sprite>(layer)!
+		sprite.texture = texture
+		this.#applyTransform(sprite, layer.matrix)
+		parent.addChild(sprite)
+		return {
+			dispose: () => {
+				texture.destroy(true)
+				layer.frame.close()
+			}
+		}
+	}
+
+	#renderTransitionLayer(
+		{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+		parent: Container,
+	) {
+		const transition = this.#transitions.get(name) ??
+			(this.#transitions.set(name, makeTransition({
+				name: "circle",
+				renderer: this.pixi.renderer
+			})),
+	  	this.#transitions.get(name)!
+		)
+		const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+		const sprite = new Sprite(texture)
+		parent.addChild(sprite)
+		return {dispose: () => sprite.destroy(false)}
+	}
+
+	#applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  	if (!worldMatrix) return
+		const mx = mat6ToMatrix(worldMatrix)
+  	target.setFromMatrix(mx)
+	}
+
+	#findOrCreate<T = Container>(layer: Layer) {
+		const object = this.#activeObjects.get(layer.id)
+		if(!object) {
+			switch (layer.kind) {
+				case 'text': {
+					const text = new Text({
+						text: layer.content,
+						style: layer.style
+					})
+					text.onmouseenter = () => console.log("enter text")
+					return this.#activeObjects
+						.set(layer.id, text)
+						.get(layer.id) as T
+				}
+				case 'image': {
+					const sprite = new Sprite()
+					sprite.onmouseenter = () => console.log("enter")
+					return this.#activeObjects
+						.set(layer.id, sprite)
+						.get(layer.id) as T
+				}
+			}
+		} else return object as T
+	}
+
+	#collectIds(layers: Layer | Composition): Set<number> {
+		const result = new Set<number>()
+		const traverse = (node: Layer | Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node) traverse(child)
+			} else {
+				result.add(node.id)
+			}
+		}
+		traverse(layers)
+		return result
+	}
+
+	#cleanup(activeIds: Set<number>) {
+		for (const id of this.#activeObjects.keys()) {
+			if (!activeIds.has(id)) {
+				const obj = this.#activeObjects.get(id)!
+				obj.destroy(true)
+				this.#activeObjects.delete(id)
+			}
+		}
+	}
+}

package/s/driver/parts/machina.ts

@@ -1,27 +1,26 @@
-import {WebMediaInfo} from "web-demuxer"
 
-type Events =
-	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
-	| {type: "info", data: WebMediaInfo}
-	| {type: "encoderQueueSize", size: number}
-
-type Handler = (event: Events) => void
+// type Events =
+// 	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+// 	| {type: "info", data: WebMediaInfo}
+// 	| {type: "encoderQueueSize", size: number}
+//
+// type Handler = (event: Events) => void
 
 export class Machina {
 	count = 0
 
-	#handlers = new Map<number, Handler>()
-
-	register(id: number, handler: Handler) {
-		this.#handlers.set(id, handler)
-	}
-
-	unregister(id: number) {
-		this.#handlers.delete(id)
-	}
-
-	dispatch(id: number, event: Events) {
-		this.#handlers.get(id)?.(event)
-	}
+	// #handlers = new Map<number, Handler>()
+	//
+	// register(id: number, handler: Handler) {
+	// 	this.#handlers.set(id, handler)
+	// }
+	//
+	// unregister(id: number) {
+	// 	this.#handlers.delete(id)
+	// }
+	//
+	// dispatch(id: number, event: Events) {
+	// 	this.#handlers.get(id)?.(event)
+	// }
 }
 

package/s/index.html.ts

@@ -15,7 +15,7 @@ export default ssg.page(import.meta.url, async orb => ({
 	head: html`
 		<link rel="preconnect" href="https://fonts.googleapis.com">
 		<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&display=swap" rel="stylesheet">
+		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&family=Space+Grotesk:wght@400;600&display=swap" rel="stylesheet">
 	`,
 
 	socialCard: {
@@ -28,31 +28,109 @@ export default ssg.page(import.meta.url, async orb => ({
 
 	body: html`
 		<section>
-			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
-			<button class=fetch>fetch</button>
-			<input type="file" class="file-input">
-			<div class=results></div>
-			<div class=filmstrip-demo>
-				<label for="viewable-range">viewable range:</label>
-				<input type="range" min="0" max="100" step="1" value="10" class="range" id="viewable-range" name="viewable-range">
-				<div class="range-view"></div>
-				<label for="range-size">viewable range size:</label>
-				<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5" id="range-size" name="range-size">
-				<label for="frequency">frequency:</label>
-				<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10" id="frequency" name="frequency">
-				<div class="frequency-view">10 (fps)</div>
-				<div id=filmstrip></div>
-			</div>
-			<div class=waveform-demo>
-				<label for="width">width:</label>
-				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
-			</div>
-			<div class=player>
-				<input class="seek" type="number" min="0">
-				<button class=play>play</button>
-				<button class=stop>stop</button>
+			<header class="hero">
+				<div>
+					<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
+					<p>Developer demos for decoding, preview, waveform, playback, and export.</p>
+				</div>
+			</header>
+
+			<div class="demo-grid">
+				<article class="demo-card" data-demo="transcode">
+					<header>
+						<h2>Transcode Test</h2>
+						<p>Full decode + composite + encode pipeline.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Run Transcode</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
+
+				<article class="demo-card" data-demo="filmstrip">
+					<header>
+						<h2>Filmstrip</h2>
+						<p>Thumbnail sampling across a time range.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Generate Filmstrip</button>
+					</div>
+					<div class="filmstrip-controls">
+						<label>viewable range</label>
+						<input type="range" min="0" max="100" step="1" value="10" class="range">
+						<div class="range-view"></div>
+						<label>range size</label>
+						<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5">
+						<label>frequency</label>
+						<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10">
+						<div class="frequency-view">10 (fps)</div>
+					</div>
+					<div class="filmstrip"></div>
+				</article>
+
+				<article class="demo-card" data-demo="waveform">
+					<header>
+						<h2>Waveform</h2>
+						<p>Audio peak rendering with adjustable width.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="audio/*,video/*" />
+						<button data-action="run">Build Waveform</button>
+					</div>
+					<div class="waveform-controls">
+						<label>width</label>
+						<input class="width" type="range" min="100" max="1000000" value="1000" step="100" />
+					</div>
+					<div class="waveform-canvas"></div>
+				</article>
+
+				<article class="demo-card" data-demo="playback">
+					<header>
+						<h2>Timeline Playback</h2>
+						<p>Build timeline and run the playback engine.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+					</div>
+					<div class="player-canvas"></div>
+					<div class="player">
+						<div class="timeline">
+							<div class="track">
+								<div class="playhead"></div>
+							</div>
+							<input class="scrub" type="range" min="0" max="0" step="1" value="0">
+						</div>
+						<div class="timecode">0:00.000 / 0:00.000</div>
+						<div class="controls">
+							<button class=play disabled>play</button>
+							<button class=stop disabled>stop</button>
+						</div>
+					</div>
+				</article>
+
+				<article class="demo-card" data-demo="export">
+					<header>
+						<h2>Timeline Export</h2>
+						<p>Build timeline and export a render.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+						<button data-action="export" disabled>Export</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
 			</div>
+
 		</section>
 	`,
 }))
-

package/s/index.ts

@@ -1,4 +1,5 @@
 
 export * from "./driver/driver.js"
+export * from "./driver/driver-worker.js"
 export * from "./timeline/index.js"
 

package/s/tests.bundle.ts

@@ -0,0 +1,11 @@
+
+import {Science} from "@e280/science"
+import omniTest from "./timeline/sugar/omni.test.js"
+import renderersTest from "./timeline/renderers/renderers.test.js"
+
+await Science.run
+({
+	omniTest,
+	renderersTest
+})
+

package/s/tests.html.ts

@@ -0,0 +1,28 @@
+
+import {ssg, html} from "@e280/scute"
+
+const title = "omnitool"
+const domain = "omnitool.omniclip.app"
+const favicon = "/assets/favicon.png"
+
+export default ssg.page(import.meta.url, async orb => ({
+	title,
+	dark: true,
+	css: "demo/demo.css",
+	js: "tests.bundle.min.js",
+
+	head: html`
+	`,
+
+	socialCard: {
+		title,
+		description: "video processing toolkit",
+		themeColor: "#3cff9c",
+		siteName: domain,
+		image: `https://${domain}${favicon}`,
+	},
+
+	body: html`
+	`,
+}))
+

package/s/timeline/index.ts

@@ -6,7 +6,12 @@ export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
 
+export * from "./parts/waveform/waveform.js"
+export * from "./parts/waveform/parts/types.js"
+export * from "./renderers/player/player.js"
+
 export * from "./sugar/o.js"
+export * from "./sugar/helpers.js"
 export * from "./sugar/omni.js"
 
 export * from "./utils/checksum.js"

package/s/timeline/parts/filmstrip.ts

@@ -17,7 +17,7 @@ export class Filmstrip {
 
 	private constructor(
 		private videoTrack: InputVideoTrack,
-		private options: Required<FilmstripOptions>
+		private options: FilmstripOptions
 	) {
 		this.#sink = new CanvasSink(videoTrack, options.canvasSinkOptions)
 	}
@@ -33,7 +33,8 @@ export class Filmstrip {
 				videoTrack, {
 					frequency: options.frequency ?? 1,
 					canvasSinkOptions: options.canvasSinkOptions ?? {width: 80, height: 50, fit: "fill"},
-					onChange: options.onChange
+					onChange: options.onChange,
+					onPlaceholders: options.onPlaceholders
 			})
 		else throw new Error("Source has no video track")
 	}
@@ -54,9 +55,28 @@ export class Filmstrip {
 		return this.options.frequency
 	}
 
-	#computeActiveRange([start, end]: TimeRange): TimeRange {
+	#computeActiveRange([start, end]: TimeRange, margin = 1): TimeRange {
 		const tileSize = end - start
-		return [start - tileSize, end + tileSize]
+		return [start - tileSize * margin, end + tileSize * margin]
+	}
+
+	async #generatePlaceholders() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededTimestamps = new Set<number>()
+
+		// duration should be computed but with trim etc also
+		const duration = await this.videoTrack.computeDuration()
+		for (
+			let timestamp = Math.max(0, rangeStart);
+			timestamp <= rangeEnd;
+			timestamp += this.options.frequency
+		) {
+			// Clamp to valid time range
+			if (timestamp >= 0 && timestamp <= duration)
+				neededTimestamps.add(timestamp)
+		}
+
+		this.options.onPlaceholders?.([...neededTimestamps])
 	}
 
 	async #generateTiles() {
@@ -66,13 +86,13 @@ export class Filmstrip {
 		// duration should be computed but with trim etc also
 		const duration = await this.videoTrack.computeDuration()
 		for (
-			let timestamp = rangeStart;
+			let timestamp = Math.max(0, rangeStart);
 			timestamp <= rangeEnd;
 			timestamp += this.options.frequency
 		) {
 			// Clamp to valid time range
 			if (timestamp >= 0 && timestamp <= duration)
-				neededTimestamps.add(+timestamp.toFixed(3))
+				neededTimestamps.add(timestamp)
 		}
 
 		const missingTimestamps = [...neededTimestamps]
@@ -106,15 +126,20 @@ export class Filmstrip {
  	* @param visibleRange - The current timeline viewport as a [start, end] tuple in seconds.
  	*/
 	set range(visibleRange: TimeRange) {
-		const newRange = this.#computeActiveRange(visibleRange)
-		// Avoid redundant updates
-		if (
-			this.#activeRange[0] === newRange[0] &&
-			this.#activeRange[1] === newRange[1]
-		)
-			return
+		const [visStart, visEnd] = visibleRange
+		const visibleSize = visEnd - visStart
+		const [actStart, actEnd] = this.#activeRange
+
+		// trigger when we're 1x visible width away from margin edges
+		const leftTrigger = actStart + visibleSize
+		const rightTrigger = actEnd - visibleSize
+
+		const nearLeftEdge = visStart < leftTrigger
+		const nearRightEdge = visEnd > rightTrigger
+
+		if (!nearLeftEdge && !nearRightEdge) return
 
-		this.#activeRange = newRange
+		this.#activeRange = this.#computeActiveRange(visibleRange, 2)
 		this.#update()
 	}
 
@@ -122,6 +147,7 @@ export class Filmstrip {
 	#shouldRunAgain = false
 
 	async #update() {
+		this.#generatePlaceholders()
 		// Perform update immediately. If multiple updates are requested while updating,
 		// only the latest one will run after the current finishes (skips intermediate ones).
 		if(this.#updating) {
@@ -147,11 +173,12 @@ export class Filmstrip {
 	}
 }
 
-type TimeRange = [number, number]
+export type TimeRange = [start: number, end: number]
 
 interface FilmstripOptions {
-	frequency?: number
+	frequency: number
 	canvasSinkOptions?: CanvasSinkOptions
+	onPlaceholders?: (timestamps: number[]) => void
 	onChange: (tiles: {
 		time: number
 		canvas: WrappedCanvas

package/s/timeline/parts/item.ts

@@ -1,6 +1,9 @@
 
+import {TextStyleOptions} from "pixi.js"
+
 import {Id, Hash} from "./basics.js"
 import {Transform} from "../types.js"
+import {Ms} from "../../units/ms.js"
 
 export enum Kind {
 	Sequence,
@@ -11,6 +14,7 @@ export enum Kind {
 	Gap,
 	Spatial,
 	Transition,
+	TextStyle
 }
 
 export enum Effect {
@@ -18,11 +22,18 @@ export enum Effect {
 }
 
 export namespace Item {
-  export type Spatial = {
-    id: Id
-    kind: Kind.Spatial
-    transform: Transform
-  }
+	export type TextStyle = {
+		id: Id
+		kind: Kind.TextStyle
+		style: TextStyleOptions
+	}
+
+	export type Spatial = {
+		id: Id
+		kind: Kind.Spatial
+		transform: Transform
+		enabled: boolean
+	}
 
 	export type Gap = {
 		id: Id
@@ -59,14 +70,16 @@ export namespace Item {
 		mediaHash: Hash
 		start: number
 		duration: number
+		gain?: number
 	}
 
 	export type Text = {
 		id: Id
 		kind: Kind.Text
 		content: string
+		duration: number
 		spatialId?: Id
-		color: string
+		styleId?: Id
 	}
 
 	export type Transition = {
@@ -85,6 +98,14 @@ export namespace Item {
 		| Gap
 		| Transition
 		| Spatial
+		| TextStyle
 	)
 }
 
+export type ContainerItem = Item.Sequence | Item.Stack
+export type NonContainerItem = Exclude<Item.Any, ContainerItem>
+
+export type PlayableItem = Item.Any & {
+	start: Ms
+	duration: Ms
+}

package/s/timeline/parts/media.ts

@@ -14,13 +14,23 @@ export class Media {
 
 	static async analyze(datafile: Datafile) {
 		const media = new this(datafile)
-		media.duration = 10
-		const {video, audio} = await this.#has("/assets/temp/gl.mp4")
+		const duration = (await this.duration(datafile.url)) * 1000
+		media.duration = duration
+		const {video, audio} = await this.#has(datafile.url)
 		media.hasAudio = audio
 		media.hasVideo = video
 		return media
 	}
 
+	static async duration(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		const duration =  await input.computeDuration()
+		return Number(duration.toFixed(5)) // fix weird floating points
+	}
+
 	static async #has(source: DecoderSource) {
 		const input = new Input({
 			formats: ALL_FORMATS,