@omnimedia/omnitool 1.1.0-8 → 1.1.0-80

package/s/driver/parts/compositor.ts

@@ -0,0 +1,316 @@
+
+import {pub} from "@e280/stz"
+import {
+	autoDetectRenderer,
+	ColorMatrixFilter,
+	Container,
+	FederatedPointerEvent,
+	Filter,
+	Graphics,
+	Renderer,
+	Sprite,
+	Text,
+	Texture
+} from "pixi.js"
+import * as PixiFilters from "pixi-filters"
+
+import {Id} from "../../timeline/index.js"
+import {Crop} from "../../timeline/parts/item.js"
+import {findPixiFilter} from "../utils/find-pixi-filter.js"
+import {Composition, FilterSpec, Layer} from "../fns/schematic.js"
+import {Mat6, mat6ToMatrix} from "../../timeline/utils/matrix.js"
+import {makeTransition} from "../../features/transition/transition.js"
+
+export class Compositor {
+	onPointerDown = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onPointerMove = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onPointerUp = pub<[{event: FederatedPointerEvent, id: Id, object: Container}]>()
+	onDispose = pub<[{id: Id, object: Container}]>()
+
+	static async setup() {
+		const renderer = await autoDetectRenderer({
+			width: 1920,
+			height: 1080,
+			preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+			background: "black",
+			preferWebGLVersion: 2
+		})
+		const stage = new Container()
+		stage.interactive = true
+		return new this({renderer, stage})
+	}
+
+	constructor(public pixi: {renderer: Renderer, stage: Container}) {}
+
+	#transitions: Map<string, ReturnType<typeof makeTransition>> = new Map()
+	// objects rendered for current Composition
+	#activeObjects = new Map<number, {sprite: Container, dispose: () => void}>()
+	#cropMasks = new WeakMap<Container, Graphics>()
+
+	async composite(
+		composition: Composition,
+	) {
+		const {stage, renderer} = this.pixi
+
+		this.#cleanup(this.#collectIds(composition))
+		const {dispose} = await this.#renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: 0,
+			duration: 0,
+		})
+
+		dispose()
+
+		return frame
+	}
+
+	/**
+	 * get object for current Composition
+	 * */
+	getActiveObject(id: Id) {
+		return this.#activeObjects.get(id)?.sprite
+	}
+
+	async #renderLayer(
+		layer: Layer | Composition,
+		parent: Container,
+	) {
+		if (Array.isArray(layer)) {
+			const disposers: (() => void)[] = []
+			for (const child of [...layer].reverse()) {
+				const result = await this.#renderLayer(child, parent)
+				disposers.push(result.dispose)
+			}
+			return {dispose: () => disposers.forEach(d => d())}
+		}
+
+		switch (layer.kind) {
+			case 'text':
+				return this.#renderTextLayer(layer, parent)
+			case 'image':
+				return this.#renderImageLayer(layer, parent)
+			case 'transition':
+				return this.#renderTransitionLayer(layer, parent)
+			case 'gap': {
+				this.pixi?.renderer.clear()
+				return {dispose: () => {}}
+			}
+			default:
+				console.warn('Unknown layer kind', (layer as any).kind)
+				return {dispose: () => {}}
+		}
+	}
+
+	#renderTextLayer(
+		layer: Extract<Layer, {kind: 'text'}>,
+		parent: Container,
+	) {
+		const sprite = this.#findOrCreate<Text>(layer)!
+		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
+		this.#applyCrop(sprite, layer.crop)
+		this.#applyFilters(sprite, layer.filters)
+		parent.addChild(sprite)
+		return {
+			dispose: () => {}
+		}
+	}
+
+	#renderImageLayer(
+		layer: Extract<Layer, {kind: 'image'}>,
+		parent: Container,
+	) {
+		const sprite = this.#findOrCreate<Sprite>(layer)!
+
+		if (sprite.texture && sprite.texture !== Texture.EMPTY) {
+			sprite.texture.destroy(true)
+		}
+
+		const texture = Texture.from(layer.frame)
+		sprite.texture = texture
+		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
+		this.#applyCrop(sprite, layer.crop)
+		this.#applyFilters(sprite, layer.filters)
+		parent.addChild(sprite)
+
+		return {
+			dispose: () => {
+				layer.frame.close()
+			}
+		}
+	}
+
+	#renderTransitionLayer(
+		{from, to, progress, name}: Extract<Layer, {kind: 'transition'}>,
+		parent: Container,
+	) {
+		const transition = this.#transitions.get(name) ??
+			(this.#transitions.set(name, makeTransition({
+				name: "circle",
+				renderer: this.pixi.renderer
+			})),
+	  	this.#transitions.get(name)!
+		)
+		const texture = transition.render({from, to, progress, width: from.displayWidth, height: from.displayHeight})
+		const sprite = new Sprite(texture)
+		parent.addChild(sprite)
+		return {dispose: () => sprite.destroy(false)}
+	}
+
+	#applyTransform(target: Sprite | Text, worldMatrix?: Mat6) {
+  	if (!worldMatrix) return
+		const mx = mat6ToMatrix(worldMatrix)
+  	target.setFromMatrix(mx)
+	}
+
+	#applyAlpha(target: Container, alpha?: number) {
+		target.alpha = alpha ?? 1
+	}
+
+	#applyCrop(target: Container, crop?: Crop) {
+		const existing = this.#cropMasks.get(target)
+		if (existing) {
+			if (target.mask === existing)
+				target.mask = null
+			existing.removeFromParent()
+			existing.clear()
+		}
+
+		if (!crop || crop.every(value => value === 0))
+			return
+
+		const [top, right, bottom, left] = crop
+		const activeMask = target.mask
+		target.mask = null
+		const bounds = target.getLocalBounds()
+		target.mask = activeMask
+		const x = bounds.x + bounds.width * left
+		const y = bounds.y + bounds.height * top
+		const width = bounds.width * Math.max(0, 1 - left - right)
+		const height = bounds.height * Math.max(0, 1 - top - bottom)
+
+		const mask = existing ?? new Graphics()
+		mask.clear()
+		mask.beginFill(0xffffff)
+		mask.drawRect(x, y, width, height)
+		mask.endFill()
+		target.addChild(mask)
+		target.mask = mask
+		this.#cropMasks.set(target, mask)
+	}
+
+	#applyFilters(target: Container, specs: FilterSpec[] | undefined) {
+		if (!specs?.length) {
+			target.filters = null
+			return
+		}
+
+		target.filters = specs
+			.map((spec): Filter | undefined => {
+				switch (spec.type) {
+					case "ColorMatrixFilter": {
+						const {matrix, ...params} = spec.params ?? {}
+						const filter = new ColorMatrixFilter(params)
+						if (matrix)
+							filter.matrix = matrix
+						return filter
+					}
+					case "EmbossFilter":
+						return new PixiFilters.EmbossFilter(spec.params?.strength)
+					case "PixelateFilter":
+						return new PixiFilters.PixelateFilter(spec.params?.size)
+					default: {
+						const PixiFilter = findPixiFilter(spec.type)
+						return PixiFilter ? new PixiFilter(spec.params) : undefined
+					}
+				}
+			})
+			.filter((filter): filter is Filter => !!filter)
+	}
+
+	#findOrCreate<T = Container>(layer: Layer) {
+		const object = this.#activeObjects.get(layer.id)
+		if(!object) {
+			switch (layer.kind) {
+				case 'text': {
+					const text = new Text({
+						text: layer.content,
+						style: layer.style
+					})
+					text.eventMode = "static"
+					const down = (event: FederatedPointerEvent) => this.onPointerDown.publish({event, id: layer.id, object: text})
+					const up = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: text})
+					const move = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: text})
+
+					text.on("pointerdown", down)
+					text.on("pointerup", up)
+					text.on("pointermove", move)
+
+					return this.#activeObjects
+						.set(layer.id, {
+							sprite: text,
+							dispose: () => {
+								this.onDispose.publish({id: layer.id, object: text})
+								text.off("pointermove", move)
+								text.off("pointerdown", down)
+								text.off("pointerup", up)
+							}
+						})
+						.get(layer.id)?.sprite as T
+				}
+				case 'image': {
+					const sprite = new Sprite()
+					sprite.eventMode = "static"
+					const down = (event: FederatedPointerEvent) => this.onPointerDown.publish({event, id: layer.id, object: sprite})
+					const up = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: sprite})
+					const move = (event: FederatedPointerEvent) => this.onPointerUp.publish({event, id: layer.id, object: sprite})
+
+					sprite.on("pointerdown", down)
+					sprite.on("pointermove", move)
+					sprite.on("pointerup", up)
+
+					return this.#activeObjects
+						.set(layer.id, {
+							sprite,
+							dispose: () => {
+								this.onDispose.publish({id: layer.id, object: sprite})
+								sprite.off("pointerdown", down)
+								sprite.off("pointermove", move)
+								sprite.off("pointerup", up)
+							}
+						})
+						.get(layer.id)?.sprite as T
+				}
+			}
+		} else return object.sprite as T
+	}
+
+	#collectIds(layers: Layer | Composition): Set<number> {
+		const result = new Set<number>()
+		const traverse = (node: Layer | Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node) traverse(child)
+			} else {
+				result.add(node.id)
+			}
+		}
+		traverse(layers)
+		return result
+	}
+
+	#cleanup(activeIds: Set<number>) {
+		for (const id of this.#activeObjects.keys()) {
+			if (!activeIds.has(id)) {
+				const {sprite, dispose} = this.#activeObjects.get(id)!
+				dispose()
+				sprite.destroy(true)
+				this.#activeObjects.delete(id)
+			}
+		}
+	}
+}

package/s/driver/parts/machina.ts

@@ -1,27 +1,26 @@
-import {WebMediaInfo} from "web-demuxer"
 
-type Events =
-	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
-	| {type: "info", data: WebMediaInfo}
-	| {type: "encoderQueueSize", size: number}
-
-type Handler = (event: Events) => void
+// type Events =
+// 	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+// 	| {type: "info", data: WebMediaInfo}
+// 	| {type: "encoderQueueSize", size: number}
+//
+// type Handler = (event: Events) => void
 
 export class Machina {
 	count = 0
 
-	#handlers = new Map<number, Handler>()
-
-	register(id: number, handler: Handler) {
-		this.#handlers.set(id, handler)
-	}
-
-	unregister(id: number) {
-		this.#handlers.delete(id)
-	}
-
-	dispatch(id: number, event: Events) {
-		this.#handlers.get(id)?.(event)
-	}
+	// #handlers = new Map<number, Handler>()
+	//
+	// register(id: number, handler: Handler) {
+	// 	this.#handlers.set(id, handler)
+	// }
+	//
+	// unregister(id: number) {
+	// 	this.#handlers.delete(id)
+	// }
+	//
+	// dispatch(id: number, event: Events) {
+	// 	this.#handlers.get(id)?.(event)
+	// }
 }
 

package/s/driver/utils/find-pixi-filter.ts

@@ -0,0 +1,24 @@
+import {
+	AlphaFilter,
+	BlurFilter,
+	ColorMatrixFilter,
+	NoiseFilter,
+	Filter,
+} from "pixi.js"
+import * as PixiFilters from "pixi-filters"
+
+import type {FilterType} from "../../timeline/parts/filters.js"
+
+type FilterConstructor = new (options?: any) => Filter
+
+const pixiFilters: Partial<Record<FilterType, FilterConstructor>> = {
+	AlphaFilter,
+	BlurFilter,
+	ColorMatrixFilter,
+	NoiseFilter,
+	...PixiFilters
+}
+
+export function findPixiFilter(type: FilterType) {
+	return pixiFilters[type]
+}

package/s/index.html.ts

@@ -15,7 +15,7 @@ export default ssg.page(import.meta.url, async orb => ({
 	head: html`
 		<link rel="preconnect" href="https://fonts.googleapis.com">
 		<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&display=swap" rel="stylesheet">
+		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&family=Space+Grotesk:wght@400;600&display=swap" rel="stylesheet">
 	`,
 
 	socialCard: {
@@ -28,31 +28,109 @@ export default ssg.page(import.meta.url, async orb => ({
 
 	body: html`
 		<section>
-			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
-			<button class=fetch>fetch</button>
-			<input type="file" class="file-input">
-			<div class=results></div>
-			<div class=filmstrip-demo>
-				<label for="viewable-range">viewable range:</label>
-				<input type="range" min="0" max="100" step="1" value="10" class="range" id="viewable-range" name="viewable-range">
-				<div class="range-view"></div>
-				<label for="range-size">viewable range size:</label>
-				<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5" id="range-size" name="range-size">
-				<label for="frequency">frequency:</label>
-				<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10" id="frequency" name="frequency">
-				<div class="frequency-view">10 (fps)</div>
-				<div id=filmstrip></div>
-			</div>
-			<div class=waveform-demo>
-				<label for="width">width:</label>
-				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
-			</div>
-			<div class=player>
-				<input class="seek" type="number" min="0">
-				<button class=play>play</button>
-				<button class=stop>stop</button>
+			<header class="hero">
+				<div>
+					<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
+					<p>Developer demos for decoding, preview, waveform, playback, and export.</p>
+				</div>
+			</header>
+
+			<div class="demo-grid">
+				<article class="demo-card" data-demo="transcode">
+					<header>
+						<h2>Transcode Test</h2>
+						<p>Full decode + composite + encode pipeline.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Run Transcode</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
+
+				<article class="demo-card" data-demo="filmstrip">
+					<header>
+						<h2>Filmstrip</h2>
+						<p>Thumbnail sampling across a time range.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*" />
+						<button data-action="run">Generate Filmstrip</button>
+					</div>
+					<div class="filmstrip-controls">
+						<label>viewable range</label>
+						<input type="range" min="0" max="100" step="1" value="10" class="range">
+						<div class="range-view"></div>
+						<label>range size</label>
+						<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5">
+						<label>frequency</label>
+						<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10">
+						<div class="frequency-view">10 (fps)</div>
+					</div>
+					<div class="filmstrip"></div>
+				</article>
+
+				<article class="demo-card" data-demo="waveform">
+					<header>
+						<h2>Waveform</h2>
+						<p>Audio peak rendering with adjustable width.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="audio/*,video/*" />
+						<button data-action="run">Build Waveform</button>
+					</div>
+					<div class="waveform-controls">
+						<label>width</label>
+						<input class="width" type="range" min="100" max="1000000" value="1000" step="100" />
+					</div>
+					<div class="waveform-canvas"></div>
+				</article>
+
+				<article class="demo-card" data-demo="playback">
+					<header>
+						<h2>Timeline Playback</h2>
+						<p>Build timeline and run the playback engine.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+					</div>
+					<div class="player-canvas"></div>
+					<div class="player">
+						<div class="timeline">
+							<div class="track">
+								<div class="playhead"></div>
+							</div>
+							<input class="scrub" type="range" min="0" max="0" step="1" value="0">
+						</div>
+						<div class="timecode">0:00.000 / 0:00.000</div>
+						<div class="controls">
+							<button class=play disabled>play</button>
+							<button class=stop disabled>stop</button>
+						</div>
+					</div>
+				</article>
+
+				<article class="demo-card" data-demo="export">
+					<header>
+						<h2>Timeline Export</h2>
+						<p>Build timeline and export a render.</p>
+					</header>
+					<div class="demo-controls">
+						<input type="file" accept="video/*,audio/*" />
+						<button data-action="export" disabled>Export</button>
+					</div>
+					<div class="demo-progress">
+						<progress class="progress"></progress>
+						<span class="status">idle</span>
+					</div>
+					<div class="demo-preview"></div>
+				</article>
 			</div>
+
 		</section>
 	`,
 }))
-

package/s/index.ts

@@ -1,4 +1,5 @@
 
 export * from "./driver/driver.js"
+export * from "./driver/driver-worker.js"
 export * from "./timeline/index.js"
 

package/s/tests.bundle.ts

@@ -0,0 +1,11 @@
+
+import {Science} from "@e280/science"
+import omniTest from "./timeline/sugar/omni.test.js"
+import renderersTest from "./timeline/renderers/renderers.test.js"
+
+await Science.run
+({
+	omniTest,
+	renderersTest
+})
+

package/s/tests.html.ts

@@ -0,0 +1,28 @@
+
+import {ssg, html} from "@e280/scute"
+
+const title = "omnitool"
+const domain = "omnitool.omniclip.app"
+const favicon = "/assets/favicon.png"
+
+export default ssg.page(import.meta.url, async orb => ({
+	title,
+	dark: true,
+	css: "demo/demo.css",
+	js: "tests.bundle.min.js",
+
+	head: html`
+	`,
+
+	socialCard: {
+		title,
+		description: "video processing toolkit",
+		themeColor: "#3cff9c",
+		siteName: domain,
+		image: `https://${domain}${favicon}`,
+	},
+
+	body: html`
+	`,
+}))
+

package/s/timeline/index.ts

@@ -1,15 +1,21 @@
 
 export * from "./parts/basics.js"
+export * from "./parts/animations.js"
+export * from "./parts/filters.js"
 export * from "./parts/item.js"
 export * from "./parts/media.js"
 export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
+export * from "./types.js"
+
+export * from "./parts/waveform/waveform.js"
+export * from "./parts/waveform/parts/types.js"
+export * from "./renderers/player/player.js"
 
 export * from "./sugar/o.js"
+export * from "./sugar/helpers.js"
 export * from "./sugar/omni.js"
-export * from "./timeline.js"
 
 export * from "./utils/checksum.js"
 export * from "./utils/datafile.js"
-

package/s/timeline/parts/animations.ts

@@ -0,0 +1,56 @@
+
+import type {Interpolation} from "../types.js"
+
+export type AnimationType = "scalar" | "transform"
+export type AnimationChannelType = "number"
+export type AnimationUnit = "pixel" | "scale" | "radian" | "ratio"
+
+export type AnimationChannel = {
+	path?: string
+	type: AnimationChannelType
+	default: number
+	unit?: AnimationUnit
+	range?: readonly [min: number, max: number]
+}
+
+export type AnimationDefinition = {
+	type: AnimationType
+	defaultTerp: Interpolation
+	channels: readonly AnimationChannel[]
+}
+
+export const spatialAnimations = {
+	transform: {
+		type: "transform",
+		defaultTerp: "linear",
+		channels: [
+			{path: "position.x", type: "number", default: 0, unit: "pixel"},
+			{path: "position.y", type: "number", default: 0, unit: "pixel"},
+			{path: "scale.x", type: "number", default: 1, unit: "scale"},
+			{path: "scale.y", type: "number", default: 1, unit: "scale"},
+			{path: "rotation", type: "number", default: 0, unit: "radian"},
+		],
+	},
+} as const satisfies Record<string, AnimationDefinition>
+
+export const visualAnimations = {
+	opacity: {
+		type: "scalar",
+		defaultTerp: "linear",
+		channels: [
+			{type: "number", default: 1, unit: "ratio", range: [0, 1]},
+		],
+	},
+} as const satisfies Record<string, AnimationDefinition>
+
+// const audioAnimations = {}
+
+export const animations = {
+	...spatialAnimations,
+	...visualAnimations,
+} as const
+
+export type SpatialAnimationProperty = keyof typeof spatialAnimations
+export type VisualAnimationProperty = keyof typeof visualAnimations
+export type AnimationProperty = keyof typeof animations
+