@omnimedia/omnitool 1.1.0-78 → 1.1.0-79

package/README.md

@@ -81,7 +81,7 @@ const timeline = timeline(
 
 ## 🎛 Filters
 
-Inline filter application:
+Filter application:
 
 ```ts
 const timeline = omni.timeline(o =>
@@ -146,6 +146,89 @@ const timeline = omni.timeline(o => {
 })
 ```
 
+Animated spatial transforms:
+
+```ts
+const timeline = omni.timeline(o => {
+	const slideIn = o.animatedSpatial(
+		o.anim.transform("linear", [
+			[0, o.transform({position: [-400, 0]})],
+			[1000, o.transform({position: [0, 0]})],
+		])
+	)
+
+	const title = o.text("Lower third", {
+		duration: 2000,
+		styles: {fill: "white", fontSize: 36}
+	})
+	o.set(title.id, {spatialId: slideIn.id})
+
+	return o.stack(
+		o.video(clip, {duration: 4000}),
+		title
+	)
+})
+```
+
+Animation application:
+
+```ts
+const timeline = omni.timeline(o => {
+	const title = o.animate.opacity(
+		o.text("Lower third", {
+			duration: 2000,
+			styles: {fill: "white", fontSize: 36},
+		}),
+		"easeIn",
+		[
+			[0, 0],
+			[700, 1],
+		]
+	)
+
+	return o.stack(
+		o.video(clip, {duration: 4000}),
+		title
+	)
+})
+```
+
+Reusable animation:
+
+```ts
+const timeline = omni.timeline(o => {
+	const fadeIn = o.animate.opacity.make("easeIn", [
+		[0, 0],
+		[700, 1],
+	])
+
+	const title = o.text("Lower third", {
+		duration: 2000,
+		styles: {fill: "white", fontSize: 36},
+	})
+	o.set(title.id, {animationId: fadeIn.id})
+
+	return o.stack(
+		o.video(clip, {duration: 4000}),
+		title
+	)
+})
+```
+
+Animation registry:
+
+```ts
+import {animations} from "@omnimedia/omnitool"
+
+Object.entries(animations).forEach(([property, meta]) => {
+	console.log(property, meta.value)
+	// transform transform
+	// opacity scalar
+})
+```
+
+This is useful for UI code that needs to list keyframeable properties and choose the right editor shape for each one.
+
 Worker URL notes:
 - `Driver.setup()` defaults to `/node_modules/@omnimedia/omnitool/x/driver/driver.worker.bundle.min.js`.
 - If you serve the worker from a different location, pass `workerUrl`:
@@ -228,4 +311,27 @@ omnitool ai "make a 15s promo for tea"
 
 - smooth seeking
 - keyframes
+- custom filters, likely via driver-side registration with timeline sugar such as:
+
+```ts
+// Register custom filter
+driver.registerFilter({
+	type: "vhs",
+	make: params => new Filter(/* ... */),
+	schema: {
+		intensity: {type: "number", min: 0, max: 1, default: 0.5},
+		scanlines: {type: "boolean", default: true},
+	},
+})
+
+// Use custom filter
+const timeline = omni.timeline(o =>
+	o.filter.custom(
+		"vhs",
+		o.video(clip, {duration: 3000}),
+		{intensity: 0.8}
+	)
+)
+```
+
 - server-side, not just browsers

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-78",
+	"version": "1.1.0-79",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",

package/s/demo/routines/timeline-setup.ts

@@ -6,30 +6,45 @@ export async function TimelineSchemaTest(driver: Driver, file: File) {
 	const omni = new Omni(driver)
 	const {videoA} = await omni.load({videoA: Datafile.make(file)})
 	const timeline = omni.timeline(o => {
-	const text = o.text("content", {duration: 1000})
-	const style = o.textStyle({fill: "green", fontSize: 100})
-	const spatial = o.spatial(
-		o.transform({
-			position: [240, 160],
-			scale: [1.4, 1.4],
-			rotation: 0.08,
-		}),
-		[0.15, 0.1, 0.05, 0.2],
-	)
+		const text = o.text("content", {duration: 3000})
+		const fade = o.animate.opacity.make("easeIn", [
+			[0, 0],
+			[700, 1],
+			[2200, 1],
+			[3000, 0.35],
+		])
+		const style = o.textStyle({fill: "green", fontSize: 100})
+		const videoSpatial = o.spatial(
+			o.transform({
+				position: [240, 160],
+				scale: [1.4, 1.4],
+				rotation: 0.08,
+			}),
+			[0.15, 0.1, 0.05, 0.2],
+		)
+		const textSpatial = o.animatedSpatial(
+			o.anim.transform("linear", [
+				[0, o.transform({position: [-320, 80], scale: [0.7, 0.7]})],
+				[1000, o.transform({position: [120, 80], scale: [1, 1]})],
+				[2000, o.transform({position: [200, 40], scale: [1.35, 1.35], rotation: 8})],
+				[3000, o.transform({position: [320, 0], scale: [1.15, 1.15], rotation: 0})],
+			]),
+		)
 
-	const video = o.video(videoA, {duration: 3000, start: 1000})
-	o.set<Item.Text>(text.id, {styleId: style.id})
-	o.set<Item.Video>(video.id, {spatialId: spatial.id})
+		const video = o.video(videoA, {duration: 3000, start: 1000})
+		o.set<Item.Text>(text.id, {styleId: style.id, spatialId: textSpatial.id, animationId: fade.id})
+		o.set<Item.Video>(video.id, {spatialId: videoSpatial.id})
 
-	return o.sequence(
-		o.stack(
-			text,
-			video,
-			o.audio(videoA, {duration: 1000})
-		),
-		o.gap(500),
-		o.video(videoA, {duration: 7000, start: 5000})
-	)})
+		return o.sequence(
+			o.stack(
+				text,
+				video,
+				o.audio(videoA, {duration: 1000})
+			),
+			o.gap(500),
+			o.video(videoA, {duration: 7000, start: 5000})
+		)
+	})
 
 	return {timeline, omni}
 }

package/s/driver/fns/schematic.ts

@@ -88,6 +88,7 @@ export type TextLayer = {
 	content: string
 	style?: TextStyleOptions
 	matrix?: Mat6
+	alpha?: number
 	crop?: Crop
 	filters?: FilterSpec[]
 }
@@ -97,6 +98,7 @@ export type ImageLayer = {
 	kind: 'image'
 	frame: VideoFrame
 	matrix?: Mat6
+	alpha?: number
 	crop?: Crop
 	filters?: FilterSpec[]
 }

package/s/driver/parts/compositor.ts

@@ -111,6 +111,7 @@ export class Compositor {
 	) {
 		const sprite = this.#findOrCreate<Text>(layer)!
 		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
 		this.#applyCrop(sprite, layer.crop)
 		this.#applyFilters(sprite, layer.filters)
 		parent.addChild(sprite)
@@ -132,6 +133,7 @@ export class Compositor {
 		const texture = Texture.from(layer.frame)
 		sprite.texture = texture
 		this.#applyTransform(sprite, layer.matrix)
+		this.#applyAlpha(sprite, layer.alpha)
 		this.#applyCrop(sprite, layer.crop)
 		this.#applyFilters(sprite, layer.filters)
 		parent.addChild(sprite)
@@ -166,6 +168,10 @@ export class Compositor {
   	target.setFromMatrix(mx)
 	}
 
+	#applyAlpha(target: Container, alpha?: number) {
+		target.alpha = alpha ?? 1
+	}
+
 	#applyCrop(target: Container, crop?: Crop) {
 		const existing = this.#cropMasks.get(target)
 		if (existing) {

package/s/timeline/index.ts

@@ -1,11 +1,13 @@
 
 export * from "./parts/basics.js"
+export * from "./parts/animations.js"
 export * from "./parts/filters.js"
 export * from "./parts/item.js"
 export * from "./parts/media.js"
 export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
+export * from "./types.js"
 
 export * from "./parts/waveform/waveform.js"
 export * from "./parts/waveform/parts/types.js"

package/s/timeline/parts/animations.ts

@@ -0,0 +1,24 @@
+export type AnimationValue = "scalar" | "transform"
+
+export type AnimationDefinition = {
+	value: AnimationValue
+}
+
+export const spatialAnimations = {
+	transform: {value: "transform"},
+} as const satisfies Record<string, AnimationDefinition>
+
+export const visualAnimations = {
+	opacity: {value: "scalar"},
+} as const satisfies Record<string, AnimationDefinition>
+
+// const audioAnimations = {}
+
+export const animations = {
+	...spatialAnimations,
+	...visualAnimations,
+} as const
+
+export type SpatialAnimationProperty = keyof typeof spatialAnimations
+export type VisualAnimationProperty = keyof typeof visualAnimations
+export type AnimationProperty = keyof typeof animations

package/s/timeline/parts/item.ts

@@ -2,9 +2,9 @@
 import {TextStyleOptions} from "pixi.js"
 
 import {Id, Hash} from "./basics.js"
-import {Transform} from "../types.js"
 import {Ms} from "../../units/ms.js"
 import type {FilterParams, FilterType} from "./filters.js"
+import {Anim, TrackTransform, Transform, VisualAnimations} from "../types.js"
 
 export type Crop = [top: number, right: number, bottom: number, left: number]
 
@@ -16,6 +16,8 @@ export enum Kind {
 	Text,
 	Gap,
 	Spatial,
+	AnimatedSpatial,
+	Animation,
 	Transition,
 	TextStyle,
 	Filter
@@ -40,6 +42,21 @@ export namespace Item {
 		enabled: boolean
 	}
 
+	export type AnimatedSpatial = {
+		id: Id
+		kind: Kind.AnimatedSpatial
+		anim: Anim<TrackTransform>
+		crop?: Crop
+		enabled: boolean
+	}
+
+	export type Animation = {
+		id: Id
+		kind: Kind.Animation
+		anims: VisualAnimations
+		enabled: boolean
+	}
+
 	export type Filter<T extends FilterType = FilterType> = {
 		id: Id
 		kind: Kind.Filter
@@ -77,6 +94,7 @@ export namespace Item {
 		start: number
 		duration: number
 		spatialId?: Id
+		animationId?: Id
 		filterIds?: Id[]
 	}
 
@@ -95,6 +113,7 @@ export namespace Item {
 		content: string
 		duration: number
 		spatialId?: Id
+		animationId?: Id
 		styleId?: Id
 		filterIds?: Id[]
 	}
@@ -115,6 +134,8 @@ export namespace Item {
 		| Gap
 		| Transition
 		| Spatial
+		| AnimatedSpatial
+		| Animation
 		| TextStyle
 		| Filter
 	)
@@ -123,6 +144,8 @@ export namespace Item {
 export type ContainerItem = Item.Sequence | Item.Stack
 export type NonContainerItem = Exclude<Item.Any, ContainerItem>
 export type FilterableItem = Item.Sequence | Item.Stack | Item.Video | Item.Text
+export type SpatialItem = Item.Spatial | Item.AnimatedSpatial
+export type VisualAnimatableItem = Item.Video | Item.Text
 
 export type PlayableItem = Item.Any & {
 	start: Ms

package/s/timeline/renderers/parts/handy.ts

@@ -1,26 +1,28 @@
 
 import {ms, Ms} from '../../../units/ms.js'
 import {Id, TimelineFile} from '../../parts/basics.js'
+import { SampleContext } from './samplers/visual/parts/types.js'
 import {I6, Mat6, mul6, transformToMat6} from '../../utils/matrix.js'
-import {ContainerItem, Item, Kind, PlayableItem} from '../../parts/item.js'
+import {resolveScalarAnimation, resolveSpatialTransform} from '../../utils/anim.js'
+import {ContainerItem, Item, Kind, PlayableItem, SpatialItem} from '../../parts/item.js'
 
 function isPlayableItem(item: Item.Any): item is PlayableItem {
 	return 'duration' in item
 }
 
 type WalkAtCallbacks = {
-	sequence: (x: Item.Sequence, localTime: Ms, ancestors: ContainerItem[]) => void
-	stack: (x: Item.Stack, localTime: Ms, ancestors: ContainerItem[]) => void
-	video: (x: Item.Video, localTime: Ms, ancestors: ContainerItem[]) => void
-	text: (x: Item.Text, localTime: Ms, ancestors: ContainerItem[]) => void
-	audio: (x: Item.Audio, localTime: Ms, ancestors: ContainerItem[]) => void
+	sequence: (x: Item.Sequence, localTime: Ms, ancestors: AncestorAt[]) => void
+	stack: (x: Item.Stack, localTime: Ms, ancestors: AncestorAt[]) => void
+	video: (x: Item.Video, localTime: Ms, ancestors: AncestorAt[]) => void
+	text: (x: Item.Text, localTime: Ms, ancestors: AncestorAt[]) => void
+	audio: (x: Item.Audio, localTime: Ms, ancestors: AncestorAt[]) => void
 }
 
 type WalkCallbacks = {
-	sequence?: (x: Item.Sequence, matrix: Mat6, ancestors: ContainerItem[]) => void
-	stack?: (x: Item.Stack, matrix: Mat6, ancestors: ContainerItem[]) => void
-	video?: (x: Item.Video, matrix: Mat6, ancestors: ContainerItem[]) => void
-	text?: (x: Item.Text, matrix: Mat6, ancestors: ContainerItem[]) => void
+	sequence?: (x: Item.Sequence, matrix: Mat6, ancestors: AncestorAt[]) => void
+	stack?: (x: Item.Stack, matrix: Mat6, ancestors: AncestorAt[]) => void
+	video?: (x: Item.Video, matrix: Mat6, ancestors: AncestorAt[]) => void
+	text?: (x: Item.Text, matrix: Mat6, ancestors: AncestorAt[]) => void
 	audio?: (x: Item.Audio) => void
 }
 
@@ -29,10 +31,15 @@ interface Props {
 	timecode: Ms
 }
 
+export interface AncestorAt {
+	item: ContainerItem
+	localTime: Ms
+}
+
 interface At {
 	item: Item.Any
 	localTime: Ms
-	ancestors: ContainerItem[]
+	ancestors: AncestorAt[]
 }
 
 export function itemsAt(p: Props): At[] {
@@ -72,27 +79,29 @@ export function itemsFrom(p: FromProps): At[] {
 
 export function computeWorldMatrix(
 	items: Map<Id, Item.Any>,
-	ancestors: ContainerItem[],
-	item: Item.Any
+	ancestors: AncestorAt[],
+	item: Item.Any,
+	localTime: Ms,
 ): Mat6 {
 	let world = I6
 
 	for (const ancestor of ancestors) {
-		world = applySpatialIfAny(items, ancestor, world)
+		world = applySpatialIfAny(items, ancestor.item, world, ancestor.localTime)
 	}
 
-	return applySpatialIfAny(items, item, world)
+	return applySpatialIfAny(items, item, world, localTime)
 }
 
 function applySpatialIfAny(
 	items: Map<Id, Item.Any>,
 	item: Item.Any,
-	parentMatrix: Mat6
+	parentMatrix: Mat6,
+	time: Ms,
 ) {
 	if ("spatialId" in item && item.spatialId) {
-		const spatial = items.get(item.spatialId) as Item.Spatial | undefined
+		const spatial = items.get(item.spatialId) as SpatialItem | undefined
 		if (spatial?.enabled) {
-			const local = transformToMat6(spatial.transform)
+			const local = transformToMat6(resolveSpatialTransform(spatial, time))
 			return mul6(local, parentMatrix)
 		}
 	}
@@ -103,27 +112,20 @@ export function walk(
 	id: Id,
 	items: Map<Id, Item.Any>,
 	parentMatrix: Mat6,
+	localTime: Ms,
 	callbacks: WalkCallbacks,
-	ancestors: ContainerItem[] = []
+	ancestors: AncestorAt[] = []
 ) {
 	const item = items.get(id)
 	if (!item) return
 
-	let currentMatrix = parentMatrix
-
-	if ("spatialId" in item && item.spatialId) {
-		const spatial = items.get(item.spatialId) as Item.Spatial
-		if (spatial.enabled) {
-			const local = transformToMat6(spatial.transform)
-			currentMatrix = mul6(local, currentMatrix)
-		}
-	}
+	const currentMatrix = applySpatialIfAny(items, item, parentMatrix, localTime)
 
 	switch (item.kind) {
 		case Kind.Stack:
 			callbacks.stack?.(item, currentMatrix, ancestors)
 			for (const childId of item.childrenIds) {
-				walk(childId, items, currentMatrix, callbacks, [...ancestors, item])
+				walk(childId, items, currentMatrix, localTime, callbacks, [...ancestors, {item, localTime}])
 			}
 			break
 
@@ -143,8 +145,9 @@ export function walk(
 					childId,
 					items,
 					currentMatrix,
+					localTime,
 					callbacks,
-					[...ancestors, item]
+					[...ancestors, {item, localTime}]
 				)
 			}
 
@@ -171,7 +174,7 @@ function walkAt(
 	items: Map<Id, Item.Any>,
 	time: Ms,
 	callbacks: WalkAtCallbacks,
-	ancestors: ContainerItem[] = []
+	ancestors: AncestorAt[] = []
 ) {
 	const item = items.get(id)
 	if (!item) return
@@ -180,7 +183,7 @@ function walkAt(
 		case Kind.Stack:
 			callbacks.stack(item, time, ancestors)
 			for (const childId of item.childrenIds) {
-				walkAt(childId, items, time, callbacks, [...ancestors, item])
+				walkAt(childId, items, time, callbacks, [...ancestors, {item, localTime: time}])
 			}
 			break
 
@@ -205,7 +208,7 @@ function walkAt(
 						items,
 						localTime,
 						callbacks,
-						[...ancestors, item]
+						[...ancestors, {item, localTime: time}]
 					)
 					break
 				}
@@ -235,7 +238,7 @@ function walkFrom(
 	items: Map<Id, Item.Any>,
 	from: Ms,
 	callbacks: WalkAtCallbacks,
-	ancestors: ContainerItem[] = []
+	ancestors: AncestorAt[] = []
 ) {
 	const item = items.get(id)
 	if (!item) return
@@ -244,7 +247,7 @@ function walkFrom(
 		case Kind.Stack:
 			callbacks.stack(item, from, ancestors)
 			for (const childId of item.childrenIds) {
-				walkFrom(childId, items, from, callbacks, [...ancestors, item])
+				walkFrom(childId, items, from, callbacks, [...ancestors, {item, localTime: from}])
 			}
 			break
 
@@ -274,7 +277,7 @@ function walkFrom(
 					items,
 					localTime,
 					callbacks,
-					[...ancestors, item]
+					[...ancestors, {item, localTime: from}]
 				)
 
 				offset = end
@@ -358,3 +361,17 @@ export function computeItemDuration(
 	}
 }
 
+export function computeOpacity(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+) {
+	if (!("animationId" in item) || item.animationId === undefined)
+		return 1
+
+	const animation = ctx.items.get(item.animationId) as Item.Animation | undefined
+	return animation?.enabled && animation.anims.opacity
+		? resolveScalarAnimation(time, animation.anims.opacity)
+		: 1
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/sample.ts

@@ -2,19 +2,20 @@
 import {SampleContext} from "./types.js"
 import {sampleSequence} from "./sequence.js"
 import {Ms} from "../../../../../../units/ms.js"
-import {computeWorldMatrix} from "../../../handy.js"
+import {Item, Kind, SpatialItem} from "../../../../../parts/item.js"
 import {FilterSpec, Layer} from "../../../../../../driver/fns/schematic.js"
-import {ContainerItem, Item, Kind} from "../../../../../parts/item.js"
+import {AncestorAt, computeOpacity, computeWorldMatrix} from "../../../handy.js"
 
 export async function sampleVisual(
 	ctx: SampleContext,
 	item: Item.Any,
 	time: Ms,
-	ancestors: ContainerItem[]
+	ancestors: AncestorAt[]
 ): Promise<Layer[]> {
-	const matrix = computeWorldMatrix(ctx.items, ancestors, item)
+	const matrix = computeWorldMatrix(ctx.items, ancestors, item, time)
+	const alpha = computeOpacity(ctx, item, time)
 	const crop = "spatialId" in item && item.spatialId
-		? (ctx.items.get(item.spatialId) as Item.Spatial | undefined)?.crop
+		? (ctx.items.get(item.spatialId) as SpatialItem | undefined)?.crop
 		: undefined
 	const filters = "filterIds" in item && item.filterIds
 		? item.filterIds
@@ -25,7 +26,7 @@ export async function sampleVisual(
 
 	switch (item.kind) {
 		case Kind.Stack: {
-			const nextAnc = [...ancestors, item]
+			const nextAnc = [...ancestors, {item, localTime: time}]
 
 			const layers = await Promise.all(
 				item.childrenIds
@@ -44,7 +45,7 @@ export async function sampleVisual(
 			if (time < 0 || time >= item.duration) return []
 
 			const frame = await ctx.videoSampler(item, time)
-			return frame ? [{kind: "image", frame, matrix, crop, filters, id: item.id}] : []
+			return frame ? [{kind: "image", frame, matrix, alpha, crop, filters, id: item.id}] : []
 		}
 
 		case Kind.Text: {
@@ -54,7 +55,7 @@ export async function sampleVisual(
 				? (ctx.items.get(item.styleId) as Item.TextStyle)?.style
 				: undefined
 
-			return [{id: item.id, kind: "text", content: item.content, style, matrix, crop, filters}]
+			return [{id: item.id, kind: "text", content: item.content, style, matrix, alpha, crop, filters}]
 		}
 
 		case Kind.Gap: {

package/s/timeline/renderers/parts/samplers/visual/parts/sequence.ts

@@ -3,21 +3,21 @@ import {sampleVisual} from "./sample.js"
 import {SampleContext} from "./types.js"
 import {sampleTransition} from "./transition.js"
 import {ms, Ms} from "../../../../../../units/ms.js"
-import {computeItemDuration} from "../../../handy.js"
+import {Item, Kind} from "../../../../../parts/item.js"
 import {Layer} from "../../../../../../driver/fns/schematic.js"
-import {ContainerItem, Item, Kind} from "../../../../../parts/item.js"
+import {AncestorAt, computeItemDuration} from "../../../handy.js"
 
 export async function sampleSequence(
 	ctx: SampleContext,
 	seq: Item.Sequence,
 	time: Ms,
-	ancestors: ContainerItem[]
+	ancestors: AncestorAt[]
 ): Promise<Layer[]> {
 	const state = sampleSequenceAt(ctx, seq, time)
 
 	if (!state) return []
 
-	const nextAnc = [...ancestors, seq]
+	const nextAnc = [...ancestors, {item: seq, localTime: time}]
 
 	if (!state.isTransitioning) {
 		return sampleVisual(ctx, state.item, state.localTime, nextAnc)