@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/timeline/sugar/o.ts

@@ -1,60 +1,455 @@
+import {hex} from "@e280/stz"
+import {TextStyleOptions} from "pixi.js"
 
-import {MapG} from "@e280/stz"
-import {Id} from "../parts/basics.js"
 import {Media} from "../parts/media.js"
-import {Effect, Item, Kind} from "../parts/item.js"
+import {Id, TimelineFile} from "../parts/basics.js"
+import {FilterAction, FilterActions} from "../parts/filters.js"
+import {filters, FilterParams, FilterType} from "../parts/filters.js"
+import {Transcription} from "../../features/speech/transcribe/types.js"
+import {Crop, FilterableItem, Item, Kind, VisualAnimatableItem} from "../parts/item.js"
+import {animationPresets, makeAnimationPresets, visualAnimations} from "../parts/animations/registry.js"
+import {TransitionAction, TransitionActions, transitions, TransitionName} from "../parts/transitions.js"
+import {AnimationPreset, PresetAnimateAction, PresetAnimateActions, PresetAnimation, PresetOptions} from "../parts/animations/types.js"
+import {CaptionAction, CaptionActions, captionDuration, CaptionOptions, CaptionPreset, captionPresets, CaptionSourceItem} from "../parts/captions.js"
+import {Anim, AnimateAction, Interpolation, Keyframes, ScalarAnimation, TrackTransform, Transform, TransformAnimation, TransformOptions, Vec2, VisualAnimationInput, VisualAnimationValue, VisualAnimations} from "../types.js"
+
+type VisualAnimateActions = {
+	[TKey in keyof VisualAnimations]-?: AnimateAction<TKey>
+}
 
 export class O {
-	#nextId = 0
-	#items = new MapG<Id, Item.Any>()
+	constructor(public state: {timeline: TimelineFile}) {}
+
+	require<T extends Item.Any>(id: Id | undefined) {
+    if (id === undefined)
+    	return undefined
+    const item = this.state.timeline.items.find(item => item.id === id)
+    return item as T | undefined
+	}
+
+	get timeline() {
+		return this.state.timeline
+	}
 
-	#getId() {
-		return this.#nextId++
+	getId() {
+		return hex.toInteger(hex.random())
 	}
 
+  #mutate(fn: (project: TimelineFile) => TimelineFile) {
+    this.state.timeline = fn(this.state.timeline)
+  }
+
 	register(item: Item.Any) {
-		if (!this.#items.has(item.id))
-			this.#items.set(item.id, item)
-		return item.id
-	}
-
-	get items() {
-		return [...this.#items.values()]
-	}
-
-	sequence = (...items: Item.Any[]): Item.Sequence => ({
-		id: this.#getId(),
-		kind: Kind.Sequence,
-		children: items.map(item => this.register(item)),
-	})
-
-	stack = (...items: Item.Any[]): Item.Stack => ({
-		id: this.#getId(),
-		kind: Kind.Stack,
-		children: items.map(item => this.register(item)),
-	})
-
-	clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
-		id: this.#getId(),
-		kind: Kind.Clip,
-		mediaHash: media.datafile.checksum.hash,
-		start: start ?? 0,
-		duration: duration ?? media.duration,
-	})
-
-	text = (content: string): Item.Text => ({
-		id: this.#getId(),
-		kind: Kind.Text,
-		content,
-	})
-
-	transition = {
-		crossfade: (duration: number): Item.Transition => ({
-			id: this.#getId(),
-			kind: Kind.Transition,
-			effect: Effect.Crossfade,
+		this.#mutate(state => ({
+			...state,
+			items: [...state.items, item]
+		}))
+	}
+
+  textStyle = (style: TextStyleOptions): Item.TextStyle => {
+		const item = {
+			id: this.getId(),
+			kind: Kind.TextStyle,
+			style
+		} as Item.TextStyle
+		this.register(item)
+		return item
+  }
+
+  spatial = (transform?: Transform, crop?: Crop): Item.Spatial => {
+  	const item: Item.Spatial = {
+  		id: this.getId(),
+  		kind: Kind.Spatial,
+  		transform: transform ?? this.transform(),
+  		crop,
+  		enabled: true
+  	}
+		this.register(item)
+  	return item
+  }
+
+	#registerAnimation = (anims: VisualAnimations): Item.Animation => {
+		const item: Item.Animation = {
+			id: this.getId(),
+			kind: Kind.Animation,
+			anims,
+			enabled: true
+		}
+		this.register(item)
+		return item
+	}
+
+	#attachAnimation = <T extends VisualAnimatableItem>(item: T, animation: Item.Animation): T => {
+		const next = {
+			...item,
+			animationIds: [...(item.animationIds ?? []), animation.id]
+		}
+		this.set<T>(item.id, next as Partial<T>)
+		return next
+	}
+
+	anim = {
+		scalar: (terp: Interpolation, track: Keyframes): Anim<Keyframes> => ({terp, track}),
+
+		vec2: (terp: Interpolation, source: Keyframes<Vec2>) => {
+			const track = {x: [] as Keyframes, y: [] as Keyframes}
+
+			for (const [time, [x, y]] of source) {
+				track.x.push([time, x])
+				track.y.push([time, y])
+			}
+
+			return {terp, track}
+		},
+
+		transform: (terp: Interpolation, source: Keyframes<Transform>): Anim<TrackTransform> => {
+			const track: TrackTransform = {
+				position: {x: [], y: []},
+				scale: {x: [], y: []},
+				rotation: [],
+			}
+
+			for (const [time, [position, scale, rotation]] of source) {
+				track.position.x.push([time, position[0]])
+				track.position.y.push([time, position[1]])
+				track.scale.x.push([time, scale[0]])
+				track.scale.y.push([time, scale[1]])
+				track.rotation.push([time, rotation])
+			}
+
+			return {terp, track}
+		},
+	}
+
+	#animationPresets = makeAnimationPresets(
+		(terp, track) => ({terp, track}),
+		(terp, source) => this.anim.transform(terp, source),
+		options => this.transform(options),
+	)
+
+	#makeFilter = <TFilter extends FilterType>(type: TFilter) => {
+		const make = (params?: FilterParams<TFilter>) => {
+			const item: Item.Filter<TFilter> = {
+				id: this.getId(),
+				kind: Kind.Filter,
+				type,
+				params,
+				enabled: true
+			}
+			this.register(item)
+			return item
+		}
+
+		const action = (<T extends FilterableItem>(item: T, params?: FilterParams<TFilter>): T => {
+			const filter = make(params)
+			const next = {
+				...item,
+				filterIds: [...(item.filterIds ?? []), filter.id]
+			}
+			this.set<T>(item.id, next as Partial<T>)
+			return next
+		}) as FilterAction<TFilter>
+
+		action.make = make
+		return action
+	}
+
+	#makeFilters = (): FilterActions => {
+		const entries = Object.entries(filters)
+			.map(([name, filter]) => [name, this.#makeFilter(filter.type)])
+		return Object.fromEntries(entries) as FilterActions
+	}
+
+	filter = this.#makeFilters()
+
+	#makeAnimationValue = <TKey extends keyof VisualAnimations>(
+		key: TKey,
+		terp: Interpolation,
+		track: VisualAnimationInput<TKey>
+	): VisualAnimationValue<TKey> =>
+		(
+			key === "transform"
+				? this.anim.transform(terp, track as Keyframes<Transform>)
+				: this.anim.scalar(terp, track as Keyframes)
+		) as VisualAnimationValue<TKey>
+
+	#makeAnimate = <TKey extends keyof VisualAnimations>(key: TKey): AnimateAction<TKey> => {
+		const make = (terp: Interpolation, track: VisualAnimationInput<TKey>) =>
+			this.#registerAnimation({
+				[key]: this.#makeAnimationValue(key, terp, track)
+			} as Pick<VisualAnimations, TKey>)
+
+		const action = (<T extends VisualAnimatableItem>(
+			item: T,
+			terp: Interpolation,
+			track: VisualAnimationInput<TKey>
+		): T => {
+			const animation = make(terp, track)
+			return this.#attachAnimation(item, animation)
+		}) as AnimateAction<TKey>
+
+		action.make = make
+		return action
+	}
+
+	#makePresetAnimate = <TKey extends AnimationPreset>(key: TKey): PresetAnimateAction => {
+		const make = (options?: PresetOptions) => {
+			const preset = animationPresets[key]
+			const anim = this.#animationPresets[key](options as never) as PresetAnimation
+			return this.#registerAnimation(
+				preset.type === "motion"
+					? {transform: anim as TransformAnimation}
+					: {opacity: anim as ScalarAnimation}
+			)
+		}
+
+		const action = (<T extends VisualAnimatableItem>(item: T, options?: PresetOptions): T => {
+			const animation = make(options)
+			return this.#attachAnimation(item, animation)
+		}) as PresetAnimateAction
+
+		action.make = make
+		return action
+	}
+
+	#makeAnimateActions = (): VisualAnimateActions => {
+		const entries = Object.keys(visualAnimations)
+			.map(key => [key, this.#makeAnimate(key as keyof VisualAnimations)])
+		return Object.fromEntries(entries) as VisualAnimateActions
+	}
+
+	#makePresetAnimateActions = (): PresetAnimateActions => {
+		const entries = Object.keys(animationPresets)
+			.map(key => [key, this.#makePresetAnimate(key as AnimationPreset)])
+		return Object.fromEntries(entries) as PresetAnimateActions
+	}
+
+	animate = {
+		...this.#makeAnimateActions(),
+		presets: this.#makePresetAnimateActions(),
+	}
+
+	sequence = (...items: Item.Any[]): Item.Sequence => {
+		const item =  {
+			id: this.getId(),
+			kind: Kind.Sequence,
+			childrenIds: items.map(item => item.id)
+		} as Item.Sequence
+		this.register(item)
+		return item
+	}
+
+	stack = (...items: Item.Any[]): Item.Stack => {
+		const item = {
+			kind: Kind.Stack,
+			id: this.getId(),
+			childrenIds: items.map(item => item.id)
+		} as Item.Stack
+		this.register(item)
+		return item
+	}
+
+	video = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Video => {
+
+		if(!media.hasVideo)
+			throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
+
+		const item: Item.Video = {
+			kind: Kind.Video,
+			id: this.getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+		this.register(item)
+		return item
+	}
+
+	image = (
+		media: Media,
+		options?: {
+			duration?: number
+		}): Item.Image => {
+
+		if(!media.isImage)
+			throw new Error(`Image error: media "${media.datafile.filename}" is not an image.`)
+
+		const item: Item.Image = {
+			kind: Kind.Image,
+			id: this.getId(),
+			mediaHash: media.datafile.checksum.hash,
+			duration: options?.duration ?? 2000
+		}
+		this.register(item)
+		return item
+	}
+
+	audio = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number,
+			gain?: number
+		}): Item.Audio => {
+
+		if(!media.hasAudio)
+			throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
+
+		const item: Item.Audio = {
+			kind: Kind.Audio,
+			id: this.getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration,
+			gain: options?.gain ?? 1
+		}
+		this.register(item)
+		return item
+	}
+
+	text = (content: string, options?: {
+			duration?: number,
+			styles?: TextStyleOptions
+		}): Item.Text => {
+
+		const item = {
+			id: this.getId(),
+			content,
+			kind: Kind.Text,
+			duration: options?.duration ?? 2000
+		} as Item.Text
+
+		if(options?.styles)
+			item.styleId = this.textStyle(options.styles).id
+
+		this.register(item)
+		return item
+	}
+
+	#makeCaption = (
+		transcript: Transcription,
+		preset: CaptionPreset,
+		options?: CaptionOptions,
+	): Item.Caption => {
+		const start = options?.start ?? 0
+		const duration = options?.duration ?? Math.max(0, captionDuration(transcript, options) - start)
+		const item: Item.Caption = {
+			id: this.getId(),
+			kind: Kind.Caption,
+			transcript,
+			itemId: options?.itemId,
+			start,
 			duration,
-		}),
+			maxChars: options?.maxChars,
+			maxDuration: options?.maxDuration,
+			maxSilence: options?.maxSilence,
+		}
+
+		item.styleId = this.textStyle(options?.styles ?? preset.styles).id
+		item.spatialId = this.spatial(this.transform(preset.transform)).id
+
+		this.register(item)
+		return item
+	}
+
+	#makeCaptionAction = (preset: CaptionPreset): CaptionAction => {
+		const make = (transcript: Transcription, options?: CaptionOptions) =>
+			this.#makeCaption(transcript, preset, options)
+
+		const action = ((item: CaptionSourceItem, transcript: Transcription, options?: CaptionOptions): Item.Stack => {
+			const caption = make(transcript, {
+				...options,
+				itemId: item.id,
+				start: options?.start ?? item.start,
+				duration: options?.duration ?? item.duration,
+			})
+			return this.stack(caption, item)
+		}) as CaptionAction
+
+		action.make = make
+		return action
+	}
+
+	#makeCaptionPresetActions = () => {
+		const entries = Object.entries(captionPresets)
+			.map(([name, preset]) => [name, this.#makeCaptionAction(preset)])
+		return Object.fromEntries(entries) as CaptionActions["presets"]
+	}
+
+	captions = Object.assign(
+		this.#makeCaptionAction(captionPresets.default),
+		{presets: this.#makeCaptionPresetActions()}
+	) as CaptionActions
+
+	gap = (duration: number): Item.Gap => {
+		const item = {
+			id: this.getId(),
+			kind: Kind.Gap,
+			duration
+		} as Item.Gap
+		this.register(item)
+		return item
+	}
+
+	#makeTransition = (key: TransitionName): TransitionAction => {
+		return (duration: number): Item.Transition => {
+			const item: Item.Transition = {
+				id: this.getId(),
+				kind: Kind.Transition,
+				name: transitions[key].name,
+				duration,
+			}
+			this.register(item)
+			return item
+		}
+	}
+
+	#makeTransitions = (): TransitionActions => {
+		const entries = Object.keys(transitions)
+			.map(key => [key, this.#makeTransition(key as TransitionName)])
+		return Object.fromEntries(entries) as TransitionActions
+	}
+
+	transition = this.#makeTransitions()
+
+  transform = (options?: TransformOptions): Transform => {
+    const position: Vec2 = [
+    	options?.position?.[0] ?? 0,
+    	options?.position?.[1] ?? 0
+    ]
+    const scale: Vec2 = [
+    	options?.scale?.[0] ?? 1,
+    	options?.scale?.[1] ?? 1
+    ]
+    const rotation = options?.rotation ?? 0
+    return [position, scale, rotation]
+  }
+
+  addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
+		this.#mutate(state => {
+			const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
+			parentItem.childrenIds.push(...items.map(item => item.id))
+			return state
+		})
+  }
+
+	set = <K extends Item.Any>(
+		id: Id,
+		partial: Partial<K>
+	) => {
+		this.#mutate(project => ({
+			...project,
+			items: project.items.map(item =>
+				item.id === id
+					? { ...item, ...partial }
+					: item
+			)
+		}))
 	}
 }
 

package/s/timeline/sugar/omni.test.ts

@@ -0,0 +1,210 @@
+
+import {expect, Science, test} from "@e280/science"
+
+import {O} from "./o.js"
+import {Omni} from "./omni.js"
+import {Item} from "../parts/item.js"
+import {Driver} from "../../driver/driver.js"
+import {Datafile} from "../utils/datafile.js"
+import {loadVideo} from "../../demo/routines/load-video.js"
+import {computeItemDuration} from "../renderers/parts/handy.js"
+
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
+export async function setupTest() {
+	const driver = await Driver.setup({workerUrl})
+	const omni = new Omni(driver)
+
+	const testVideo = await loadVideo("/assets/temp/test.mp4")
+	const {videoA} = await omni.load({videoA: Datafile.make(testVideo, "test.mp4")})
+
+	return {driver, omni, testVideo, videoA}
+}
+
+export default Science.suite({
+	"basic demo": test(async () => {
+		const driver = await Driver.setup({workerUrl: new URL("../driver/driver.worker.bundle.min.js", import.meta.url)})
+		const omni = new Omni(driver)
+
+		const testVideo = await loadVideo("/assets/temp/test.mp4")
+		const {videoA} = await omni.load({videoA: Datafile.make(testVideo, "test.mp4")})
+
+		const o = new O({timeline: omni.timeline(o => o.sequence())})
+		const rootItem = o.require<Item.Sequence>(o.timeline.rootId)!
+
+		const video = o.video(videoA)
+		o.addChildren(rootItem, video)
+
+		const videoDuration = computeItemDuration(video.id, o.timeline)
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+
+		expect(videoDuration).is(timelineDuration)
+	}),
+
+	"sequence duration is 3x": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA),
+			o.video(videoA),
+			o.video(videoA),
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration * 3)
+	}),
+
+	"stack duration is 1x": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.stack(
+			o.video(videoA),
+			o.video(videoA),
+			o.video(videoA),
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration)
+	}),
+
+	"nested stack inside sequence": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA),
+			o.stack(
+				o.video(videoA),
+				o.video(videoA)
+			),
+			o.video(videoA),
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration * 3)
+	}),
+
+	"nested sequence inside stack": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.stack(
+			o.video(videoA),
+			o.sequence(
+				o.video(videoA),
+				o.video(videoA)
+			)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration * 2)
+	}),
+
+	"empty timeline duration is 0": test(async () => {
+		const {omni} = await setupTest()
+		const o1 = new O({timeline: omni.timeline(o => o.sequence())})
+		const o2 = new O({timeline: omni.timeline(o => o.stack())})
+		const duration1 = computeItemDuration(o1.timeline.rootId, o1.timeline)
+		const duration2 = computeItemDuration(o2.timeline.rootId, o2.timeline)
+		expect(duration1).is(0)
+		expect(duration2).is(0)
+	}),
+
+	"transition in a sequence": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const transitionDuration = 1000
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA),
+			o.transition.fade(transitionDuration),
+			o.video(videoA)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		const expectedDuration = (videoA.duration * 2) + transitionDuration
+		expect(timelineDuration).is(expectedDuration)
+	}),
+
+	"transition in a stack": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const transitionDuration = 1000
+		const o = new O({timeline: omni.timeline(o => o.stack(
+			o.video(videoA),
+			o.transition.fade(transitionDuration),
+			o.video(videoA)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration)
+	}),
+
+	"edge transition takes sequence space": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.transition.fade(1000),
+			o.video(videoA),
+			o.video(videoA),
+			o.transition.fade(1000),
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration * 2 + 2000)
+	}),
+
+	"transition duration can be longer than neighboring clips": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const duration = 3000
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration}),
+			o.transition.fade(duration * 2),
+			o.video(videoA, {duration})
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(duration * 4)
+	}),
+
+	"multiple transitions": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const transitionDuration = 1000
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA),
+			o.transition.fade(transitionDuration),
+			o.video(videoA),
+			o.transition.fade(transitionDuration),
+			o.video(videoA)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		const expected = (videoA.duration * 3) + (transitionDuration * 2)
+		expect(timelineDuration).is(expected)
+	}),
+
+	"gap item in a sequence": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const gapDuration = 1000
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA),
+			o.gap(gapDuration),
+			o.video(videoA)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		const expected = videoA.duration * 2 + gapDuration
+		expect(timelineDuration).is(expected)
+	}),
+
+	"gap item in a stack": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.stack(
+			o.video(videoA),
+			o.gap(1000),
+			o.video(videoA)
+		))})
+		const timelineDuration = computeItemDuration(o.timeline.rootId, o.timeline)
+		expect(timelineDuration).is(videoA.duration)
+	}),
+
+	"load media into Omni": test(async () => {
+		const driver = await Driver.setup({workerUrl})
+		const omni = new Omni(driver)
+		const testVideo = await loadVideo("/assets/temp/test.mp4")
+		const {videoA} = await omni.load({videoA: Datafile.make(testVideo, "test.mp4")})
+		expect(omni.resources.require(videoA.datafile.checksum.hash)).happy()
+	}),
+
+	"playback seeks to correct time": test(async () => {
+		const {omni, videoA} = await setupTest()
+		const o = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2500})
+		))})
+		const playback = await omni.playback(o.timeline)
+		await playback.seek(1000)
+		expect(playback.currentTime).is(1000)
+	})
+})
+