@omnimedia/omnitool 1.1.0-8 → 1.1.0-80

package/s/timeline/renderers/renderers.test.ts

@@ -0,0 +1,385 @@
+
+import {Science, test, expect} from "@e280/science"
+
+import {O} from "../sugar/o.js"
+import {ms} from "../../units/ms.js"
+import {Omni} from "../sugar/omni.js"
+import {fps} from "../../units/fps.js"
+import {Datafile} from "../utils/datafile.js"
+import {Driver} from "../../driver/driver.js"
+import {AudioMix} from "./export/parts/audio-mix.js"
+import {produceAudio} from "./export/parts/produce-audio.js"
+import {resampleToPlanar} from "./export/parts/resamplers.js"
+import {loadVideo} from "../../demo/routines/load-video.js"
+import {produceVideo} from "./export/parts/produce-video.js"
+import {CursorVisualSampler} from "./export/parts/cursor.js"
+import {applyGainToPlanar} from "./export/parts/audio-gain.js"
+import {createVisualSampler} from "./parts/samplers/visual/sampler.js"
+
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
+export async function setupTest() {
+	const driver = await Driver.setup({workerUrl})
+	const omni = new Omni(driver)
+
+	const testVideo = await loadVideo("/assets/temp/test.mp4")
+	const {videoA} = await omni.load({videoA: Datafile.make(testVideo, "test.mp4")})
+
+	const resolveMedia = (hash: string) => omni.resources.require(hash).blob
+
+	return {driver, omni, testVideo, videoA, resolveMedia}
+}
+
+async function collect<T>(iterable: AsyncIterable<T>) {
+	const out: T[] = []
+	for await (const item of iterable)
+		out.push(item)
+	return out
+}
+
+const near = (actual: number, expected: number, eps = 1e-6) =>
+	Math.abs(actual - expected) < eps
+
+export default Science.suite({
+
+	"cursor visual sampler cannot get previous samples": test(async () => {
+		const {omni, videoA, resolveMedia, driver} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500}),
+		))})
+		const cursor = new CursorVisualSampler(driver, resolveMedia, timeline)
+		await cursor.next(ms(1000))
+		await expect(async () => await cursor.next(ms(100))).throwsAsync()
+	}),
+
+	"visual sampler gives correct layer at x time": test(async () => {
+		const {omni, videoA, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500}),
+			o.text("123", {duration: 1000})
+		))})
+		const sampler = createVisualSampler(resolveMedia)
+		const imgLayer = await sampler.sample(timeline, ms(1000))
+		expect(imgLayer[0].kind).is("image")
+		const gapLayer = await sampler.sample(timeline, ms(2300))
+		expect(gapLayer[0].kind).is("gap")
+		const imgLayer1 = await sampler.sample(timeline, ms(2700))
+		expect(imgLayer1[0].kind).is("image")
+		const textLayer = await sampler.sample(timeline, ms(5500))
+		expect(textLayer[0].kind).is("text")
+	}),
+
+	"audio mix sums overlapping chunks": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([0.25, 0.25, 0.25, 0.25])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([0.5, 0.5, 0.5, 0.5])],
+				sampleRate: 4,
+				timestamp: 0.5
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(2)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].frames).is(4)
+		expect(mixed[0].channels).is(1)
+		expect(mixed[0].planar[0]).is(0.25)
+		expect(mixed[0].planar[1]).is(0.25)
+		expect(mixed[0].planar[2]).is(0.75)
+		expect(mixed[0].planar[3]).is(0.75)
+		expect(mixed[1].startFrame).is(4)
+		expect(mixed[1].planar[0]).is(0.5)
+		expect(mixed[1].planar[1]).is(0.5)
+		expect(mixed[1].planar[2]).is(0)
+		expect(mixed[1].planar[3]).is(0)
+	}),
+
+	"audio mix clamps output to [-1, 1]": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: true})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([0.9, 0.9, 0.9, 0.9])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([0.8, 0.8, 0.8, 0.8])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].planar[0]).is(1)
+		expect(mixed[0].planar[1]).is(1)
+		expect(mixed[0].planar[2]).is(1)
+		expect(mixed[0].planar[3]).is(1)
+	}),
+
+	"audio mix fills gaps with silence": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([1, 1, 1, 1])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([2, 2, 2, 2])],
+				sampleRate: 4,
+				timestamp: 2
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(3)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].planar[0]).is(1)
+
+		expect(mixed[1].startFrame).is(4)
+		expect(mixed[1].planar[0]).is(0)
+		expect(mixed[1].planar[3]).is(0)
+
+		expect(mixed[2].startFrame).is(8)
+		expect(mixed[2].planar[0]).is(2)
+	}),
+
+	"audio mix truncates negative timestamps": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([1, 2, 3, 4])],
+				sampleRate: 4,
+				timestamp: -0.5
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].planar[0]).is(3)
+		expect(mixed[0].planar[1]).is(4)
+		expect(mixed[0].planar[2]).is(0)
+		expect(mixed[0].planar[3]).is(0)
+	}),
+
+	"audio mix handles stereo planar layout": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [
+					new Float32Array([1, 1, 1, 1]),
+					new Float32Array([2, 2, 2, 2])
+				],
+				sampleRate: 4,
+				timestamp: 0
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].channels).is(2)
+		expect(mixed[0].planar.length).is(8)
+		expect(mixed[0].planar[0]).is(1)
+		expect(mixed[0].planar[4]).is(2)
+	}),
+
+	"applyGainToPlanar scales values across multiple channels": test(async () => {
+		const planes = [
+			new Float32Array([0.2, -0.4]),
+			new Float32Array([0.6, -0.8])
+		]
+
+		applyGainToPlanar(planes, 0.5)
+
+		expect(near(planes[0][0], 0.1)).ok()
+		expect(near(planes[0][1], -0.2)).ok()
+		expect(near(planes[1][0], 0.3)).ok()
+		expect(near(planes[1][1], -0.4)).ok()
+	}),
+
+	"applyGainToPlanar does not mutate when gain is 1": test(async () => {
+		const planes = [new Float32Array([0.5, -0.5])]
+
+		applyGainToPlanar(planes, 1)
+
+		expect(near(planes[0][0], 0.5)).ok()
+		expect(near(planes[0][1], -0.5)).ok()
+	}),
+
+	"applyGainToPlanar mutes entirely when gain is 0": test(async () => {
+		const planes = [new Float32Array([0.9, -0.9])]
+
+		applyGainToPlanar(planes, 0)
+
+		expect(planes[0][0]).is(0)
+		expect(planes[0][1]).is(0)
+	}),
+
+	"resampleToPlanar returns original data when sample rates match": test(async () => {
+		const sample = {
+			numberOfFrames: 3,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array) => dest.set([0.1, 0.2, 0.3])
+		}
+
+		const result = resampleToPlanar(sample, 48000)
+
+		expect(result.frames).is(3)
+		expect(result.data.length).is(1)
+		expect(near(result.data[0][0], 0.1)).ok()
+		expect(near(result.data[0][1], 0.2)).ok()
+		expect(near(result.data[0][2], 0.3)).ok()
+	}),
+
+	"resampleToPlanar upsamples using linear interpolation": test(async () => {
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 1,
+			sampleRate: 1,
+			copyTo: (dest: Float32Array) => dest.set([10, 20])
+		}
+
+		const result = resampleToPlanar(sample as any, 2)
+
+		expect(result.frames).is(4)
+		expect(near(result.data[0][0], 10)).ok()
+		expect(near(result.data[0][1], 15)).ok()
+		expect(near(result.data[0][2], 20)).ok()
+		expect(near(result.data[0][3], 20)).ok()
+	}),
+
+	"resampleToPlanar downsamples correctly": test(async () => {
+		const sample = {
+			numberOfFrames: 4,
+			numberOfChannels: 1,
+			sampleRate: 4,
+			copyTo: (dest: Float32Array) => dest.set([1, 2, 3, 4])
+		}
+
+		const result = resampleToPlanar(sample, 2)
+
+		expect(result.frames).is(2)
+		expect(near(result.data[0][0], 1)).ok()
+		expect(near(result.data[0][1], 3)).ok()
+	}),
+
+	"resampleToPlanar processes multi-channel planar data": test(async () => {
+		const planes = [
+			new Float32Array([1, 2]),
+			new Float32Array([3, 4])
+		]
+
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 2,
+			sampleRate: 1,
+			copyTo: (dest: Float32Array, options: { planeIndex: number }) => {
+				dest.set(planes[options.planeIndex])
+			}
+		}
+
+		const result = resampleToPlanar(sample, 2)
+
+		expect(result.data.length).is(2)
+		expect(near(result.data[0][1], 1.5)).ok()
+		expect(near(result.data[1][1], 3.5)).ok()
+	}),
+
+	"resampleToPlanar handles zero channels gracefully": test(async () => {
+		const sample = {
+			numberOfFrames: 10,
+			numberOfChannels: 0,
+			sampleRate: 48000,
+			copyTo: () => { throw new Error("Should not be called") }
+		}
+
+		const result = resampleToPlanar(sample, 44100)
+
+		expect(result.data.length).is(0)
+		expect(result.frames).is(0)
+	}),
+
+	"resampleToPlanar enforces minimum of 1 frame on aggressive downsample": test(async () => {
+		const sample = {
+			numberOfFrames: 1,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array) => dest.set([0.8])
+		}
+
+		const result = resampleToPlanar(sample, 8000)
+
+		expect(result.frames).is(1)
+		expect(result.data[0].length).is(1)
+		expect(near(result.data[0][0], 0.8)).ok()
+	}),
+
+	"resampleToPlanar calls copyTo with strict contract shape": test(async () => {
+		let passedOptions: any = null
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array, options: any) => {
+				passedOptions = options
+			}
+		}
+
+		resampleToPlanar(sample, 44100)
+
+		expect(passedOptions).not.is(null)
+		expect(passedOptions.planeIndex).is(0)
+		expect(passedOptions.format).is("f32-planar")
+	}),
+
+	"5s long export at 30fps renders exacly 150 frames": test(async () => {
+		const {omni, videoA, driver, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500})
+		))})
+		const readable = produceVideo({timeline, fps: fps(30), driver, resolveMedia})
+		let frames = 0
+		for await (const frame of readable) {
+			frame.close()
+			frames++
+		}
+		expect(frames).is(150)
+	}),
+
+	"5s long audio export at 48000Hz renders exactly 240000 frames": test(async () => {
+		const {omni, videoA, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500})
+		))})
+
+		const readable = produceAudio({timeline, resolveMedia})
+		let totalFrames = 0
+
+		for await (const chunk of readable) {
+			totalFrames += chunk.numberOfFrames
+			chunk.close()
+		}
+
+		expect(totalFrames).is(240000)
+	})
+})

package/s/timeline/sugar/helpers.ts

@@ -0,0 +1,190 @@
+
+import {TextStyleOptions} from "pixi.js"
+
+import {O} from "./o.js"
+import {Media} from "../parts/media.js"
+import {TimelineFile} from "../parts/basics.js"
+import {FilterAction} from "../parts/filters.js"
+import {visualAnimations} from "../parts/animations.js"
+import {filters, FilterParams, FilterType} from "../parts/filters.js"
+import {Crop, FilterableItem, Item, VisualAnimatableItem} from "../parts/item.js"
+import {Anim, AnimateAction, Interpolation, Keyframes, TrackTransform, Transform, Vec2, VisualAnimations} from "../types.js"
+
+export type Build<T extends Item.Any = Item.Any> = (o: O) => T
+type BuildVisualAnimateActions = {
+	[TKey in keyof VisualAnimations]-?: BuildAnimateAction
+}
+
+function createTimeline(): TimelineFile {
+	return {
+		format: "timeline",
+		info: "https://omniclip.app/",
+		version: 0,
+		items: [],
+		rootId: 0
+	}
+}
+
+export function timeline(root: Build): TimelineFile {
+	const o = new O({timeline: createTimeline()})
+	const item = root(o)
+	o.timeline.rootId = item.id
+	return o.timeline
+}
+
+export function sequence(...items: Build[]): Build<Item.Sequence> {
+	return o => o.sequence(...items.map(item => item(o)))
+}
+
+export function stack(...items: Build[]): Build<Item.Stack> {
+	return o => o.stack(...items.map(item => item(o)))
+}
+
+export function video(
+	media: Media,
+	options?: {
+		start?: number,
+		duration?: number
+	}
+): Build<Item.Video> {
+	return o => o.video(media, options)
+}
+
+export function audio(
+	media: Media,
+	options?: {
+		start?: number,
+		duration?: number,
+		gain?: number
+	}
+): Build<Item.Audio> {
+	return o => o.audio(media, options)
+}
+
+export function text(
+	content: string,
+	options?: {
+		duration?: number,
+		styles?: TextStyleOptions
+	}
+): Build<Item.Text> {
+	return o => o.text(content, options)
+}
+
+export function gap(duration: number): Build<Item.Gap> {
+	return o => o.gap(duration)
+}
+
+export function spatial(transform?: Transform, crop?: Crop): Build<Item.Spatial> {
+	return o => o.spatial(transform, crop)
+}
+
+export function animatedSpatial(anim: Anim<TrackTransform>, crop?: Crop): Build<Item.AnimatedSpatial> {
+	return o => o.animatedSpatial(anim, crop)
+}
+
+export const anim = {
+	scalar(terp: Interpolation, track: Keyframes): Anim<Keyframes> {
+		return {terp, track}
+	},
+
+	vec2(terp: Interpolation, source: Keyframes<Vec2>): Anim<{x: Keyframes, y: Keyframes}> {
+		const track = {x: [] as Keyframes, y: [] as Keyframes}
+
+		for (const [time, [x, y]] of source) {
+			track.x.push([time, x])
+			track.y.push([time, y])
+		}
+
+		return {terp, track}
+	},
+
+	transform(terp: Interpolation, source: Keyframes<Transform>): Anim<TrackTransform> {
+		const track: TrackTransform = {
+			position: {x: [], y: []},
+			scale: {x: [], y: []},
+			rotation: [],
+		}
+
+		for (const [time, [position, scale, rotation]] of source) {
+			track.position.x.push([time, position[0]])
+			track.position.y.push([time, position[1]])
+			track.scale.x.push([time, scale[0]])
+			track.scale.y.push([time, scale[1]])
+			track.rotation.push([time, rotation])
+		}
+
+		return {terp, track}
+	},
+}
+
+interface BuildFilterAction<TFilter extends FilterType> {
+	<T extends FilterableItem>(item: Build<T>, params?: FilterParams<TFilter>): Build<T>
+	make(params?: FilterParams<TFilter>): Build<Item.Filter<TFilter>>
+}
+
+interface BuildAnimateAction {
+	<T extends VisualAnimatableItem>(
+		item: Build<T>,
+		terp: Interpolation,
+		track: Keyframes
+	): Build<T>
+	make(terp: Interpolation, track: Keyframes): Build<Item.Animation>
+}
+
+type BuildFilterActions = {
+	[TName in keyof typeof filters]: BuildFilterAction<(typeof filters)[TName]["type"]>
+}
+
+function makeFilter<TFilter extends FilterType>(
+	get: (o: O) => FilterAction<TFilter>
+): BuildFilterAction<TFilter> {
+	const action = (<T extends FilterableItem>(
+		item: Build<T>,
+		params?: FilterParams<TFilter>
+	): Build<T> => o => get(o)(item(o), params)) as BuildFilterAction<TFilter>
+	action.make = (params?: FilterParams<TFilter>) => o => get(o).make(params)
+	return action
+}
+
+function makeFilters(): BuildFilterActions {
+	const names = Object.keys(filters) as (keyof typeof filters)[]
+	const entries = names.map(name => [
+		name,
+		makeFilter(o => o.filter[name] as FilterAction<any>)
+	])
+	return Object.fromEntries(entries) as BuildFilterActions
+}
+
+export const filter = makeFilters()
+
+function makeAnimate(
+	get: (o: O) => AnimateAction
+): BuildAnimateAction {
+	const action = (<T extends VisualAnimatableItem>(
+		item: Build<T>,
+		terp: Interpolation,
+		track: Keyframes
+	): Build<T> => o => get(o)(item(o), terp, track)) as BuildAnimateAction
+	action.make = (terp: Interpolation, track: Keyframes) => o => get(o).make(terp, track)
+	return action
+}
+
+function makeAnimateActions(): BuildVisualAnimateActions {
+	const entries = Object.keys(visualAnimations)
+		.map(key => [key, makeAnimate(o => o.animate[key as keyof VisualAnimations] as AnimateAction)])
+	return Object.fromEntries(entries) as BuildVisualAnimateActions
+}
+
+export const animate = makeAnimateActions()
+
+export function textStyle(style: TextStyleOptions): Build<Item.TextStyle> {
+	return o => o.textStyle(style)
+}
+
+export const transition = {
+	crossfade(duration: number): Build<Item.Transition> {
+		return o => o.transition.crossfade(duration)
+	}
+}
+