@omnimedia/omnitool 1.1.0-5 → 1.1.0-51

package/s/timeline/renderers/renderers.test.ts

@@ -0,0 +1,387 @@
+
+import {Science, test, expect} from "@e280/science"
+
+import {O} from "../sugar/o.js"
+import {ms} from "../../units/ms.js"
+import {Omni} from "../sugar/omni.js"
+import {fps} from "../../units/fps.js"
+import {Datafile} from "../utils/datafile.js"
+import {Driver} from "../../driver/driver.js"
+import {AudioMix} from "./export/parts/audio-mix.js"
+import {produceAudio} from "./export/parts/produce-audio.js"
+import {resampleToPlanar} from "./export/parts/resamplers.js"
+import {loadVideo} from "../../demo/routines/load-video.js"
+import {produceVideo} from "./export/parts/produce-video.js"
+import {CursorVisualSampler} from "./export/parts/cursor.js"
+import {applyGainToPlanar} from "./export/parts/audio-gain.js"
+import {createVisualSampler} from "./parts/samplers/visual/sampler.js"
+
+const workerUrl = new URL("../driver/driver.worker.bundle.min.js", import.meta.url)
+export async function setupTest() {
+	const driver = await Driver.setup({workerUrl})
+	const omni = new Omni(driver)
+
+	const testVideo = await loadVideo("/assets/temp/test.mp4")
+	const {videoA} = await omni.load({videoA: Datafile.make(testVideo, "test.mp4")})
+
+	const resolveMedia = (hash: string) => omni.resources.require(hash).blob
+
+	return {driver, omni, testVideo, videoA, resolveMedia}
+}
+
+async function collect<T>(iterable: AsyncIterable<T>) {
+	const out: T[] = []
+	for await (const item of iterable)
+		out.push(item)
+	return out
+}
+
+const near = (actual: number, expected: number, eps = 1e-6) =>
+	Math.abs(actual - expected) < eps
+
+export default Science.suite({
+
+	"cursor visual sampler cannot get previous samples": test(async () => {
+		const {omni, videoA, resolveMedia, driver} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500}),
+		))})
+		const sampler = new CursorVisualSampler(driver, resolveMedia)
+		const cursor = sampler.cursor(timeline)
+		await cursor.next(ms(1000))
+		await expect(async () => await cursor.next(ms(100))).throwsAsync()
+	}),
+
+	"visual sampler gives correct layer at x time": test(async () => {
+		const {omni, videoA, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500}),
+			o.text("123", {duration: 1000})
+		))})
+		const sampler = createVisualSampler(resolveMedia)
+		const imgLayer = await sampler.sample(timeline, ms(1000))
+		expect(imgLayer[0].kind).is("image")
+		const gapLayer = await sampler.sample(timeline, ms(2300))
+		expect(gapLayer[0].kind).is("gap")
+		const imgLayer1 = await sampler.sample(timeline, ms(2700))
+		expect(imgLayer1[0].kind).is("image")
+		const textLayer = await sampler.sample(timeline, ms(5500))
+		expect(textLayer[0].kind).is("text")
+	}),
+
+	"audio mix sums overlapping chunks": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([0.25, 0.25, 0.25, 0.25])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([0.5, 0.5, 0.5, 0.5])],
+				sampleRate: 4,
+				timestamp: 0.5
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(2)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].frames).is(4)
+		expect(mixed[0].channels).is(1)
+		expect(mixed[0].planar[0]).is(0.25)
+		expect(mixed[0].planar[1]).is(0.25)
+		expect(mixed[0].planar[2]).is(0.75)
+		expect(mixed[0].planar[3]).is(0.75)
+		expect(mixed[1].startFrame).is(4)
+		expect(mixed[1].planar[0]).is(0.5)
+		expect(mixed[1].planar[1]).is(0.5)
+		expect(mixed[1].planar[2]).is(0)
+		expect(mixed[1].planar[3]).is(0)
+	}),
+
+	"audio mix clamps output to [-1, 1]": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: true})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([0.9, 0.9, 0.9, 0.9])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([0.8, 0.8, 0.8, 0.8])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].planar[0]).is(1)
+		expect(mixed[0].planar[1]).is(1)
+		expect(mixed[0].planar[2]).is(1)
+		expect(mixed[0].planar[3]).is(1)
+	}),
+
+	"audio mix fills gaps with silence": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([1, 1, 1, 1])],
+				sampleRate: 4,
+				timestamp: 0
+			}
+			yield {
+				planes: [new Float32Array([2, 2, 2, 2])],
+				sampleRate: 4,
+				timestamp: 2
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(3)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].planar[0]).is(1)
+
+		expect(mixed[1].startFrame).is(4)
+		expect(mixed[1].planar[0]).is(0)
+		expect(mixed[1].planar[3]).is(0)
+
+		expect(mixed[2].startFrame).is(8)
+		expect(mixed[2].planar[0]).is(2)
+	}),
+
+	"audio mix truncates negative timestamps": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [new Float32Array([1, 2, 3, 4])],
+				sampleRate: 4,
+				timestamp: -0.5
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].startFrame).is(0)
+		expect(mixed[0].planar[0]).is(3)
+		expect(mixed[0].planar[1]).is(4)
+		expect(mixed[0].planar[2]).is(0)
+		expect(mixed[0].planar[3]).is(0)
+	}),
+
+	"audio mix handles stereo planar layout": test(async () => {
+		const mixer = new AudioMix({chunkFrames: 4, clamp: false})
+		async function *samples() {
+			yield {
+				planes: [
+					new Float32Array([1, 1, 1, 1]),
+					new Float32Array([2, 2, 2, 2])
+				],
+				sampleRate: 4,
+				timestamp: 0
+			}
+		}
+
+		const mixed = await collect(mixer.mix(samples()))
+		expect(mixed.length).is(1)
+		expect(mixed[0].channels).is(2)
+		expect(mixed[0].planar.length).is(8)
+		expect(mixed[0].planar[0]).is(1)
+		expect(mixed[0].planar[4]).is(2)
+	}),
+
+	"applyGainToPlanar scales values across multiple channels": test(async () => {
+		const planes = [
+			new Float32Array([0.2, -0.4]),
+			new Float32Array([0.6, -0.8])
+		]
+
+		applyGainToPlanar(planes, 0.5)
+
+		expect(near(planes[0][0], 0.1)).ok()
+		expect(near(planes[0][1], -0.2)).ok()
+		expect(near(planes[1][0], 0.3)).ok()
+		expect(near(planes[1][1], -0.4)).ok()
+	}),
+
+	"applyGainToPlanar does not mutate when gain is 1": test(async () => {
+		const planes = [new Float32Array([0.5, -0.5])]
+
+		applyGainToPlanar(planes, 1)
+
+		expect(near(planes[0][0], 0.5)).ok()
+		expect(near(planes[0][1], -0.5)).ok()
+	}),
+
+	"applyGainToPlanar mutes entirely when gain is 0": test(async () => {
+		const planes = [new Float32Array([0.9, -0.9])]
+
+		applyGainToPlanar(planes, 0)
+
+		expect(planes[0][0]).is(0)
+		expect(planes[0][1]).is(0)
+	}),
+
+	"resampleToPlanar returns original data when sample rates match": test(async () => {
+		const sample = {
+			numberOfFrames: 3,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array) => dest.set([0.1, 0.2, 0.3])
+		}
+
+		const result = resampleToPlanar(sample, 48000)
+
+		expect(result.frames).is(3)
+		expect(result.data.length).is(1)
+		expect(near(result.data[0][0], 0.1)).ok()
+		expect(near(result.data[0][1], 0.2)).ok()
+		expect(near(result.data[0][2], 0.3)).ok()
+	}),
+
+	"resampleToPlanar upsamples using linear interpolation": test(async () => {
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 1,
+			sampleRate: 1,
+			copyTo: (dest: Float32Array) => dest.set([10, 20])
+		}
+
+		const result = resampleToPlanar(sample as any, 2)
+
+		expect(result.frames).is(4)
+		expect(near(result.data[0][0], 10)).ok()
+		expect(near(result.data[0][1], 15)).ok()
+		expect(near(result.data[0][2], 20)).ok()
+		expect(near(result.data[0][3], 20)).ok()
+	}),
+
+	"resampleToPlanar downsamples correctly": test(async () => {
+		const sample = {
+			numberOfFrames: 4,
+			numberOfChannels: 1,
+			sampleRate: 4,
+			copyTo: (dest: Float32Array) => dest.set([1, 2, 3, 4])
+		}
+
+		const result = resampleToPlanar(sample, 2)
+
+		expect(result.frames).is(2)
+		expect(near(result.data[0][0], 1)).ok()
+		expect(near(result.data[0][1], 3)).ok()
+	}),
+
+	"resampleToPlanar processes multi-channel planar data": test(async () => {
+		const planes = [
+			new Float32Array([1, 2]),
+			new Float32Array([3, 4])
+		]
+
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 2,
+			sampleRate: 1,
+			copyTo: (dest: Float32Array, options: { planeIndex: number }) => {
+				dest.set(planes[options.planeIndex])
+			}
+		}
+
+		const result = resampleToPlanar(sample, 2)
+
+		expect(result.data.length).is(2)
+		expect(near(result.data[0][1], 1.5)).ok()
+		expect(near(result.data[1][1], 3.5)).ok()
+	}),
+
+	"resampleToPlanar handles zero channels gracefully": test(async () => {
+		const sample = {
+			numberOfFrames: 10,
+			numberOfChannels: 0,
+			sampleRate: 48000,
+			copyTo: () => { throw new Error("Should not be called") }
+		}
+
+		const result = resampleToPlanar(sample, 44100)
+
+		expect(result.data.length).is(0)
+		expect(result.frames).is(0)
+	}),
+
+	"resampleToPlanar enforces minimum of 1 frame on aggressive downsample": test(async () => {
+		const sample = {
+			numberOfFrames: 1,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array) => dest.set([0.8])
+		}
+
+		const result = resampleToPlanar(sample, 8000)
+
+		expect(result.frames).is(1)
+		expect(result.data[0].length).is(1)
+		expect(near(result.data[0][0], 0.8)).ok()
+	}),
+
+	"resampleToPlanar calls copyTo with strict contract shape": test(async () => {
+		let passedOptions: any = null
+		const sample = {
+			numberOfFrames: 2,
+			numberOfChannels: 1,
+			sampleRate: 48000,
+			copyTo: (dest: Float32Array, options: any) => {
+				passedOptions = options
+			}
+		}
+
+		resampleToPlanar(sample, 44100)
+
+		expect(passedOptions).not.is(null)
+		expect(passedOptions.planeIndex).is(0)
+		expect(passedOptions.format).is("f32-planar")
+	}),
+
+	"5s long export at 30fps renders exacly 150 frames": test(async () => {
+		const {omni, videoA, driver, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500})
+		))})
+		const readable = produceVideo({timeline, fps: fps(30), driver, resolveMedia})
+		let frames = 0
+		for await (const frame of readable) {
+			frame.close()
+			frames++
+		}
+		expect(frames).is(150)
+	}),
+
+	"5s long audio export at 48000Hz renders exactly 240000 frames": test(async () => {
+		const {omni, videoA, resolveMedia} = await setupTest()
+		const {timeline} = new O({timeline: omni.timeline(o => o.sequence(
+			o.video(videoA, {duration: 2000}),
+			o.gap(500),
+			o.video(videoA, {duration: 2000}),
+			o.audio(videoA, {duration: 500})
+		))})
+
+		const readable = produceAudio({timeline, resolveMedia})
+		let totalFrames = 0
+
+		for await (const chunk of readable) {
+			totalFrames += chunk.numberOfFrames
+			chunk.close()
+		}
+
+		expect(totalFrames).is(240000)
+	})
+})
+

package/s/timeline/sugar/helpers.ts

@@ -0,0 +1,85 @@
+
+import {TextStyleOptions} from "pixi.js"
+
+import {O} from "./o.js"
+import {Item} from "../parts/item.js"
+import {Transform} from "../types.js"
+import {Media} from "../parts/media.js"
+import {TimelineFile} from "../parts/basics.js"
+
+type Build<T extends Item.Any = Item.Any> = (o: O) => T
+
+function createTimeline(): TimelineFile {
+	return {
+		format: "timeline",
+		info: "https://omniclip.app/",
+		version: 0,
+		items: [],
+		rootId: 0
+	}
+}
+
+export function timeline(root: Build): TimelineFile {
+	const o = new O({timeline: createTimeline()})
+	const item = root(o)
+	o.timeline.rootId = item.id
+	return o.timeline
+}
+
+export function sequence(...items: Build[]): Build<Item.Sequence> {
+	return o => o.sequence(...items.map(item => item(o)))
+}
+
+export function stack(...items: Build[]): Build<Item.Stack> {
+	return o => o.stack(...items.map(item => item(o)))
+}
+
+export function video(
+	media: Media,
+	options?: {
+		start?: number,
+		duration?: number
+	}
+): Build<Item.Video> {
+	return o => o.video(media, options)
+}
+
+export function audio(
+	media: Media,
+	options?: {
+		start?: number,
+		duration?: number,
+		gain?: number
+	}
+): Build<Item.Audio> {
+	return o => o.audio(media, options)
+}
+
+export function text(
+	content: string,
+	options?: {
+		duration?: number,
+		styles?: TextStyleOptions
+	}
+): Build<Item.Text> {
+	return o => o.text(content, options)
+}
+
+export function gap(duration: number): Build<Item.Gap> {
+	return o => o.gap(duration)
+}
+
+export function spatial(transform: Transform): Build<Item.Spatial> {
+	return o => o.spatial(transform)
+}
+
+export function textStyle(style: TextStyleOptions): Build<Item.TextStyle> {
+	return o => o.textStyle(style)
+}
+
+export const transition = {
+	crossfade(duration: number): Build<Item.Transition> {
+		return o => o.transition.crossfade(duration)
+	}
+}
+

package/s/timeline/sugar/o.ts

@@ -1,60 +1,200 @@
+import {hex} from "@e280/stz"
+import {TextStyleOptions} from "pixi.js"
 
-import {MapG} from "@e280/stz"
-import {Id} from "../parts/basics.js"
 import {Media} from "../parts/media.js"
+import {Id, TimelineFile} from "../parts/basics.js"
 import {Effect, Item, Kind} from "../parts/item.js"
+import {Transform, TransformOptions, Vec2} from "../types.js"
 
 export class O {
-	#nextId = 0
-	#items = new MapG<Id, Item.Any>()
+	constructor(public state: {timeline: TimelineFile}) {}
+
+	require<T extends Item.Any>(id: Id | undefined) {
+    if (id === undefined)
+    	return undefined
+    const item = this.state.timeline.items.find(item => item.id === id)
+    return item as T | undefined
+	}
+
+	get timeline() {
+		return this.state.timeline
+	}
 
 	#getId() {
-		return this.#nextId++
+		return hex.toInteger(hex.random())
 	}
 
+  #mutate(fn: (project: TimelineFile) => TimelineFile) {
+    this.state.timeline = fn(this.state.timeline)
+  }
+
 	register(item: Item.Any) {
-		if (!this.#items.has(item.id))
-			this.#items.set(item.id, item)
-		return item.id
+		this.#mutate(state => ({
+			...state,
+			items: [...state.items, item]
+		}))
+	}
+
+  textStyle = (style: TextStyleOptions): Item.TextStyle => {
+		const item = {
+			id: this.#getId(),
+			kind: Kind.TextStyle,
+			style
+		} as Item.TextStyle
+		this.register(item)
+		return item
+  }
+
+  spatial = (transform: Transform): Item.Spatial => {
+  	const item: Item.Spatial = {
+  		id: this.#getId(),
+  		kind: Kind.Spatial,
+  		transform,
+  		enabled: true
+  	}
+		this.register(item)
+  	return item
+  }
+
+	sequence = (...items: Item.Any[]): Item.Sequence => {
+		const item =  {
+			id: this.#getId(),
+			kind: Kind.Sequence,
+			childrenIds: items.map(item => item.id)
+		} as Item.Sequence
+		this.register(item)
+		return item
 	}
 
-	get items() {
-		return [...this.#items.values()]
+	stack = (...items: Item.Any[]): Item.Stack => {
+		const item = {
+			kind: Kind.Stack,
+			id: this.#getId(),
+			childrenIds: items.map(item => item.id)
+		} as Item.Stack
+		this.register(item)
+		return item
 	}
 
-	sequence = (...items: Item.Any[]): Item.Sequence => ({
-		id: this.#getId(),
-		kind: Kind.Sequence,
-		children: items.map(item => this.register(item)),
-	})
+	video = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Video => {
 
-	stack = (...items: Item.Any[]): Item.Stack => ({
-		id: this.#getId(),
-		kind: Kind.Stack,
-		children: items.map(item => this.register(item)),
-	})
+		if(!media.hasVideo)
+			throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
 
-	clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
-		id: this.#getId(),
-		kind: Kind.Clip,
-		mediaHash: media.datafile.checksum.hash,
-		start: start ?? 0,
-		duration: duration ?? media.duration,
-	})
+		const item: Item.Video = {
+			kind: Kind.Video,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+		this.register(item)
+		return item
+	}
 
-	text = (content: string): Item.Text => ({
-		id: this.#getId(),
-		kind: Kind.Text,
-		content,
-	})
+	audio = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number,
+			gain?: number
+		}): Item.Audio => {
 
-	transition = {
-		crossfade: (duration: number): Item.Transition => ({
+		if(!media.hasAudio)
+			throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
+
+		const item: Item.Audio = {
+			kind: Kind.Audio,
 			id: this.#getId(),
-			kind: Kind.Transition,
-			effect: Effect.Crossfade,
-			duration,
-		}),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration,
+			gain: options?.gain ?? 1
+		}
+		this.register(item)
+		return item
+	}
+
+	text = (content: string, options?: {
+			duration?: number,
+			styles?: TextStyleOptions
+		}): Item.Text => {
+
+		const item = {
+			id: this.#getId(),
+			content,
+			kind: Kind.Text,
+			duration: options?.duration ?? 2000
+		} as Item.Text
+
+		if(options?.styles)
+			item.styleId = this.textStyle(options.styles).id
+
+		this.register(item)
+		return item
+	}
+
+	gap = (duration: number): Item.Gap => {
+		const item = {
+			id: this.#getId(),
+			kind: Kind.Gap,
+			duration
+		} as Item.Gap
+		this.register(item)
+		return item
+	}
+
+	transition = {
+		crossfade: (duration: number): Item.Transition => {
+			const item = {
+				id: this.#getId(),
+				kind: Kind.Transition,
+				effect: Effect.Crossfade,
+				duration,
+			} as Item.Transition
+			this.register(item)
+			return item
+		},
+	}
+
+  transform = (options?: TransformOptions): Transform => {
+    const position: Vec2 = [
+    	options?.position?.[0] ?? 0,
+    	options?.position?.[1] ?? 0
+    ]
+    const scale: Vec2 = [
+    	options?.scale?.[0] ?? 1,
+    	options?.scale?.[1] ?? 1
+    ]
+    const rotation = options?.rotation ?? 0
+    return [position, scale, rotation]
+  }
+
+  addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
+		this.#mutate(state => {
+			const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
+			parentItem.childrenIds.push(...items.map(item => item.id))
+			return state
+		})
+  }
+
+	set = <K extends Item.Any>(
+		id: Id,
+		partial: Partial<K>
+	) => {
+		this.#mutate(project => ({
+			...project,
+			items: project.items.map(item =>
+				item.id === id
+					? { ...item, ...partial }
+					: item
+			)
+		}))
 	}
 }