@omnimedia/omnitool 1.1.0-5 → 1.1.0-8

package/s/timeline/parts/media.ts

@@ -1,14 +1,35 @@
 
+import {ALL_FORMATS, Input} from "mediabunny"
+
 import {Datafile} from "../utils/datafile.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../driver/utils/load-decoder-source.js"
 
 export class Media {
 	duration = 0
+	hasVideo = false
+	hasAudio = false
+
 	constructor(public datafile: Datafile) {}
 
 	static async analyze(datafile: Datafile) {
 		const media = new this(datafile)
 		media.duration = 10
+		const {video, audio} = await this.#has("/assets/temp/gl.mp4")
+		media.hasAudio = audio
+		media.hasVideo = video
 		return media
 	}
+
+	static async #has(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		return {
+			audio: !!(await input.getPrimaryAudioTrack()),
+			video: !!(await input.getPrimaryVideoTrack())
+		}
+	}
 }
 

package/s/timeline/parts/waveform.ts

@@ -21,7 +21,7 @@ export class Waveform {
 
 	static async init(source: DecoderSource, container: HTMLElement) {
 		const driver = await context.driver
-		const reader = driver.decode({ source }).audio.getReader()
+		const reader = driver.decodeAudio({source}).getReader()
 
 		const peaks: number[] = []
 		let buffer: number[] = []

package/s/timeline/sugar/builders.ts

@@ -0,0 +1,102 @@
+import {O} from "./o.js"
+import {Id} from "../parts/basics.js"
+import {Item} from "../parts/item.js"
+
+export class TimelineItem {
+	public readonly id: Id
+
+	constructor(public item: Item.Any) {
+		this.id = item.id
+	}
+
+	toJSON() {
+		return {
+			...this.item
+		}
+	}
+}
+
+abstract class VisualItem extends TimelineItem {
+  abstract spatial(spatial: Spatial): TimelineItem
+}
+
+export class Stack extends VisualItem {
+  constructor(private o: O, public item: Item.Stack) {
+  	super(item)
+  }
+
+  spatial(spatial: Spatial) {
+  	this.item.spatialId = spatial.item.id
+    return this
+  }
+
+  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
+  	const result = fn(this.o)
+  	const items = Array.isArray(result) ? result : [result]
+    this.item.childrenIds.push(...items.map(c => c.item.id))
+    return this
+  }
+}
+
+export class Spatial extends TimelineItem {
+	constructor(public item: Item.Spatial) {super(item)}
+}
+
+export class Gap extends TimelineItem {
+	constructor(public item: Item.Gap) {super(item)}
+}
+
+export class Audio extends TimelineItem {
+	constructor(public item: Item.Audio) {super(item)}
+}
+
+export class Video extends VisualItem {
+  constructor(public item: Item.Video) {
+  	super(item)
+  }
+
+  spatial(spatial: Spatial) {
+  	this.item.spatialId = spatial.item.id
+    return this
+  }
+}
+
+export class Text extends VisualItem {
+  constructor(public item: Item.Text) {
+  	super(item)
+  }
+
+  color(color: string) {
+  	this.item.color = color
+		return this
+  }
+
+  spatial(spatial: Spatial) {
+  	this.item.spatialId = spatial.item.id
+    return this
+  }
+}
+
+export class Sequence extends VisualItem {
+	constructor(private o: O, public item: Item.Sequence) {
+		super(item)
+	}
+
+  spatial(spatial: Spatial) {
+  	this.item.spatialId = spatial.item.id
+    return this
+  }
+
+  addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
+  	const result = fn(this.o)
+  	const items = Array.isArray(result) ? result : [result]
+    this.item.childrenIds.push(...items.map(c => c.item.id))
+    return this
+  }
+}
+
+export class Transition extends TimelineItem {
+	constructor(public item: Item.Transition) {
+		super(item)
+	}
+}

package/s/timeline/sugar/o.ts

@@ -3,16 +3,18 @@ import {MapG} from "@e280/stz"
 import {Id} from "../parts/basics.js"
 import {Media} from "../parts/media.js"
 import {Effect, Item, Kind} from "../parts/item.js"
+import {Transform, TransformOptions, Vec2} from "../types.js"
+import {Video, Gap, Sequence, Stack, Text, TimelineItem, Spatial, Audio, Transition} from "./builders.js"
 
 export class O {
 	#nextId = 0
-	#items = new MapG<Id, Item.Any>()
+	#items = new MapG<Id, TimelineItem>()
 
 	#getId() {
 		return this.#nextId++
 	}
 
-	register(item: Item.Any) {
+	register(item: TimelineItem) {
 		if (!this.#items.has(item.id))
 			this.#items.set(item.id, item)
 		return item.id
@@ -22,39 +24,96 @@ export class O {
 		return [...this.#items.values()]
 	}
 
-	sequence = (...items: Item.Any[]): Item.Sequence => ({
+	get itemsMap() {
+		return this.#items
+	}
+
+  spatial = (transform: Transform) => {
+  	const item: Item.Spatial = {
+  		id: this.#getId(),
+  		kind: Kind.Spatial,
+  		transform
+  	}
+  	const spatial = new Spatial(item)
+  	this.register(spatial)
+  	return spatial
+  }
+
+	sequence = (...items: TimelineItem[]) => new Sequence(this, {
 		id: this.#getId(),
 		kind: Kind.Sequence,
-		children: items.map(item => this.register(item)),
+		childrenIds: items.map((item) => this.register(item))
 	})
 
-	stack = (...items: Item.Any[]): Item.Stack => ({
-		id: this.#getId(),
+	stack = (...items: TimelineItem[]) => new Stack(this, {
 		kind: Kind.Stack,
-		children: items.map(item => this.register(item)),
+		id: this.#getId(),
+		childrenIds: items.map(item => this.register(item))
 	})
 
-	clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
+	video = (media: Media, options?: {start?: number, duration?: number}) => {
+		if(!media.hasVideo)
+			throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
+
+		const item: Item.Video = {
+			kind: Kind.Video,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return new Video(item)
+	}
+
+	audio = (media: Media, options?: {start?: number, duration?: number}) => {
+		if(!media.hasAudio)
+			throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
+
+		const item: Item.Audio = {
+			kind: Kind.Audio,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return new Audio(item)
+	}
+
+	text = (content: string) => new Text({
 		id: this.#getId(),
-		kind: Kind.Clip,
-		mediaHash: media.datafile.checksum.hash,
-		start: start ?? 0,
-		duration: duration ?? media.duration,
+		content,
+		kind: Kind.Text,
+		color: "#FFFFF"
 	})
 
-	text = (content: string): Item.Text => ({
+	gap = (duration: number) => new Gap({
 		id: this.#getId(),
-		kind: Kind.Text,
-		content,
+		kind: Kind.Gap,
+		duration
 	})
 
 	transition = {
-		crossfade: (duration: number): Item.Transition => ({
+		crossfade: (duration: number) => new Transition({
 			id: this.#getId(),
 			kind: Kind.Transition,
 			effect: Effect.Crossfade,
 			duration,
 		}),
 	}
+
+  transform = (options?: TransformOptions): Transform => {
+    const position: Vec2 = [
+    	options?.position?.[0] ?? 0,
+    	options?.position?.[1] ?? 0
+    ]
+    const scale: Vec2 = [
+    	options?.scale?.[0] ?? 1,
+    	options?.scale?.[1] ?? 1
+    ]
+    const rotation = options?.rotation ?? 0
+    return [position, scale, rotation]
+  }
 }
 

package/s/timeline/sugar/omni-test.ts

@@ -22,10 +22,10 @@ const {mediaA, mediaB} = await omni.load({
 //
 
 const timeline = omni.timeline(o => o.sequence(
-	o.clip(mediaA),
+	o.video(mediaA),
 	o.transition.crossfade(600),
 	o.stack(
-		o.clip(mediaB),
+		o.video(mediaB),
 		o.text("hello world"),
 	),
 ))

package/s/timeline/sugar/omni.ts

@@ -1,13 +1,17 @@
 
 import {O} from "./o.js"
-import {Item} from "../parts/item.js"
+import {Timeline} from "../timeline.js"
 import {Media} from "../parts/media.js"
-import {TimelineFile} from "../parts/basics.js"
+import {TimelineItem} from "./builders.js"
 import {Datafile} from "../utils/datafile.js"
+import {TimelineFile} from "../parts/basics.js"
+import {Export} from "../parts/compositor/export.js"
 import {ResourcePool} from "../parts/resource-pool.js"
+import {RenderConfig} from "../../driver/fns/schematic.js"
 
 export class Omni {
 	resources = new ResourcePool()
+	#export = new Export()
 
 	load = async<S extends Record<string, Promise<Datafile>>>(spec: S) => {
 		return Object.fromEntries(await Promise.all(Object.entries(spec).map(
@@ -15,16 +19,15 @@ export class Omni {
 		))) as {[K in keyof S]: Media}
 	}
 
-	timeline = (fn: (o: O) => Item.Sequence): TimelineFile => {
+	timeline = (fn: (o: O) => TimelineItem): Timeline => {
 		const o = new O()
-		const sequence = fn(o)
-		return {
-			format: "timeline",
-			info: "https://omniclip.app/",
-			version: 0,
-			root: o.register(sequence),
-			items: o.items,
-		}
+		const root = fn(o)
+		o.register(root)
+		return new Timeline(root, o.itemsMap)
+	}
+
+	render = async (timeline: TimelineFile, config: RenderConfig) => {
+		await this.#export.render(timeline)
 	}
 }
 

package/s/timeline/timeline.ts

@@ -0,0 +1,22 @@
+import {TimelineItem} from "./sugar/builders.js"
+import {Id, TimelineFile} from "./parts/basics.js"
+
+export class Timeline {
+  constructor(public root: TimelineItem, private items: Map<Id, TimelineItem>) {}
+
+  require<T extends TimelineItem>(id: Id): T {
+    const item = this.items.get(id)
+    return item as T
+  }
+
+  toJSON(): TimelineFile {
+    return {
+			format: "timeline",
+			info: "https://omniclip.app/",
+			version: 0,
+			rootId: this.root.item.id,
+			items: Array.from(this.items.values()).map(item => item.toJSON())
+    }
+  }
+}
+

package/s/timeline/types.ts

@@ -0,0 +1,29 @@
+export type Interpolation = "linear" | "catmullRom"
+export type Keyframe<Value = number> = [time: number, value: Value]
+export type Keyframes<Value = number> = Keyframe<Value>[]
+export type Vec2 = [x: number, y: number]
+export type Transform = [position: Vec2, scale: Vec2, rotation: number]
+
+export type TrackVec2 = {
+	x: Keyframes
+	y: Keyframes
+}
+
+export type Anim<T> = {
+  terp: Interpolation
+  track: T
+}
+
+export type Animations = Anim<TrackTransform>
+
+export type TrackTransform = {
+	position: TrackVec2
+	scale: TrackVec2
+	rotation: Keyframes
+}
+
+export type TransformOptions = {
+  position?: Vec2
+  scale?: Vec2
+  rotation?: number
+}

package/s/timeline/utils/audio-stream.ts

@@ -0,0 +1,15 @@
+export class AudioStream {
+	constructor(private reader: ReadableStreamDefaultReader<AudioData>) {}
+
+	async *stream(): AsyncGenerator<AudioData> {
+		while (true) {
+			const {done, value: hit} = await this.reader.read()
+			if (done) {
+				break
+			}
+			yield hit
+		}
+	}
+
+	cancel = async () => await this.reader.cancel()
+}

package/s/timeline/utils/matrix.ts

@@ -0,0 +1,33 @@
+import {Matrix} from "pixi.js"
+import {Transform} from "../types.js"
+
+export const transformToMat6 = (t: Transform): Mat6 => {
+	const [pos, scl, rotDeg] = t
+	const [x, y] = pos
+	const [sx, sy] = scl
+	const r = rotDeg * Math.PI / 180
+	const cos = Math.cos(r)
+	const sin = Math.sin(r)
+	return [cos * sx, sin * sx, -sin * sy, cos * sy, x, y]
+}
+
+export const mat6ToMatrix = ([a, b, c, d, tx, ty]: Mat6): Matrix =>
+	new Matrix(a, b, c, d, tx, ty)
+
+export const transformToMatrix = (t: Transform) => mat6ToMatrix(transformToMat6(t))
+
+export const mul6 = (local: Mat6, parent: Mat6): Mat6 => {
+	const [a1, b1, c1, d1, tx1, ty1] = local
+	const [a2, b2, c2, d2, tx2, ty2] = parent
+	return [
+		a1 * a2 + c1 * b2,
+		b1 * a2 + d1 * b2,
+		a1 * c2 + c1 * d2,
+		b1 * c2 + d1 * d2,
+		a1 * tx2 + c1 * ty2 + tx1,
+		b1 * tx2 + d1 * ty2 + ty1
+	]
+}
+
+export const I6: Mat6 = [1, 0, 0, 1, 0, 0]
+export type Mat6 = [a: number, b: number, c: number, d: number, tx: number, ty: number]

package/s/timeline/utils/video-cursor.ts

@@ -0,0 +1,40 @@
+/**
+ * A stateful, forward-only frame cursor for a single clip instance.
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
+ */
+
+export class VideoCursor {
+  constructor(private reader: ReadableStreamDefaultReader<VideoFrame>) {}
+
+  async atOrNear(targetUs: number): Promise<VideoFrame | undefined> {
+    let prev: VideoFrame | null = null
+    while (true) {
+      const {done, value: hit} = await this.reader.read()
+
+      if (done) {
+        const out = prev ? new VideoFrame(prev) : undefined
+        prev?.close()
+        return out
+      }
+
+      const hitUs = hit.timestamp ?? 0
+      if (hitUs >= targetUs) {
+        const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
+        const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
+
+        const chosen = usePrev ? prev! : hit
+        const other = usePrev ? hit : prev
+
+        const copy = new VideoFrame(chosen)
+        chosen.close()
+        other?.close()
+        return copy
+      }
+
+      prev?.close()
+      prev = hit
+    }
+  }
+
+  cancel = async () => await this.reader.cancel()
+}