@omnimedia/omnitool 1.0.0 → 1.1.0-10

package/s/timeline/parts/filmstrip.ts

@@ -0,0 +1,159 @@
+import {
+	ALL_FORMATS,
+	CanvasSink,
+	CanvasSinkOptions,
+	Input,
+	InputVideoTrack,
+	WrappedCanvas,
+} from "mediabunny"
+
+import {DecoderSource} from '../../driver/fns/schematic.js'
+import {loadDecoderSource} from '../../driver/utils/load-decoder-source.js'
+
+export class Filmstrip {
+	#sink: CanvasSink
+	#cache: Map<number, WrappedCanvas> = new Map()
+	#activeRange: TimeRange = [0, 0]
+
+	private constructor(
+		private videoTrack: InputVideoTrack,
+		private options: Required<FilmstripOptions>
+	) {
+		this.#sink = new CanvasSink(videoTrack, options.canvasSinkOptions)
+	}
+
+	static async init(source: DecoderSource, options: FilmstripOptions) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		const videoTrack = await input.getPrimaryVideoTrack()
+		if(videoTrack)
+			return new Filmstrip(
+				videoTrack, {
+					frequency: options.frequency ?? 1,
+					canvasSinkOptions: options.canvasSinkOptions ?? {width: 80, height: 50, fit: "fill"},
+					onChange: options.onChange
+			})
+		else throw new Error("Source has no video track")
+	}
+
+	/**
+ 	* Sets the frequency (granularity) of filmstrip thumbnails.
+ 	* Changing this triggers a filmstrip refresh after any ongoing update finishes.
+ 	* @param value - The new frequency in seconds.
+ 	*/
+	set frequency(value: number) {
+		if(value !== this.options.frequency) {
+			this.options.frequency = value
+			this.#update()
+		}
+	}
+
+	get frequency() {
+		return this.options.frequency
+	}
+
+	#computeActiveRange([start, end]: TimeRange): TimeRange {
+		const tileSize = end - start
+		return [start - tileSize, end + tileSize]
+	}
+
+	async #generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededTimestamps = new Set<number>()
+
+		// duration should be computed but with trim etc also
+		const duration = await this.videoTrack.computeDuration()
+		for (
+			let timestamp = rangeStart;
+			timestamp <= rangeEnd;
+			timestamp += this.options.frequency
+		) {
+			// Clamp to valid time range
+			if (timestamp >= 0 && timestamp <= duration)
+				neededTimestamps.add(+timestamp.toFixed(3))
+		}
+
+		const missingTimestamps = [...neededTimestamps]
+			.filter(t => !this.#cache.has(t))
+
+		let i = 0
+		for await (const canvas of this.#sink.canvasesAtTimestamps(missingTimestamps)) {
+			if(canvas) {
+				const requestedTime = missingTimestamps[i++]
+				this.#cache.set(requestedTime, canvas)
+			}
+		}
+
+		// Dispose canvases outside the new range
+		for (const key of this.#cache.keys()) {
+			if (!neededTimestamps.has(key)) {
+				this.#cache.delete(key)
+			}
+		}
+
+		const tiles = [...this.#cache.entries()]
+			.map(([time, canvas]) => ({time, canvas}))
+		this.options.onChange(tiles)
+	}
+
+	/**
+ 	* Updates the visible time range for the filmstrip.
+ 	*
+ 	* Triggers a thumbnails update, with extended margins to preload
+ 	* thumbnails slightly outside the visible range.
+ 	* @param visibleRange - The current timeline viewport as a [start, end] tuple in seconds.
+ 	*/
+	set range(visibleRange: TimeRange) {
+		const newRange = this.#computeActiveRange(visibleRange)
+		// Avoid redundant updates
+		if (
+			this.#activeRange[0] === newRange[0] &&
+			this.#activeRange[1] === newRange[1]
+		)
+			return
+
+		this.#activeRange = newRange
+		this.#update()
+	}
+
+	#updating: Promise<void> | null = null
+	#shouldRunAgain = false
+
+	async #update() {
+		// Perform update immediately. If multiple updates are requested while updating,
+		// only the latest one will run after the current finishes (skips intermediate ones).
+		if(this.#updating) {
+			this.#shouldRunAgain = true
+			return
+		}
+
+		this.#updating = this.#generateTiles()
+		await this.#updating
+		this.#updating = null
+
+		if(this.#shouldRunAgain) {
+			this.#shouldRunAgain = false
+			await this.#update()
+		}
+	}
+	/**
+ 	* Returns the cached thumbnail (if any) for a given timestamp.
+ 	* @param time - The timestamp to retrieve the canvas for.
+ 	*/
+	getThumbnail(time: number) {
+		return this.#cache.get(time)
+	}
+}
+
+type TimeRange = [number, number]
+
+interface FilmstripOptions {
+	frequency?: number
+	canvasSinkOptions?: CanvasSinkOptions
+	onChange: (tiles: {
+		time: number
+		canvas: WrappedCanvas
+	}[]) => void
+}

package/s/timeline/parts/item.ts

@@ -0,0 +1,90 @@
+
+import {Id, Hash} from "./basics.js"
+import {Transform} from "../types.js"
+
+export enum Kind {
+	Sequence,
+	Stack,
+	Video,
+	Audio,
+	Text,
+	Gap,
+	Spatial,
+	Transition,
+}
+
+export enum Effect {
+	Crossfade,
+}
+
+export namespace Item {
+  export type Spatial = {
+    id: Id
+    kind: Kind.Spatial
+    transform: Transform
+  }
+
+	export type Gap = {
+		id: Id
+		kind: Kind.Gap
+		duration: number
+	}
+
+	export type Sequence = {
+		id: Id
+		kind: Kind.Sequence
+		childrenIds: Id[]
+		spatialId?: Id
+	}
+
+	export type Stack = {
+		id: Id
+		kind: Kind.Stack
+		childrenIds: Id[]
+		spatialId?: Id
+	}
+
+	export type Video = {
+		id: Id
+		kind: Kind.Video
+		mediaHash: Hash
+		start: number
+		duration: number
+		spatialId?: Id
+	}
+
+	export type Audio = {
+		id: Id
+		kind: Kind.Audio
+		mediaHash: Hash
+		start: number
+		duration: number
+	}
+
+	export type Text = {
+		id: Id
+		kind: Kind.Text
+		content: string
+		spatialId?: Id
+		color: string
+	}
+
+	export type Transition = {
+		id: Id
+		kind: Kind.Transition
+		effect: Effect.Crossfade
+		duration: number
+	}
+
+	export type Any = (
+		| Sequence
+		| Stack
+		| Video
+		| Audio
+		| Text
+		| Gap
+		| Transition
+		| Spatial
+	)
+}
+

package/s/timeline/parts/media.ts

@@ -0,0 +1,35 @@
+
+import {ALL_FORMATS, Input} from "mediabunny"
+
+import {Datafile} from "../utils/datafile.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../driver/utils/load-decoder-source.js"
+
+export class Media {
+	duration = 0
+	hasVideo = false
+	hasAudio = false
+
+	constructor(public datafile: Datafile) {}
+
+	static async analyze(datafile: Datafile) {
+		const media = new this(datafile)
+		media.duration = 10
+		const {video, audio} = await this.#has("/assets/temp/gl.mp4")
+		media.hasAudio = audio
+		media.hasVideo = video
+		return media
+	}
+
+	static async #has(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		return {
+			audio: !!(await input.getPrimaryAudioTrack()),
+			video: !!(await input.getPrimaryVideoTrack())
+		}
+	}
+}
+

package/s/timeline/parts/resource-pool.ts

@@ -0,0 +1,27 @@
+
+import {MapG} from "@e280/stz"
+import {Hash} from "./basics.js"
+import {Media} from "./media.js"
+import {Resource} from "./resource.js"
+import {Datafile} from "../utils/datafile.js"
+
+export class ResourcePool {
+	#map = new MapG<Hash, Resource.Any>
+
+	/** store a media file (avoids duplicates via hash) */
+	async store(datafile: Datafile) {
+		const media = await Media.analyze(datafile)
+		const {hash} = media.datafile.checksum
+		const {filename, bytes} = media.datafile
+
+		if (this.#map.has(hash)) {
+			const alreadyExists = this.#map.require(hash)
+			alreadyExists.filename = filename
+		}
+		else
+			this.#map.set(hash, {kind: "media", filename, bytes})
+
+		return media
+	}
+}
+

package/s/timeline/parts/resource.ts

@@ -0,0 +1,11 @@
+
+export namespace Resource {
+	export type Media = {
+		kind: "media"
+		filename: string
+		bytes: Uint8Array
+	}
+
+	export type Any = Media
+}
+

package/s/timeline/parts/waveform.ts

@@ -0,0 +1,62 @@
+import WaveSurfer from "wavesurfer.js"
+
+import {context} from "../../context.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+
+export class Waveform {
+	wavesurfer: WaveSurfer
+
+	constructor(peaks: number[], container: HTMLElement, duration: number) {
+		this.wavesurfer = WaveSurfer.create({
+  		container,
+  		waveColor: 'rgb(200, 0, 200)',
+  		progressColor: 'rgb(100, 0, 100)',
+  		barWidth: 10,
+  		barRadius: 10,
+  		barGap: 2,
+  		peaks: [peaks],
+  		duration
+		})
+	}
+
+	static async init(source: DecoderSource, container: HTMLElement) {
+		const driver = await context.driver
+		const reader = driver.decodeAudio({source}).getReader()
+
+		const peaks: number[] = []
+		let buffer: number[] = []
+		const samplesPerPeak = 1024
+		const duration = await driver.getAudioDuration(source)
+
+		while (true) {
+			const {done, value: audioData} = await reader.read()
+			if (done) break
+
+			const frames = audioData.numberOfFrames
+			const plane = new Float32Array(frames)
+			audioData.copyTo(plane, {planeIndex: 0}) // Use left channel only
+
+			for (let i = 0; i < plane.length; i++) {
+				buffer.push(plane[i])
+				if (buffer.length >= samplesPerPeak) {
+					const chunk = buffer.splice(0, samplesPerPeak)
+					const min = Math.min(...chunk)
+					const max = Math.max(...chunk)
+					peaks.push(min, max)
+				}
+			}
+
+			audioData.close()
+		}
+
+		return new Waveform(peaks, container, duration ?? 0)
+	}
+
+	// set zoom(value: number) {
+	// 	this.wavesurfer.zoom(value)
+	// }
+
+	set width(value: number) {
+		this.wavesurfer.setOptions({width: value})
+	}
+}

package/s/timeline/sugar/builders.ts

@@ -0,0 +1,102 @@
+// import {O} from "./o.js"
+// import {Id} from "../parts/basics.js"
+// import {Item} from "../parts/item.js"
+//
+// export class TimelineItem {
+// 	public readonly id: Id
+//
+// 	constructor(public item: Item.Any) {
+// 		this.id = item.id
+// 	}
+//
+// 	toJSON() {
+// 		return {
+// 			...this.item
+// 		}
+// 	}
+// }
+//
+// abstract class VisualItem extends TimelineItem {
+//   abstract spatial(spatial: Spatial): TimelineItem
+// }
+//
+// export class Stack extends VisualItem {
+//   constructor(private o: O, public item: Item.Stack) {
+//   	super(item)
+//   }
+//
+//   spatial(spatial: Spatial) {
+//   	this.item.spatialId = spatial.item.id
+//     return this
+//   }
+//
+//   addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
+//   	const result = fn(this.o)
+//   	const items = Array.isArray(result) ? result : [result]
+//     this.item.childrenIds.push(...items.map(c => c.item.id))
+//     return this
+//   }
+// }
+//
+// export class Spatial extends TimelineItem {
+// 	constructor(public item: Item.Spatial) {super(item)}
+// }
+//
+// export class Gap extends TimelineItem {
+// 	constructor(public item: Item.Gap) {super(item)}
+// }
+//
+// export class Audio extends TimelineItem {
+// 	constructor(public item: Item.Audio) {super(item)}
+// }
+//
+// export class Video extends VisualItem {
+//   constructor(public item: Item.Video) {
+//   	super(item)
+//   }
+//
+//   spatial(spatial: Spatial) {
+//   	this.item.spatialId = spatial.item.id
+//     return this
+//   }
+// }
+//
+// export class Text extends VisualItem {
+//   constructor(public item: Item.Text) {
+//   	super(item)
+//   }
+//
+//   color(color: string) {
+//   	this.item.color = color
+// 		return this
+//   }
+//
+//   spatial(spatial: Spatial) {
+//   	this.item.spatialId = spatial.item.id
+//     return this
+//   }
+// }
+//
+// export class Sequence extends VisualItem {
+// 	constructor(private o: O, public item: Item.Sequence) {
+// 		super(item)
+// 	}
+//
+//   spatial(spatial: Spatial) {
+//   	this.item.spatialId = spatial.item.id
+//     return this
+//   }
+//
+//   addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
+//   	const result = fn(this.o)
+//   	const items = Array.isArray(result) ? result : [result]
+//     this.item.childrenIds.push(...items.map(c => c.item.id))
+//     return this
+//   }
+// }
+//
+// export class Transition extends TimelineItem {
+// 	constructor(public item: Item.Transition) {
+// 		super(item)
+// 	}
+// }

package/s/timeline/sugar/o.ts

@@ -0,0 +1,150 @@
+
+import {Media} from "../parts/media.js"
+import {Id, TimelineFile} from "../parts/basics.js"
+import {Effect, Item, Kind} from "../parts/item.js"
+import {Transform, TransformOptions, Vec2} from "../types.js"
+
+export class O {
+	#nextId = 0
+
+	constructor(public state: {project: TimelineFile}) {}
+
+  require<T extends Item.Any>(id: Id): T {
+    const item = this.state.project.items.find(item => item.id === id)
+    return item as T
+  }
+
+	#getId() {
+		return this.#nextId++
+	}
+
+  #mutate(fn: (project: TimelineFile) => TimelineFile) {
+    this.state.project = fn(this.state.project)
+  }
+
+  spatial = (transform: Transform): Item.Spatial => {
+  	const item: Item.Spatial = {
+  		id: this.#getId(),
+  		kind: Kind.Spatial,
+  		transform
+  	}
+  	return item
+  }
+
+	sequence = (...items: Item.Any[]): Item.Any => {
+		const item =  {
+			id: this.#getId(),
+			kind: Kind.Sequence,
+			childrenIds: items.map(item => item.id)
+		} as Item.Sequence
+
+		this.#mutate(state => {
+			state.items.push(item, ...items)
+			return state
+		})
+
+		return item
+	}
+
+	stack = (...items: Item.Any[]): Item.Any => {
+		const item = {
+			kind: Kind.Stack,
+			id: this.#getId(),
+			childrenIds: items.map(item => item.id)
+		} as Item.Stack
+
+		this.#mutate(state => {
+			state.items.push(item, ...items)
+			return state
+		})
+
+		return item
+	}
+
+	video = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Video => {
+
+		if(!media.hasVideo)
+			throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
+
+		const item: Item.Video = {
+			kind: Kind.Video,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return item
+	}
+
+	audio = (
+		media: Media,
+		options?: {
+			start?: number,
+			duration?: number
+		}): Item.Audio => {
+
+		if(!media.hasAudio)
+			throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
+
+		const item: Item.Audio = {
+			kind: Kind.Audio,
+			id: this.#getId(),
+			mediaHash: media.datafile.checksum.hash,
+			start: options?.start ?? 0,
+			duration: options?.duration ?? media.duration
+		}
+
+		return item
+	}
+
+	text = (content: string): Item.Text => ({
+		id: this.#getId(),
+		content,
+		kind: Kind.Text,
+		color: "#FFFFF"
+	})
+
+	gap = (duration: number): Item.Gap => ({
+		id: this.#getId(),
+		kind: Kind.Gap,
+		duration
+	})
+
+	transition = {
+		crossfade: (duration: number): Item.Transition => ({
+			id: this.#getId(),
+			kind: Kind.Transition,
+			effect: Effect.Crossfade,
+			duration,
+		}),
+	}
+
+  transform = (options?: TransformOptions): Transform => {
+    const position: Vec2 = [
+    	options?.position?.[0] ?? 0,
+    	options?.position?.[1] ?? 0
+    ]
+    const scale: Vec2 = [
+    	options?.scale?.[0] ?? 1,
+    	options?.scale?.[1] ?? 1
+    ]
+    const rotation = options?.rotation ?? 0
+    return [position, scale, rotation]
+  }
+
+  addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
+		this.#mutate(state => {
+			const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
+			parentItem.childrenIds.push(...items.map(item => item.id))
+			state.items.push(...items)
+			return state
+		})
+  }
+}
+

package/s/timeline/sugar/omni-test.ts

@@ -0,0 +1,38 @@
+
+import {Omni} from "./omni.js"
+import {dummyData} from "../utils/dummy-data.js"
+
+//
+// create an omni context
+//
+
+const omni = new Omni()
+
+//
+// load in some media resources
+//
+
+const {mediaA, mediaB} = await omni.load({
+	mediaA: dummyData(),
+	mediaB: dummyData(),
+})
+
+//
+// create a timeline
+//
+
+const timeline = omni.timeline(o => o.sequence(
+	o.video(mediaA),
+	o.transition.crossfade(600),
+	o.stack(
+		o.video(mediaB),
+		o.text("hello world"),
+	),
+))
+
+//
+// log the timeline
+//
+
+console.log(JSON.stringify(timeline, undefined, "  "))
+