@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/timeline/parts/item.ts

@@ -1,58 +1,171 @@
 
+import {TextStyleOptions} from "pixi.js"
+
 import {Id, Hash} from "./basics.js"
+import {Ms} from "../../units/ms.js"
+import type {TransitionName} from "./transitions.js"
+import {Transform, VisualAnimations} from "../types.js"
+import type {FilterParams, FilterType} from "./filters.js"
+import type {Transcription} from "../../features/speech/transcribe/types.js"
+
+export type Crop = [top: number, right: number, bottom: number, left: number]
 
 export enum Kind {
 	Sequence,
 	Stack,
-	Clip,
+	Video,
+	Audio,
 	Text,
+	Gap,
+	Spatial,
+	Animation,
 	Transition,
-}
-
-export enum Effect {
-	Crossfade,
+	TextStyle,
+	Filter,
+	Caption,
+	Image
 }
 
 export namespace Item {
+	export type TextStyle = {
+		id: Id
+		kind: Kind.TextStyle
+		style: TextStyleOptions
+	}
+
+	export type Spatial = {
+		id: Id
+		kind: Kind.Spatial
+		transform: Transform
+		crop?: Crop
+		enabled: boolean
+	}
+
+	export type Animation = {
+		id: Id
+		kind: Kind.Animation
+		anims: VisualAnimations
+		enabled: boolean
+	}
+
+	export type Filter<T extends FilterType = FilterType> = {
+		id: Id
+		kind: Kind.Filter
+		type: T
+		params?: FilterParams<T>
+		enabled: boolean
+	}
+
+	export type Gap = {
+		id: Id
+		kind: Kind.Gap
+		duration: number
+	}
+
 	export type Sequence = {
 		id: Id
 		kind: Kind.Sequence
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
+		filterIds?: Id[]
 	}
 
 	export type Stack = {
 		id: Id
 		kind: Kind.Stack
-		children: Id[]
+		childrenIds: Id[]
+		spatialId?: Id
+		filterIds?: Id[]
 	}
 
-	export type Clip = {
+	export type Video = {
 		id: Id
-		kind: Kind.Clip
+		kind: Kind.Video
 		mediaHash: Hash
 		start: number
 		duration: number
+		spatialId?: Id
+		animationIds?: Id[]
+		filterIds?: Id[]
+	}
+
+	export type Image = {
+		id: Id
+		kind: Kind.Image
+		mediaHash: Hash
+		duration: number
+		spatialId?: Id
+		animationIds?: Id[]
+		filterIds?: Id[]
+	}
+
+	export type Audio = {
+		id: Id
+		kind: Kind.Audio
+		mediaHash: Hash
+		start: number
+		duration: number
+		gain?: number
 	}
 
 	export type Text = {
 		id: Id
 		kind: Kind.Text
 		content: string
+		duration: number
+		spatialId?: Id
+		animationIds?: Id[]
+		styleId?: Id
+		filterIds?: Id[]
+	}
+
+
+	export type Caption = {
+		id: Id
+		kind: Kind.Caption
+		transcript: Transcription
+		itemId?: Id
+		start: number
+		duration: number
+		maxChars?: number
+		maxDuration?: number
+		maxSilence?: number
+		spatialId?: Id
+		animationIds?: Id[]
+		styleId?: Id
+		filterIds?: Id[]
 	}
 
 	export type Transition = {
 		id: Id
 		kind: Kind.Transition
-		effect: Effect.Crossfade
+		name: TransitionName
 		duration: number
 	}
 
 	export type Any = (
 		| Sequence
 		| Stack
-		| Clip
+		| Video
+		| Image
+		| Audio
 		| Text
+		| Caption
+		| Gap
 		| Transition
+		| Spatial
+		| Animation
+		| TextStyle
+		| Filter
 	)
 }
 
+export type ContainerItem = Item.Sequence | Item.Stack
+export type NonContainerItem = Exclude<Item.Any, ContainerItem>
+export type FilterableItem = Item.Sequence | Item.Stack | Item.Video | Item.Image | Item.Text | Item.Caption
+export type VisualAnimatableItem = Item.Video | Item.Image | Item.Text | Item.Caption
+
+export type PlayableItem = Item.Any & {
+	start: Ms
+	duration: Ms
+}

package/s/timeline/parts/media.ts

@@ -1,14 +1,56 @@
 
+import {ALL_FORMATS, Input} from "mediabunny"
+
 import {Datafile} from "../utils/datafile.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../driver/utils/load-decoder-source.js"
 
 export class Media {
 	duration = 0
+	isImage = false
+	hasVideo = false
+	hasAudio = false
+
 	constructor(public datafile: Datafile) {}
 
 	static async analyze(datafile: Datafile) {
 		const media = new this(datafile)
-		media.duration = 10
+
+		if (this.#isImage(datafile)) {
+			media.isImage = true
+			return media
+		}
+
+		const duration = (await this.duration(datafile.url)) * 1000
+		media.duration = duration
+		const {video, audio} = await this.#has(datafile.url)
+		media.hasAudio = audio
+		media.hasVideo = video
 		return media
 	}
+
+	static #isImage(datafile: Datafile) {
+		return datafile.blob.type.startsWith("image/")
+	}
+
+	static async duration(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		const duration =  await input.computeDuration()
+		return Number(duration.toFixed(5)) // fix weird floating points
+	}
+
+	static async #has(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		return {
+			audio: !!(await input.getPrimaryAudioTrack()),
+			video: !!(await input.getPrimaryVideoTrack())
+		}
+	}
 }
 

package/s/timeline/parts/resource-pool.ts

@@ -1,27 +1,30 @@
 
-import {MapG} from "@e280/stz"
+import {GMap} from "@e280/stz"
 import {Hash} from "./basics.js"
 import {Media} from "./media.js"
 import {Resource} from "./resource.js"
 import {Datafile} from "../utils/datafile.js"
 
 export class ResourcePool {
-	#map = new MapG<Hash, Resource.Any>
+	#map = new GMap<Hash, Resource.Any>
 
 	/** store a media file (avoids duplicates via hash) */
 	async store(datafile: Datafile) {
 		const media = await Media.analyze(datafile)
 		const {hash} = media.datafile.checksum
-		const {filename, bytes} = media.datafile
+		const {filename, bytes, url, blob} = media.datafile
 
 		if (this.#map.has(hash)) {
 			const alreadyExists = this.#map.require(hash)
 			alreadyExists.filename = filename
 		}
 		else
-			this.#map.set(hash, {kind: "media", filename, bytes})
+			this.#map.set(hash, {kind: "media", filename, bytes, url, blob, duration: media.duration})
 
 		return media
 	}
-}
 
+	require(hash: Hash) {
+		return this.#map.require(hash)
+	}
+}

package/s/timeline/parts/resource.ts

@@ -4,6 +4,9 @@ export namespace Resource {
 		kind: "media"
 		filename: string
 		bytes: Uint8Array
+		blob: Blob
+		url: string
+		duration: number
 	}
 
 	export type Any = Media

package/s/timeline/parts/transitions.ts

@@ -0,0 +1,102 @@
+import type {Item} from "./item.js"
+
+export const transitionNames = [
+	"Bounce",
+	"BowTieHorizontal",
+	"BowTieVertical",
+	"ButterflyWaveScrawler",
+	"CircleCrop",
+	"ColourDistance",
+	"CrazyParametricFun",
+	"CrossZoom",
+	"Directional",
+	"DoomScreenTransition",
+	"Dreamy",
+	"DreamyZoom",
+	"GlitchDisplace",
+	"GlitchMemories",
+	"GridFlip",
+	"InvertedPageCurl",
+	"LinearBlur",
+	"Mosaic",
+	"PolkaDotsCurtain",
+	"Radial",
+	"SimpleZoom",
+	"StereoViewer",
+	"Swirl",
+	"WaterDrop",
+	"ZoomInCircles",
+	"angular",
+	"burn",
+	"cannabisleaf",
+	"circle",
+	"circleopen",
+	"colorphase",
+	"crosshatch",
+	"crosswarp",
+	"cube",
+	"directionalwarp",
+	"directionalwipe",
+	"displacement",
+	"doorway",
+	"fade",
+	"fadecolor",
+	"fadegrayscale",
+	"flyeye",
+	"heart",
+	"hexagonalize",
+	"kaleidoscope",
+	"luma",
+	"luminance_melt",
+	"morph",
+	"multiply_blend",
+	"perlin",
+	"pinwheel",
+	"pixelize",
+	"polar_function",
+	"randomsquares",
+	"ripple",
+	"rotate_scale_fade",
+	"squareswire",
+	"squeeze",
+	"swap",
+	"undulatingBurnOut",
+	"wind",
+	"windowblinds",
+	"windowslice",
+	"wipeDown",
+	"wipeLeft",
+	"wipeRight",
+	"wipeUp",
+] as const
+
+export type TransitionName = typeof transitionNames[number]
+
+export type Transition = {
+	name: TransitionName
+	label: string
+}
+
+export interface TransitionAction {
+	(duration: number): Item.Transition
+}
+
+export type TransitionActions = {
+	[TName in TransitionName]: TransitionAction
+}
+
+export const transitionRegistry = Object.fromEntries(
+	transitionNames.map(name => [name, {
+		name,
+		label: labelizeTransitionName(name),
+	}])
+) as Record<TransitionName, Transition>
+
+export const transitions = transitionRegistry
+
+function labelizeTransitionName(name: string) {
+	return name
+		.replace(/_/g, " ")
+		.replace(/([a-z])([A-Z])/g, "$1 $2")
+		.replace(/\b\w/g, letter => letter.toUpperCase())
+}

package/s/timeline/parts/waveform/parts/collect.ts

@@ -0,0 +1,72 @@
+
+import {Driver} from "../../../../driver/driver.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export const PEAK_LEVELS = [2048, 1024, 512, 256, 128, 64, 32] as const
+
+export async function collectPeakLevels(driver: Driver, source: DecoderSource) {
+	const duration = (await driver.getAudioDuration(source)) ?? 0
+	const readable = driver.decodeAudio({source}).readable
+	const finestSamplesPerPeak = PEAK_LEVELS[PEAK_LEVELS.length - 1]
+	const finestPeaks: number[] = []
+
+	let currentMax = 0
+	let sampleCount = 0
+	let sampleRate = 0
+
+	for await (const audioData of readable) {
+		sampleRate ||= audioData.sampleRate
+
+		const frames = audioData.numberOfFrames
+		const plane = new Float32Array(frames)
+		audioData.copyTo(plane, {planeIndex: 0})
+
+		for (let i = 0; i < plane.length; i++) {
+			const amplitude = Math.abs(plane[i]!)
+			if (amplitude > currentMax) currentMax = amplitude
+
+			sampleCount++
+			if (sampleCount >= finestSamplesPerPeak) {
+				finestPeaks.push(currentMax)
+				currentMax = 0
+				sampleCount = 0
+			}
+		}
+
+		audioData.close()
+	}
+
+	if (sampleCount > 0) finestPeaks.push(currentMax)
+
+	const base = new Float32Array(finestPeaks)
+	const levels = PEAK_LEVELS.map(samplesPerPeak => {
+		const factor = Math.max(1, Math.round(samplesPerPeak / finestSamplesPerPeak))
+		const peaks = factor === 1 ? base : downsampleMax(base, factor)
+		return {
+			samplesPerPeak,
+			peaks,
+			peaksPerSecond: sampleRate > 0 ? sampleRate / samplesPerPeak : 0,
+		}
+	})
+
+	return {duration, levels}
+}
+
+function downsampleMax(peaks: Float32Array, factor: number) {
+	const downsampled = new Float32Array(Math.ceil(peaks.length / factor))
+
+	for (let i = 0; i < downsampled.length; i++) {
+		let maxPeak = 0
+		const start = i * factor
+		const end = Math.min(start + factor, peaks.length)
+
+		for (let j = start; j < end; j++) {
+			if (peaks[j]! > maxPeak) maxPeak = peaks[j]!
+		}
+
+		downsampled[i] = maxPeak
+	}
+
+	return downsampled
+}
+

package/s/timeline/parts/waveform/parts/render.ts

@@ -0,0 +1,45 @@
+
+export function renderTile(
+	peaks: Float32Array,
+	opts: {
+		width: number
+		height: number
+		color: string
+	}
+) {
+	const dpr = typeof window !== "undefined" ? window.devicePixelRatio || 1 : 1
+	const canvas = document.createElement("canvas")
+	canvas.width = opts.width * dpr
+	canvas.height = opts.height * dpr
+	canvas.style.width = `${opts.width}px`
+	canvas.style.height = `${opts.height}px`
+
+	const ctx = canvas.getContext("2d")
+	if (!ctx) return canvas
+
+	ctx.scale(dpr, dpr)
+	ctx.fillStyle = opts.color
+
+	const centerY = opts.height / 2
+	const columns = Math.max(1, opts.width)
+	const peaksPerPixel = peaks.length / columns
+
+	for (let px = 0; px < columns; px++) {
+		const startIndex = Math.floor(px * peaksPerPixel)
+		const endIndex = Math.max(
+			startIndex + 1,
+			Math.floor((px + 1) * peaksPerPixel)
+		)
+
+		let maxPeak = 0
+		for (let i = startIndex; i < endIndex && i < peaks.length; i++) {
+			if (peaks[i]! > maxPeak) maxPeak = peaks[i]!
+		}
+
+		const barHeight = maxPeak * opts.height
+		ctx.fillRect(px, centerY - barHeight / 2, 1, barHeight)
+	}
+
+	return canvas
+}
+

package/s/timeline/parts/waveform/parts/types.ts

@@ -0,0 +1,24 @@
+export interface WaveformTileData {
+	startTime: number
+	endTime: number
+	peaks: Float32Array
+	canvas: HTMLCanvasElement
+}
+
+export interface WaveformOptions {
+	tileSize?: number
+	zoom?: number
+	tileWidth?: number
+	tileHeight?: number
+	preloadMargin?: number
+	color?: string
+	onChange?: (tiles: WaveformTileData[]) => void
+}
+
+export type WaveformTimeRange = [start: number, end: number]
+
+export type WaveformPeakLevel = {
+	samplesPerPeak: number
+	peaks: Float32Array
+	peaksPerSecond: number
+}

package/s/timeline/parts/waveform/waveform.ts

@@ -0,0 +1,161 @@
+
+import {renderTile} from "./parts/render.js"
+import {Driver} from "../../../driver/driver.js"
+import {collectPeakLevels} from "./parts/collect.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {WaveformOptions, WaveformPeakLevel, WaveformTileData, WaveformTimeRange} from "./parts/types.js"
+
+const MAX_TILE_WIDTH = 4096
+
+export class Waveform {
+	#tiles = new Map<number, WaveformTileData>()
+	#activeRange: WaveformTimeRange = [0, 0]
+	#visibleRange: WaveformTimeRange = [0, 0]
+
+	#zoom
+	#levels
+	#onChange
+	#updateQueued = false
+
+	readonly color
+	readonly duration
+	readonly tileSize
+	readonly tileHeight
+	readonly preloadMargin
+
+	private constructor(levels: WaveformPeakLevel[], duration: number, options: WaveformOptions) {
+		this.#levels = levels
+		this.duration = duration
+		this.tileSize = options.tileSize ?? 1
+		this.#zoom = options.zoom ?? ((options.tileWidth ?? 256) / this.tileSize)
+		this.tileHeight = options.tileHeight ?? 96
+		this.preloadMargin = options.preloadMargin ?? 2
+		this.color = options.color ?? "rgb(3, 148, 129)"
+		this.#onChange = options.onChange
+	}
+
+	static async init(driver: Driver, source: DecoderSource, options: WaveformOptions = {}) {
+		const {duration, levels} = await collectPeakLevels(driver, source)
+		return new Waveform(levels, duration, options)
+	}
+
+	/** Waveform render density in pixels per second. */
+	set zoom(pixelsPerSecond: number) {
+		const next = Math.max(1, pixelsPerSecond)
+		if (next === this.#zoom)
+			return
+
+		this.#zoom = next
+		this.#tiles.clear()
+		this.#queueUpdate()
+	}
+
+	get zoom() {
+		return this.#zoom
+	}
+
+	get range() {
+		return this.#visibleRange
+	}
+
+	#computeActiveRange([start, end]: WaveformTimeRange, margin = 1): WaveformTimeRange {
+		const visibleSize = end - start
+		return [
+			Math.max(0, start - visibleSize * margin),
+			Math.min(this.duration, end + visibleSize * margin),
+		]
+	}
+
+	set range(visibleRange: WaveformTimeRange) {
+		this.#visibleRange = visibleRange
+
+		const [visibleStart, visibleEnd] = visibleRange
+		const visibleSize = visibleEnd - visibleStart
+		const [activeStart, activeEnd] = this.#activeRange
+
+		const leftBuffered = activeStart > 0
+		const rightBuffered = activeEnd < this.duration
+		const leftSettled = !leftBuffered || visibleStart >= activeStart + visibleSize
+		const rightSettled = !rightBuffered || visibleEnd <= activeEnd - visibleSize
+
+		if (leftSettled && rightSettled) return
+
+		this.#activeRange = this.#computeActiveRange(visibleRange, this.preloadMargin)
+		this.#queueUpdate()
+	}
+
+	#queueUpdate() {
+		if (this.#updateQueued) return
+		this.#updateQueued = true
+
+		queueMicrotask(() => {
+			this.#updateQueued = false
+			this.#generateTiles()
+		})
+	}
+
+	#generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededStarts = new Set<number>()
+		const level = this.#levelForZoom()
+
+		const firstStart = Math.max(0, Math.floor(rangeStart / this.tileSize) * this.tileSize)
+		const lastStart = Math.min(this.duration, rangeEnd)
+
+		for (let startTime = firstStart; startTime <= lastStart; startTime += this.tileSize) {
+			neededStarts.add(startTime)
+		}
+
+		for (const startTime of neededStarts) {
+			if (!this.#tiles.has(startTime)) {
+				const endTime = Math.min(startTime + this.tileSize, this.duration)
+				this.#tiles.set(startTime, this.#buildTileData(startTime, endTime, level))
+			}
+		}
+
+		for (const startTime of this.#tiles.keys()) {
+			if (!neededStarts.has(startTime)) this.#tiles.delete(startTime)
+		}
+
+		this.#emit()
+	}
+
+	#buildTileData(startTime: number, endTime: number, level: WaveformPeakLevel): WaveformTileData {
+		const peaks = this.#slicePeaks(level, startTime, endTime)
+		return {
+			startTime,
+			endTime,
+			peaks,
+			canvas: renderTile(peaks, {
+				width: this.#tilePixelWidth(startTime, endTime),
+				height: this.tileHeight,
+				color: this.color,
+			}),
+		}
+	}
+
+	#levelForZoom() {
+		return this.#levels.find(level => level.peaksPerSecond >= this.#zoom)
+			?? this.#levels[this.#levels.length - 1]!
+	}
+
+	#slicePeaks(level: WaveformPeakLevel, startTime: number, endTime: number) {
+		if (!level.peaksPerSecond) return new Float32Array()
+		const from = Math.max(0, Math.floor(startTime * level.peaksPerSecond))
+		const to = Math.max(from + 1, Math.min(level.peaks.length, Math.ceil(endTime * level.peaksPerSecond)))
+		return level.peaks.slice(from, to)
+	}
+
+	#tilePixelWidth(startTime: number, endTime: number) {
+		return Math.min(MAX_TILE_WIDTH, Math.max(1, Math.ceil((endTime - startTime) * this.#zoom)))
+	}
+
+	#emit() {
+		if (!this.#onChange) return
+		this.#onChange([...this.#tiles.values()].sort((a, b) => a.startTime - b.startTime))
+	}
+
+	getTiles() {
+		return this.#tiles
+	}
+}

package/s/timeline/renderers/export/parts/audio-gain.ts

@@ -0,0 +1,17 @@
+// this is mutating fn
+export const applyGainToPlanar = (
+	planes: Float32Array[],
+	gain: number
+) => {
+	if (gain === 1)
+		return
+	// planes
+
+	for (const plane of planes) {
+		for (let i = 0; i < plane.length; i++) {
+			plane[i] *= gain
+		}
+	}
+
+	// return planes
+}