@omnimedia/omnitool 1.1.0-9 → 1.1.0-90

package/s/timeline/renderers/parts/samplers/visual/parts/sequence.ts

@@ -0,0 +1,111 @@
+
+import {sampleVisual} from "./sample.js"
+import {SampleContext} from "./types.js"
+import {sampleTransition} from "./transition.js"
+import {ms, Ms} from "../../../../../../units/ms.js"
+import {Item, Kind} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+import {AncestorAt, computeItemDuration} from "../../../handy.js"
+
+export async function sampleSequence(
+	ctx: SampleContext,
+	seq: Item.Sequence,
+	time: Ms,
+	ancestors: AncestorAt[]
+): Promise<Layer[]> {
+	const state = sampleSequenceAt(ctx, seq, time)
+
+	if (!state) return []
+
+	const nextAnc = [...ancestors, {item: seq, localTime: time}]
+
+	if (!state.isTransitioning) {
+		return sampleVisual(ctx, state.item, state.localTime, nextAnc)
+	}
+
+	return sampleTransition(
+		state.transition,
+		state.progress,
+		sampleVisual(ctx, state.outgoing, state.outgoingTime, nextAnc),
+		sampleVisual(ctx, state.incoming, state.incomingTime, nextAnc)
+	)
+}
+
+function sampleSequenceAt(
+	ctx: SampleContext,
+	seq: Item.Sequence,
+	time: Ms
+) {
+	const children = seq.childrenIds
+		.map(id => ctx.items.get(id))
+		.filter((i): i is Item.Any => !!i)
+
+	let cursor = ms(0)
+
+	for (let i = 0; i < children.length; i++) {
+		const currentItem = children[i]
+		if (currentItem.kind === Kind.Transition)
+			continue
+
+		const currentItemStart = cursor
+		const currentItemDuration = computeItemDuration(currentItem.id, ctx.timeline)
+		const currentItemEnd = ms(currentItemStart + currentItemDuration)
+
+		const next = children[i + 1]
+		const hasTransition = next?.kind === Kind.Transition
+
+		if (!hasTransition) {
+			if (time < currentItemEnd) {
+				return {
+					isTransitioning: false,
+					item: currentItem,
+					localTime: ms(time - currentItemStart)
+				} as const
+			}
+
+			cursor = currentItemEnd
+			continue
+		}
+
+		const transition = next as Item.Transition
+		const incoming = children[i + 2]
+
+		if (!incoming || incoming.kind === Kind.Transition) {
+			cursor = currentItemEnd
+			continue
+		}
+
+		const incomingItemDuration = computeItemDuration(incoming.id, ctx.timeline)
+		const overlap = Math.max(0, Math.min(transition.duration, currentItemDuration, incomingItemDuration))
+		const currentItemSoloEnd = ms(currentItemEnd - overlap)
+
+		if (time < currentItemSoloEnd) {
+			return {
+				isTransitioning: false,
+				item: currentItem,
+				localTime: ms(time - currentItemStart)
+			} as const
+		}
+
+		if (time < currentItemEnd) {
+			const inLocal = ms(time - currentItemSoloEnd)
+			const outLocal = ms(time - currentItemStart)
+
+			return {
+				isTransitioning: true,
+				incoming,
+				outgoing: currentItem,
+				outgoingTime: outLocal,
+				incomingTime: inLocal,
+				progress: overlap > 0 ? inLocal / overlap : 1,
+				transition
+			} as const
+		}
+
+		cursor = currentItemSoloEnd
+		i++
+	}
+
+	return null
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/sink.ts

@@ -0,0 +1,38 @@
+
+import {ALL_FORMATS, Input, VideoSampleSink} from "mediabunny"
+
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../../../driver/utils/load-decoder-source.js"
+
+type SinkState = {
+	input: Input
+	sink: VideoSampleSink | null
+}
+
+export class VideoSink {
+	readonly #sinks = new Map<string, SinkState>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource
+	) {}
+
+	async getSink(hash: string) {
+		const existing = this.#sinks.get(hash)
+
+		if (existing)
+			return existing.sink
+
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(this.resolveMedia(hash)),
+		})
+
+		const videoTrack = await input.getPrimaryVideoTrack()
+		const canDecodeVideo = !!videoTrack && await videoTrack.canDecode()
+		const sink = canDecodeVideo && videoTrack ? new VideoSampleSink(videoTrack) : null
+
+		this.#sinks.set(hash, {input, sink})
+
+		return sink
+	}
+}

package/s/timeline/renderers/parts/samplers/visual/parts/transition.ts

@@ -0,0 +1,28 @@
+
+import {Item} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+
+export async function sampleTransition(
+	item: Item.Transition,
+	progress: number,
+	p1: Promise<Layer[]>,
+	p2: Promise<Layer[]>
+): Promise<Layer[]> {
+	const [l1, l2] = await Promise.all([p1, p2])
+	const f1 = l1.find(l => l.kind === "image")?.frame
+	const f2 = l2.find(l => l.kind === "image")?.frame
+
+	const rest = [
+		...l1.filter(l => l.kind !== "image"),
+		...l2.filter(l => l.kind !== "image")
+	]
+
+	return f1 && f2 ? [{
+		id: item.id,
+		kind: "transition",
+		name: "circle",
+		progress,
+		from: f1,
+		to: f2
+	}, ...rest] : rest
+}

package/s/timeline/renderers/parts/samplers/visual/parts/types.ts

@@ -0,0 +1,10 @@
+
+import {VideoSampler} from "./defaults.js"
+import {Item} from "../../../../../parts/item.js"
+import {TimelineFile} from "../../../../../parts/basics.js"
+
+export type SampleContext = {
+	videoSampler: VideoSampler
+	timeline: TimelineFile
+	items: Map<number, Item.Any>
+}

package/s/timeline/renderers/parts/samplers/visual/sampler.ts

@@ -0,0 +1,28 @@
+
+import {VideoSink} from "./parts/sink.js"
+import {sampleVisual} from "./parts/sample.js"
+import {Ms} from "../../../../../units/ms.js"
+import {TimelineFile} from "../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../driver/fns/schematic.js"
+import {createDefaultVideoSampler, VideoSampler} from "./parts/defaults.js"
+
+export function createVisualSampler(
+	resolveMedia: (hash: string) => DecoderSource,
+	sampleVideo?: VideoSampler
+) {
+	const sink = new VideoSink(resolveMedia)
+	const videoSampler = sampleVideo ?? createDefaultVideoSampler(sink)
+
+	return {
+		async sample(timeline: TimelineFile, timecode: Ms) {
+			const items = new Map(timeline.items.map(item => [item.id, item]))
+			const root = items.get(timeline.rootId)
+
+			if (!root)
+				return []
+
+			return sampleVisual({videoSampler, timeline, items}, root, timecode, [])
+		}
+	}
+}
+

package/s/timeline/renderers/parts/schedulers.ts

@@ -0,0 +1,96 @@
+
+import {Pub, pub} from "@e280/stz"
+
+import {ms, Ms} from "../../../units/ms.js"
+import {fps, Fps} from "../../../units/fps.js"
+
+export type RealtimeGenerator = {
+	play(): void
+	pause(): void
+	setFPS(v: Fps): void
+	isPlaying(): boolean
+	ticks(): AsyncGenerator<void>
+	onTick: Pub<[]>
+}
+
+export const realtime = (): RealtimeGenerator => {
+
+	let playing = false
+	let frameRate = fps(60)
+	let frameDuration = 1000 / frameRate
+
+	let lastNow = 0
+	let lastComposite = 0
+
+	const onTick = pub()
+	let resolveTick: (() => void) | null = null
+
+	const loop = (now: number) => {
+		requestAnimationFrame(loop)
+
+		if (!playing) return
+
+		lastNow = now
+
+		while (now - lastComposite >= frameDuration) {
+			lastComposite += frameDuration
+
+			resolveTick?.()
+			resolveTick = null
+			onTick()
+		}
+	}
+
+	async function* ticks(): AsyncGenerator<void> {
+		lastNow = performance.now()
+		lastComposite = lastNow
+		requestAnimationFrame(loop)
+
+		while (true) {
+			await new Promise<void>(r => resolveTick = r)
+			yield
+		}
+
+	}
+
+	return {
+		play() {
+			if (playing) return
+			playing = true
+			lastNow = performance.now()
+			lastComposite = lastNow
+		},
+		pause() {
+			playing = false
+		},
+		setFPS(v: Fps) {
+			frameRate = v
+			frameDuration = 1000 / frameRate
+		},
+		isPlaying() {
+			return playing
+		},
+		ticks,
+		onTick
+	}
+}
+
+export type FixedStepOptions = {
+	fps: Fps
+	duration: Ms
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: Ms, index: number) => Promise<void> | void
+) => {
+	const dt = ms(1000 / opts.fps)
+	const durationInSeconds = opts.duration / 1000
+	const total = Math.ceil(durationInSeconds * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		const t = ms(i * dt)
+		await onFrame(t, i)
+	}
+}
+

package/s/timeline/renderers/player/parts/playback.ts

@@ -0,0 +1,213 @@
+
+import {Fps} from '../../../../units/fps.js'
+import {ms, Ms} from '../../../../units/ms.js'
+import {Driver} from '../../../../driver/driver.js'
+import {realtime} from '../../parts/schedulers.js'
+import {seconds} from '../../../../units/seconds.js'
+import {TimelineFile} from '../../../parts/basics.js'
+import {computeItemDuration} from '../../parts/handy.js'
+import {CursorVisualSampler, ReverseCursorVisualSampler} from '../../export/parts/cursor.js'
+import {DecoderSource} from '../../../../driver/fns/schematic.js'
+import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
+import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
+
+export class Playback {
+	audioSampler
+	seekVisualSampler
+	playVisualSampler: CursorVisualSampler | null = null
+	reversePlayVisualSampler: ReverseCursorVisualSampler | null = null
+
+	#playbackStart = ms(0)
+	#audioStartSec: number | null = null
+	#playbackRate = 1
+
+	#controller = realtime()
+	onTick = this.#controller.onTick
+
+	audioContext = new AudioContext({sampleRate: 48000})
+	audioGain = this.audioContext.createGain()
+	audioNodes = new Set<AudioBufferSourceNode>()
+	#audioAbort: AbortController | null = null
+
+	constructor(
+		private driver: Driver,
+		private timeline: TimelineFile,
+		private resolveMedia: (hash: string) => DecoderSource
+	) {
+		this.audioGain.connect(this.audioContext.destination)
+		this.audioGain.gain.value = 0.7 ** 2
+		this.seekVisualSampler = createVisualSampler(this.resolveMedia)
+		this.audioSampler = createAudioSampler(this.resolveMedia)
+		this.#samples()
+	}
+
+	update(timeline: TimelineFile) {
+		this.timeline = timeline
+	}
+
+	get isPlaying() {
+		return this.#controller.isPlaying()
+	}
+
+	async #samples() {
+		for await (const _ of this.#controller.ticks()) {
+			const time = this.currentTime
+			const layers = this.#playbackRate >= 0
+				? await this.playVisualSampler?.next(time) ?? []
+				: await this.reversePlayVisualSampler?.next(time) ?? []
+
+			const frame = await this.driver.composite(layers)
+			frame.close()
+
+			const hasEnded = this.#playbackRate >= 0
+				? time >= this.duration
+				: time <= 0
+
+			if (hasEnded)
+				this.pause()
+		}
+	}
+
+	async seek(time: Ms) {
+		this.pause()
+		this.#playbackStart = time
+		return await this.seekVisualSampler.sample(this.timeline, time)
+	}
+
+	async start() {
+		if(this.#controller.isPlaying())
+			return
+
+		await this.audioContext.resume()
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.audioContext.currentTime
+
+		this.#stopAudio()
+		this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+		this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+
+		this.#controller.play()
+		this.#startAudio()
+	}
+
+	pause() {
+		this.#playbackStart = this.currentTime
+		this.#controller.pause()
+		this.#stopAudio()
+
+		if (this.playVisualSampler) {
+			this.playVisualSampler.cancel()
+			this.playVisualSampler = null
+		}
+		if (this.reversePlayVisualSampler) {
+			this.reversePlayVisualSampler.cancel()
+			this.reversePlayVisualSampler = null
+		}
+
+	}
+
+	get duration() {
+		return computeItemDuration(
+			this.timeline.rootId,
+			this.timeline
+		)
+	}
+
+	get currentTime() {
+		if (!this.#controller.isPlaying() || this.#audioStartSec === null)
+			return this.#playbackStart
+
+		const elapsedMs = (this.audioContext.currentTime - this.#audioStartSec) * 1000
+		const current = this.#playbackStart + elapsedMs * this.#playbackRate
+		return ms(Math.max(0, Math.min(this.duration, current)))
+	}
+
+	get playbackRate() {
+		return this.#playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		if (!Number.isFinite(rate) || rate === 0)
+			throw new Error(`Invalid playback rate "${rate}".`)
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.#controller.isPlaying()
+			? this.audioContext.currentTime
+			: null
+		const wasReversed = this.#playbackRate < 0
+		this.#playbackRate = rate
+
+		if (this.#controller.isPlaying()) {
+			if (wasReversed && rate > 0) {
+				this.playVisualSampler?.cancel()
+				this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+			else if (!wasReversed && rate < 0) {
+				this.reversePlayVisualSampler?.cancel()
+				this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+
+			this.#syncAudio()
+		}
+	}
+
+	setFps(fps: Fps) {
+		this.#controller.setFPS(fps)
+	}
+
+	#syncAudio() {
+		this.#stopAudio()
+		this.#startAudio()
+	}
+
+	#stopAudio() {
+		this.#audioAbort?.abort()
+		this.#audioAbort = null
+
+		for (const node of this.audioNodes)
+			node.stop()
+
+		this.audioNodes.clear()
+	}
+
+	async #startAudio() {
+		if (this.#playbackRate !== 1)
+			return
+
+		const from = seconds(this.#playbackStart / 1000)
+		this.#audioAbort = new AbortController()
+		const signal = this.#audioAbort.signal
+
+		const ctx = this.audioContext
+
+		if (this.#audioStartSec === null)
+			return
+
+		for await (const {sample, timestamp} of this.audioSampler.sampleAudio(
+			this.timeline,
+			ms(from * 1000)
+		)) {
+
+			if (signal.aborted || !this.#controller.isPlaying())
+				return
+
+			while (timestamp - (ctx.currentTime - this.#audioStartSec + from) > 0.75)
+				await new Promise(r => setTimeout(r, 25))
+
+			const node = ctx.createBufferSource()
+			node.buffer = sample.toAudioBuffer()
+			node.connect(this.audioGain)
+			node.onended = () => this.audioNodes.delete(node)
+			this.audioNodes.add(node)
+
+			const startAt = this.#audioStartSec + timestamp - from
+
+			startAt >= ctx.currentTime
+				? node.start(startAt)
+				: node.start(ctx.currentTime, ctx.currentTime - startAt)
+
+		}
+	}
+}
+

package/s/timeline/renderers/player/player.ts

@@ -0,0 +1,86 @@
+
+import {ms} from "../../../units/ms.js"
+import {fps} from "../../../units/fps.js"
+import {Playback} from "./parts/playback.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	canvas: HTMLCanvasElement
+	playback: Playback
+
+	#pendingSeek: number | null = null
+	#flushTask: Promise<void> | null = null
+
+	constructor(
+		private driver: Driver,
+		resolveMedia: ResolveMedia,
+		timeline: TimelineFile
+	) {
+		this.playback = new Playback(driver, timeline, resolveMedia)
+		this.canvas = driver.compositor.pixi.renderer.canvas
+	}
+
+	async play() {
+		await this.playback.start()
+	}
+
+	pause() {
+		this.playback.pause()
+	}
+
+	seek(timeMs: number) {
+		this.#pendingSeek = timeMs
+		return this.#flushTask ??= this.#flushSeeks().finally(() => this.#flushTask = null)
+	}
+
+	setFPS(value: number) {
+		this.playback.setFps(fps(value))
+	}
+
+	get isSeeking() {
+		return this.#flushTask !== null
+	}
+
+	get isPlaying() {
+		return this.playback.isPlaying
+	}
+
+	get duration() {
+		return this.playback.duration
+	}
+
+	get currentTime() {
+		return this.playback.currentTime
+	}
+
+	get playbackRate() {
+		return this.playback.playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		this.playback.playbackRate = rate
+	}
+
+	/**
+	 call this whenever your timeline state changes
+	*/
+	async update(timeline: TimelineFile) {
+		this.playback.update(timeline)
+		if (!this.isPlaying)
+			await this.seek(this.currentTime)
+	}
+
+	async #flushSeeks() {
+		while (this.#pendingSeek !== null) {
+			const next = this.#pendingSeek
+			this.#pendingSeek = null
+			const layers = await this.playback.seek(ms(next))
+			const frame = await this.driver.composite(layers)
+			frame.close()
+		}
+	}
+}