@omnimedia/omnitool 1.1.0-8 → 1.1.0-81

package/s/timeline/renderers/parts/samplers/audio/sampler.ts

@@ -0,0 +1,35 @@
+
+import {AudioSample} from "mediabunny"
+
+import {itemsFrom} from "../../handy.js"
+import {initStreams} from "./parts/init.js"
+import {AudioSinkPool} from "./parts/sink.js"
+import {Ms} from "../../../../../units/ms.js"
+import {findEarliestStream} from "./parts/find.js"
+import {TimelineFile} from "../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../driver/fns/schematic.js"
+
+export function createAudioSampler(resolveMedia: (hash: string) => DecoderSource) {
+	const sinkPool = new AudioSinkPool(resolveMedia)
+
+	return {
+		async *sampleAudio(timeline: TimelineFile, from: Ms) {
+			const items = itemsFrom({timeline, from})
+			const streams = await initStreams(sinkPool, items, from)
+
+			while (streams.length > 0) {
+				const {stream, index} = findEarliestStream(streams)
+
+				yield stream.output()
+
+				const advancing = await stream.advance()
+
+				if (!advancing) {
+					streams.splice(index, 1)
+				}
+			}
+
+		}
+	}
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/defaults.ts

@@ -0,0 +1,16 @@
+
+import {VideoSink} from "./sink.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Item} from "../../../../../parts/item.js"
+
+export type VideoSampler = (item: Item.Video, time: Ms) => Promise<VideoFrame | undefined>
+
+export function createDefaultVideoSampler(sink: VideoSink): VideoSampler {
+	return async (item, time) => {
+		const s = await sink.getSink(item.mediaHash)
+		const sample = await s?.getSample(time / 1000)
+		const frame = sample?.toVideoFrame()
+		sample?.close()
+		return frame ?? undefined
+	}
+}

package/s/timeline/renderers/parts/samplers/visual/parts/sample.ts

@@ -0,0 +1,69 @@
+
+import {SampleContext} from "./types.js"
+import {sampleSequence} from "./sequence.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Item, Kind, SpatialItem} from "../../../../../parts/item.js"
+import {FilterSpec, Layer} from "../../../../../../driver/fns/schematic.js"
+import {AncestorAt, computeOpacity, computeWorldMatrix} from "../../../handy.js"
+
+export async function sampleVisual(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+	ancestors: AncestorAt[]
+): Promise<Layer[]> {
+	const matrix = computeWorldMatrix(ctx.items, ancestors, item, time)
+	const alpha = computeOpacity(ctx, item, time)
+	const crop = "spatialId" in item && item.spatialId
+		? (ctx.items.get(item.spatialId) as SpatialItem | undefined)?.crop
+		: undefined
+	const filters = "filterIds" in item && item.filterIds
+		? item.filterIds
+			.map(id => ctx.items.get(id) as Item.Filter | undefined)
+			.filter((filter): filter is Item.Filter => !!filter?.enabled)
+			.map(filter => ({type: filter.type, params: filter.params}) as FilterSpec)
+		: undefined
+
+	switch (item.kind) {
+		case Kind.Stack: {
+			const nextAnc = [...ancestors, {item, localTime: time}]
+
+			const layers = await Promise.all(
+				item.childrenIds
+					.map(id => ctx.items.get(id))
+					.filter((child): child is Item.Any => !!child)
+					.map(child => sampleVisual(ctx, child, time, nextAnc))
+			)
+
+			return layers.flat()
+		}
+
+		case Kind.Sequence:
+			return sampleSequence(ctx, item, time, ancestors)
+
+		case Kind.Video: {
+			if (time < 0 || time >= item.duration) return []
+
+			const frame = await ctx.videoSampler(item, time)
+			return frame ? [{kind: "image", frame, matrix, alpha, crop, filters, id: item.id}] : []
+		}
+
+		case Kind.Text: {
+			if (time < 0 || time >= item.duration) return []
+
+			const style = item.styleId
+				? (ctx.items.get(item.styleId) as Item.TextStyle)?.style
+				: undefined
+
+			return [{id: item.id, kind: "text", content: item.content, style, matrix, alpha, crop, filters}]
+		}
+
+		case Kind.Gap: {
+			return [{id: item.id, kind: "gap"}]
+		}
+
+		default:
+			return []
+	}
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/sequence.ts

@@ -0,0 +1,111 @@
+
+import {sampleVisual} from "./sample.js"
+import {SampleContext} from "./types.js"
+import {sampleTransition} from "./transition.js"
+import {ms, Ms} from "../../../../../../units/ms.js"
+import {Item, Kind} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+import {AncestorAt, computeItemDuration} from "../../../handy.js"
+
+export async function sampleSequence(
+	ctx: SampleContext,
+	seq: Item.Sequence,
+	time: Ms,
+	ancestors: AncestorAt[]
+): Promise<Layer[]> {
+	const state = sampleSequenceAt(ctx, seq, time)
+
+	if (!state) return []
+
+	const nextAnc = [...ancestors, {item: seq, localTime: time}]
+
+	if (!state.isTransitioning) {
+		return sampleVisual(ctx, state.item, state.localTime, nextAnc)
+	}
+
+	return sampleTransition(
+		state.transition,
+		state.progress,
+		sampleVisual(ctx, state.outgoing, state.outgoingTime, nextAnc),
+		sampleVisual(ctx, state.incoming, state.incomingTime, nextAnc)
+	)
+}
+
+function sampleSequenceAt(
+	ctx: SampleContext,
+	seq: Item.Sequence,
+	time: Ms
+) {
+	const children = seq.childrenIds
+		.map(id => ctx.items.get(id))
+		.filter((i): i is Item.Any => !!i)
+
+	let cursor = ms(0)
+
+	for (let i = 0; i < children.length; i++) {
+		const currentItem = children[i]
+		if (currentItem.kind === Kind.Transition)
+			continue
+
+		const currentItemStart = cursor
+		const currentItemDuration = computeItemDuration(currentItem.id, ctx.timeline)
+		const currentItemEnd = ms(currentItemStart + currentItemDuration)
+
+		const next = children[i + 1]
+		const hasTransition = next?.kind === Kind.Transition
+
+		if (!hasTransition) {
+			if (time < currentItemEnd) {
+				return {
+					isTransitioning: false,
+					item: currentItem,
+					localTime: ms(time - currentItemStart)
+				} as const
+			}
+
+			cursor = currentItemEnd
+			continue
+		}
+
+		const transition = next as Item.Transition
+		const incoming = children[i + 2]
+
+		if (!incoming || incoming.kind === Kind.Transition) {
+			cursor = currentItemEnd
+			continue
+		}
+
+		const incomingItemDuration = computeItemDuration(incoming.id, ctx.timeline)
+		const overlap = Math.max(0, Math.min(transition.duration, currentItemDuration, incomingItemDuration))
+		const currentItemSoloEnd = ms(currentItemEnd - overlap)
+
+		if (time < currentItemSoloEnd) {
+			return {
+				isTransitioning: false,
+				item: currentItem,
+				localTime: ms(time - currentItemStart)
+			} as const
+		}
+
+		if (time < currentItemEnd) {
+			const inLocal = ms(time - currentItemSoloEnd)
+			const outLocal = ms(time - currentItemStart)
+
+			return {
+				isTransitioning: true,
+				incoming,
+				outgoing: currentItem,
+				outgoingTime: outLocal,
+				incomingTime: inLocal,
+				progress: overlap > 0 ? inLocal / overlap : 1,
+				transition
+			} as const
+		}
+
+		cursor = currentItemSoloEnd
+		i++
+	}
+
+	return null
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/sink.ts

@@ -0,0 +1,38 @@
+
+import {ALL_FORMATS, Input, VideoSampleSink} from "mediabunny"
+
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../../../driver/utils/load-decoder-source.js"
+
+type SinkState = {
+	input: Input
+	sink: VideoSampleSink | null
+}
+
+export class VideoSink {
+	readonly #sinks = new Map<string, SinkState>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource
+	) {}
+
+	async getSink(hash: string) {
+		const existing = this.#sinks.get(hash)
+
+		if (existing)
+			return existing.sink
+
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(this.resolveMedia(hash)),
+		})
+
+		const videoTrack = await input.getPrimaryVideoTrack()
+		const canDecodeVideo = !!videoTrack && await videoTrack.canDecode()
+		const sink = canDecodeVideo && videoTrack ? new VideoSampleSink(videoTrack) : null
+
+		this.#sinks.set(hash, {input, sink})
+
+		return sink
+	}
+}

package/s/timeline/renderers/parts/samplers/visual/parts/transition.ts

@@ -0,0 +1,28 @@
+
+import {Item} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+
+export async function sampleTransition(
+	item: Item.Transition,
+	progress: number,
+	p1: Promise<Layer[]>,
+	p2: Promise<Layer[]>
+): Promise<Layer[]> {
+	const [l1, l2] = await Promise.all([p1, p2])
+	const f1 = l1.find(l => l.kind === "image")?.frame
+	const f2 = l2.find(l => l.kind === "image")?.frame
+
+	const rest = [
+		...l1.filter(l => l.kind !== "image"),
+		...l2.filter(l => l.kind !== "image")
+	]
+
+	return f1 && f2 ? [{
+		id: item.id,
+		kind: "transition",
+		name: "circle",
+		progress,
+		from: f1,
+		to: f2
+	}, ...rest] : rest
+}

package/s/timeline/renderers/parts/samplers/visual/parts/types.ts

@@ -0,0 +1,10 @@
+
+import {VideoSampler} from "./defaults.js"
+import {Item} from "../../../../../parts/item.js"
+import {TimelineFile} from "../../../../../parts/basics.js"
+
+export type SampleContext = {
+	videoSampler: VideoSampler
+	timeline: TimelineFile
+	items: Map<number, Item.Any>
+}

package/s/timeline/renderers/parts/samplers/visual/sampler.ts

@@ -0,0 +1,28 @@
+
+import {VideoSink} from "./parts/sink.js"
+import {sampleVisual} from "./parts/sample.js"
+import {Ms} from "../../../../../units/ms.js"
+import {TimelineFile} from "../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../driver/fns/schematic.js"
+import {createDefaultVideoSampler, VideoSampler} from "./parts/defaults.js"
+
+export function createVisualSampler(
+	resolveMedia: (hash: string) => DecoderSource,
+	sampleVideo?: VideoSampler
+) {
+	const sink = new VideoSink(resolveMedia)
+	const videoSampler = sampleVideo ?? createDefaultVideoSampler(sink)
+
+	return {
+		async sample(timeline: TimelineFile, timecode: Ms) {
+			const items = new Map(timeline.items.map(item => [item.id, item]))
+			const root = items.get(timeline.rootId)
+
+			if (!root)
+				return []
+
+			return sampleVisual({videoSampler, timeline, items}, root, timecode, [])
+		}
+	}
+}
+

package/s/timeline/renderers/parts/schedulers.ts

@@ -0,0 +1,96 @@
+
+import {Pub, pub} from "@e280/stz"
+
+import {ms, Ms} from "../../../units/ms.js"
+import {fps, Fps} from "../../../units/fps.js"
+
+export type RealtimeGenerator = {
+	play(): void
+	pause(): void
+	setFPS(v: Fps): void
+	isPlaying(): boolean
+	ticks(): AsyncGenerator<void>
+	onTick: Pub<[]>
+}
+
+export const realtime = (): RealtimeGenerator => {
+
+	let playing = false
+	let frameRate = fps(60)
+	let frameDuration = 1000 / frameRate
+
+	let lastNow = 0
+	let lastComposite = 0
+
+	const onTick = pub()
+	let resolveTick: (() => void) | null = null
+
+	const loop = (now: number) => {
+		requestAnimationFrame(loop)
+
+		if (!playing) return
+
+		lastNow = now
+
+		while (now - lastComposite >= frameDuration) {
+			lastComposite += frameDuration
+
+			resolveTick?.()
+			resolveTick = null
+			onTick()
+		}
+	}
+
+	async function* ticks(): AsyncGenerator<void> {
+		lastNow = performance.now()
+		lastComposite = lastNow
+		requestAnimationFrame(loop)
+
+		while (true) {
+			await new Promise<void>(r => resolveTick = r)
+			yield
+		}
+
+	}
+
+	return {
+		play() {
+			if (playing) return
+			playing = true
+			lastNow = performance.now()
+			lastComposite = lastNow
+		},
+		pause() {
+			playing = false
+		},
+		setFPS(v: Fps) {
+			frameRate = v
+			frameDuration = 1000 / frameRate
+		},
+		isPlaying() {
+			return playing
+		},
+		ticks,
+		onTick
+	}
+}
+
+export type FixedStepOptions = {
+	fps: Fps
+	duration: Ms
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: Ms, index: number) => Promise<void> | void
+) => {
+	const dt = ms(1000 / opts.fps)
+	const durationInSeconds = opts.duration / 1000
+	const total = Math.ceil(durationInSeconds * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		const t = ms(i * dt)
+		await onFrame(t, i)
+	}
+}
+

package/s/timeline/renderers/player/parts/playback.ts

@@ -0,0 +1,159 @@
+
+import {Fps} from '../../../../units/fps.js'
+import {ms, Ms} from '../../../../units/ms.js'
+import {Driver} from '../../../../driver/driver.js'
+import {realtime} from '../../parts/schedulers.js'
+import {TimelineFile} from '../../../parts/basics.js'
+import {computeItemDuration} from '../../parts/handy.js'
+import {seconds, Seconds} from '../../../../units/seconds.js'
+import {CursorVisualSampler} from '../../export/parts/cursor.js'
+import {DecoderSource} from '../../../../driver/fns/schematic.js'
+import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
+import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
+
+export class Playback {
+	audioSampler
+	seekVisualSampler
+	playVisualSampler: CursorVisualSampler | null = null
+
+	#playbackStart = ms(0)
+	#audioStartSec: number | null = null
+
+	#controller = realtime()
+	onTick = this.#controller.onTick
+
+	audioContext = new AudioContext({sampleRate: 48000})
+	audioGain = this.audioContext.createGain()
+	audioNodes = new Set<AudioBufferSourceNode>()
+	#audioAbort: AbortController | null = null
+
+	constructor(
+		private driver: Driver,
+		private timeline: TimelineFile,
+		private resolveMedia: (hash: string) => DecoderSource
+	) {
+		this.audioGain.connect(this.audioContext.destination)
+		this.audioGain.gain.value = 0.7 ** 2
+		this.seekVisualSampler = createVisualSampler(this.resolveMedia)
+		this.audioSampler = createAudioSampler(this.resolveMedia)
+		this.#samples()
+	}
+
+	update(timeline: TimelineFile) {
+		this.timeline = timeline
+	}
+
+	get isPlaying() {
+		return this.#controller.isPlaying()
+	}
+
+	async #samples() {
+		for await (const _ of this.#controller.ticks()) {
+			const layers = await this.playVisualSampler?.next(this.currentTime) ?? []
+
+			const frame = await this.driver.composite(layers)
+			frame.close()
+
+			if (this.currentTime >= this.duration)
+				this.pause()
+		}
+	}
+
+	async seek(time: Ms) {
+		this.pause()
+		this.#playbackStart = time
+		return await this.seekVisualSampler.sample(this.timeline, time)
+	}
+
+	async start() {
+		if(this.#controller.isPlaying())
+			return
+
+		await this.audioContext.resume()
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.audioContext.currentTime
+
+		this.#audioAbort?.abort()
+		this.#audioAbort = new AbortController()
+
+		for (const node of this.audioNodes)
+			node.stop()
+
+		this.audioNodes.clear()
+
+		this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+
+		this.#controller.play()
+		this.#startAudio(this.#audioAbort.signal, seconds(this.#playbackStart / 1000))
+	}
+
+	pause() {
+		this.#playbackStart = this.currentTime
+		this.#controller.pause()
+		this.#audioAbort?.abort()
+
+		for (const node of this.audioNodes)
+			node.stop()
+
+		this.audioNodes.clear()
+
+		if (this.playVisualSampler) {
+			this.playVisualSampler.cancel()
+			this.playVisualSampler = null
+		}
+
+	}
+
+	get duration() {
+		return computeItemDuration(
+			this.timeline.rootId,
+			this.timeline
+		)
+	}
+
+	get currentTime() {
+		if (!this.#controller.isPlaying() || this.#audioStartSec === null)
+			return this.#playbackStart
+
+		const elapsedMs = (this.audioContext.currentTime - this.#audioStartSec) * 1000
+		return ms(this.#playbackStart + elapsedMs)
+	}
+
+	setFps(fps: Fps) {
+		this.#controller.setFPS(fps)
+	}
+
+	async #startAudio(signal: AbortSignal, from: Seconds) {
+		const ctx = this.audioContext
+
+		if (this.#audioStartSec === null)
+			return
+
+		for await (const {sample, timestamp} of this.audioSampler.sampleAudio(
+			this.timeline,
+			ms(from * 1000)
+		)) {
+
+			if (signal.aborted || !this.#controller.isPlaying())
+				return
+
+			while (timestamp - (ctx.currentTime - this.#audioStartSec + from) > 0.75)
+				await new Promise(r => setTimeout(r, 25))
+
+			const node = ctx.createBufferSource()
+			node.buffer = sample.toAudioBuffer()
+			node.connect(this.audioGain)
+			node.onended = () => this.audioNodes.delete(node)
+			this.audioNodes.add(node)
+
+			const startAt = this.#audioStartSec + timestamp - from
+
+			startAt >= ctx.currentTime
+				? node.start(startAt)
+				: node.start(ctx.currentTime, ctx.currentTime - startAt)
+
+		}
+	}
+}
+

package/s/timeline/renderers/player/player.ts

@@ -0,0 +1,77 @@
+
+import {ms} from "../../../units/ms.js"
+import {fps} from "../../../units/fps.js"
+import {Playback} from "./parts/playback.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	canvas: HTMLCanvasElement
+	playback: Playback
+
+	#pendingSeek: number | null = null
+	#flushTask: Promise<void> | null = null
+
+	constructor(
+		private driver: Driver,
+		resolveMedia: ResolveMedia,
+		timeline: TimelineFile
+	) {
+		this.playback = new Playback(driver, timeline, resolveMedia)
+		this.canvas = driver.compositor.pixi.renderer.canvas
+	}
+
+	async play() {
+		await this.playback.start()
+	}
+
+	pause() {
+		this.playback.pause()
+	}
+
+	seek(timeMs: number) {
+		this.#pendingSeek = timeMs
+		return this.#flushTask ??= this.#flushSeeks().finally(() => this.#flushTask = null)
+	}
+
+	setFPS(value: number) {
+		this.playback.setFps(fps(value))
+	}
+
+	get isSeeking() {
+		return this.#flushTask !== null
+	}
+
+	get isPlaying() {
+		return this.playback.isPlaying
+	}
+
+	get duration() {
+		return this.playback.duration
+	}
+
+	get currentTime() {
+		return this.playback.currentTime
+	}
+
+	/**
+	 call this whenever your timeline state changes
+	*/
+	update(timeline: TimelineFile) {
+		this.playback.update(timeline)
+	}
+
+	async #flushSeeks() {
+		while (this.#pendingSeek !== null) {
+			const next = this.#pendingSeek
+			this.#pendingSeek = null
+			const layers = await this.playback.seek(ms(next))
+			const frame = await this.driver.composite(layers)
+			frame.close()
+		}
+	}
+}
+