@omnimedia/omnitool 1.1.0-1 → 1.1.0-100

package/s/timeline/renderers/parts/samplers/visual/parts/sample.ts

@@ -0,0 +1,99 @@
+
+import {SampleContext} from "./types.js"
+import {sampleSequence} from "./sequence.js"
+import {Ms} from "../../../../../../units/ms.js"
+import {Item, Kind} from "../../../../../parts/item.js"
+import {segmentTranscript} from "../../../../../parts/captions.js"
+import {FilterSpec, Layer} from "../../../../../../driver/fns/schematic.js"
+import {AncestorAt, computeOpacity, computeWorldMatrix} from "../../../handy.js"
+
+export async function sampleVisual(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+	ancestors: AncestorAt[],
+	options: {
+		/** Allow sampling trimmed media handles. */
+		allowHandles?: boolean
+	} = {}
+): Promise<Layer[]> {
+	const matrix = computeWorldMatrix(ctx.items, ancestors, item, time)
+	const alpha = computeOpacity(ctx, item, time)
+	const crop = "spatialId" in item && item.spatialId
+		? (ctx.items.get(item.spatialId) as Item.Spatial | undefined)?.crop
+		: undefined
+	const filters = "filterIds" in item && item.filterIds
+		? item.filterIds
+			.map(id => ctx.items.get(id) as Item.Filter | undefined)
+			.filter((filter): filter is Item.Filter => !!filter?.enabled)
+			.map(filter => ({type: filter.type, params: filter.params}) as FilterSpec)
+		: undefined
+
+	switch (item.kind) {
+		case Kind.Stack: {
+			const nextAnc = [...ancestors, {item, localTime: time}]
+
+			const layers = await Promise.all(
+				item.childrenIds
+					.map(id => ctx.items.get(id))
+					.filter((child): child is Item.Any => !!child)
+					.map(child => sampleVisual(ctx, child, time, nextAnc, options))
+			)
+
+			return layers.flat()
+		}
+
+		case Kind.Sequence:
+			return sampleSequence(ctx, item, time, ancestors)
+
+		case Kind.Video: {
+			if (!options.allowHandles && (time < 0 || time >= item.duration)) return []
+
+			const frame = await ctx.videoSampler(item, time)
+			return frame ? [{kind: "image", frame, matrix, alpha, crop, filters, id: item.id}] : []
+		}
+
+		case Kind.Image: {
+			if (!options.allowHandles && (time < 0 || time >= item.duration)) return []
+
+			const frame = await ctx.imageSampler(item, time)
+			return frame ? [{kind: "image", frame, matrix, alpha, crop, filters, id: item.id}] : []
+		}
+
+		case Kind.Text: {
+			if (!options.allowHandles && (time < 0 || time >= item.duration)) return []
+
+			const style = item.styleId
+				? (ctx.items.get(item.styleId) as Item.TextStyle)?.style
+				: undefined
+
+			return [{id: item.id, kind: "text", content: item.content, style, matrix, alpha, crop, filters}]
+		}
+
+		case Kind.Caption: {
+			if (!options.allowHandles && (time < 0 || time >= item.duration)) return []
+
+			const transcriptTime = item.start + time
+			const segment = segmentTranscript(item.transcript, item).find(segment => {
+				const [start, end] = segment.timestamp
+				return transcriptTime >= start && transcriptTime < end
+			})
+			if (!segment)
+				return []
+
+			const style = item.styleId
+				? (ctx.items.get(item.styleId) as Item.TextStyle)?.style
+				: undefined
+
+			return [{id: item.id, kind: "text", content: segment.text, style, matrix, alpha, crop, filters}]
+		}
+
+		case Kind.Gap: {
+			return [{id: item.id, kind: "gap"}]
+		}
+
+		default:
+			return []
+	}
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/sequence.ts

@@ -0,0 +1,97 @@
+
+import {sampleVisual} from "./sample.js"
+import {SampleContext} from "./types.js"
+import {sampleTransition} from "./transition.js"
+import {ms, Ms} from "../../../../../../units/ms.js"
+import {Item, Kind} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+import {AncestorAt, computeItemDuration} from "../../../handy.js"
+
+export async function sampleSequence(
+	ctx: SampleContext,
+	seq: Item.Sequence,
+	time: Ms,
+	ancestors: AncestorAt[]
+): Promise<Layer[]> {
+	const state = sampleSequenceAt(ctx, seq, time)
+	if (!state) return []
+
+	const nextAnc = [...ancestors, {item: seq, localTime: time}]
+
+	if (!state.isTransitioning)
+		return sampleVisual(ctx, state.item, state.localTime, nextAnc)
+
+	return sampleTransition(
+		state.transition,
+		state.progress,
+		sampleTransitionHandle(ctx, state.outgoing, state.outgoingTime, nextAnc),
+		sampleTransitionHandle(ctx, state.incoming, state.incomingTime, nextAnc)
+	)
+}
+
+function sampleSequenceAt(ctx: SampleContext, seq: Item.Sequence, time: Ms) {
+	const children = seq.childrenIds
+		.map(id => ctx.items.get(id))
+		.filter((i): i is Item.Any => !!i)
+
+	let cursor = ms(0)
+
+	for (let index = 0; index < children.length; index++) {
+		const child = children[index]
+		const duration = computeItemDuration(child.id, ctx.timeline)
+		const end = ms(cursor + duration)
+
+		if (duration <= 0) {
+			cursor = end
+			continue
+		}
+
+		if (time < cursor || time >= end) {
+			cursor = end
+			continue
+		}
+
+		const localTime = ms(time - cursor)
+
+		if (child.kind !== Kind.Transition)
+			return {isTransitioning: false, item: child, localTime} as const
+
+		const outgoing = children[index - 1]
+		const incoming = children[index + 1]
+		const isValidTransition =
+			outgoing && incoming &&
+			outgoing.kind !== Kind.Transition &&
+			incoming.kind !== Kind.Transition
+
+		if (!isValidTransition) return null
+
+		const outgoingDuration = computeItemDuration(outgoing.id, ctx.timeline)
+		return {
+			isTransitioning: true,
+			incoming,
+			outgoing,
+			outgoingTime: ms(outgoingDuration + localTime),
+			incomingTime: ms(localTime - duration),
+			progress: localTime / duration,
+			transition: child
+		} as const
+	}
+
+	return null
+}
+
+async function sampleTransitionHandle(
+	ctx: SampleContext,
+	item: Item.Any,
+	time: Ms,
+	ancestors: AncestorAt[]
+) {
+	const handleLayers = await sampleVisual(ctx, item, time, ancestors, {allowHandles: true})
+	if (handleLayers.some(layer => layer.kind === "image"))
+		return handleLayers
+
+	const duration = computeItemDuration(item.id, ctx.timeline)
+	const clampedTime = ms(Math.max(0, Math.min(time, duration > 0 ? duration - 1 : 0)))
+	return sampleVisual(ctx, item, clampedTime, ancestors)
+}
+

package/s/timeline/renderers/parts/samplers/visual/parts/transition.ts

@@ -0,0 +1,28 @@
+
+import {Item} from "../../../../../parts/item.js"
+import {Layer} from "../../../../../../driver/fns/schematic.js"
+
+export async function sampleTransition(
+	item: Item.Transition,
+	progress: number,
+	p1: Promise<Layer[]>,
+	p2: Promise<Layer[]>
+): Promise<Layer[]> {
+	const [l1, l2] = await Promise.all([p1, p2])
+	const f1 = l1.find(l => l.kind === "image")?.frame
+	const f2 = l2.find(l => l.kind === "image")?.frame
+
+	const rest = [
+		...l1.filter(l => l.kind !== "image"),
+		...l2.filter(l => l.kind !== "image")
+	]
+
+	return f1 && f2 ? [{
+		id: item.id,
+		kind: "transition",
+		name: item.name,
+		progress,
+		from: f1,
+		to: f2
+	}, ...rest] : rest
+}

package/s/timeline/renderers/parts/samplers/visual/parts/types.ts

@@ -0,0 +1,11 @@
+
+import {ImageSampler, VideoSampler} from "./defaults.js"
+import {Item} from "../../../../../parts/item.js"
+import {TimelineFile} from "../../../../../parts/basics.js"
+
+export type SampleContext = {
+	imageSampler: ImageSampler
+	videoSampler: VideoSampler
+	timeline: TimelineFile
+	items: Map<number, Item.Any>
+}

package/s/timeline/renderers/parts/samplers/visual/parts/video-sink.ts

@@ -0,0 +1,38 @@
+
+import {ALL_FORMATS, Input, VideoSampleSink} from "mediabunny"
+
+import {DecoderSource} from "../../../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../../../driver/utils/load-decoder-source.js"
+
+type SinkState = {
+	input: Input
+	sink: VideoSampleSink | null
+}
+
+export class VideoSink {
+	readonly #sinks = new Map<string, SinkState>()
+
+	constructor(
+		private resolveMedia: (hash: string) => DecoderSource
+	) {}
+
+	async getSink(hash: string) {
+		const existing = this.#sinks.get(hash)
+
+		if (existing)
+			return existing.sink
+
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(this.resolveMedia(hash)),
+		})
+
+		const videoTrack = await input.getPrimaryVideoTrack()
+		const canDecodeVideo = !!videoTrack && await videoTrack.canDecode()
+		const sink = canDecodeVideo && videoTrack ? new VideoSampleSink(videoTrack) : null
+
+		this.#sinks.set(hash, {input, sink})
+
+		return sink
+	}
+}

package/s/timeline/renderers/parts/samplers/visual/sampler.ts

@@ -0,0 +1,31 @@
+
+import {VideoSink} from "./parts/video-sink.js"
+import {ImageSink} from "./parts/image-sink.js"
+import {sampleVisual} from "./parts/sample.js"
+import {Ms} from "../../../../../units/ms.js"
+import {TimelineFile} from "../../../../parts/basics.js"
+import {DecoderSource} from "../../../../../driver/fns/schematic.js"
+import {createDefaultImageSampler, createDefaultVideoSampler, VideoSampler} from "./parts/defaults.js"
+
+export function createVisualSampler(
+	resolveMedia: (hash: string) => DecoderSource,
+	sampleVideo?: VideoSampler
+) {
+	const imageSink = new ImageSink(resolveMedia)
+	const videoSink = new VideoSink(resolveMedia)
+	const imageSampler = createDefaultImageSampler(imageSink)
+	const videoSampler = sampleVideo ?? createDefaultVideoSampler(videoSink)
+
+	return {
+		async sample(timeline: TimelineFile, timecode: Ms) {
+			const items = new Map(timeline.items.map(item => [item.id, item]))
+			const root = items.get(timeline.rootId)
+
+			if (!root)
+				return []
+
+			return sampleVisual({imageSampler, videoSampler, timeline, items}, root, timecode, [])
+		}
+	}
+}
+

package/s/timeline/renderers/parts/schedulers.ts

@@ -0,0 +1,96 @@
+
+import {Pub, pub} from "@e280/stz"
+
+import {ms, Ms} from "../../../units/ms.js"
+import {fps, Fps} from "../../../units/fps.js"
+
+export type RealtimeGenerator = {
+	play(): void
+	pause(): void
+	setFPS(v: Fps): void
+	isPlaying(): boolean
+	ticks(): AsyncGenerator<void>
+	onTick: Pub<[]>
+}
+
+export const realtime = (): RealtimeGenerator => {
+
+	let playing = false
+	let frameRate = fps(60)
+	let frameDuration = 1000 / frameRate
+
+	let lastNow = 0
+	let lastComposite = 0
+
+	const onTick = pub()
+	let resolveTick: (() => void) | null = null
+
+	const loop = (now: number) => {
+		requestAnimationFrame(loop)
+
+		if (!playing) return
+
+		lastNow = now
+
+		while (now - lastComposite >= frameDuration) {
+			lastComposite += frameDuration
+
+			resolveTick?.()
+			resolveTick = null
+			onTick()
+		}
+	}
+
+	async function* ticks(): AsyncGenerator<void> {
+		lastNow = performance.now()
+		lastComposite = lastNow
+		requestAnimationFrame(loop)
+
+		while (true) {
+			await new Promise<void>(r => resolveTick = r)
+			yield
+		}
+
+	}
+
+	return {
+		play() {
+			if (playing) return
+			playing = true
+			lastNow = performance.now()
+			lastComposite = lastNow
+		},
+		pause() {
+			playing = false
+		},
+		setFPS(v: Fps) {
+			frameRate = v
+			frameDuration = 1000 / frameRate
+		},
+		isPlaying() {
+			return playing
+		},
+		ticks,
+		onTick
+	}
+}
+
+export type FixedStepOptions = {
+	fps: Fps
+	duration: Ms
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: Ms, index: number) => Promise<void> | void
+) => {
+	const dt = ms(1000 / opts.fps)
+	const durationInSeconds = opts.duration / 1000
+	const total = Math.ceil(durationInSeconds * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		const t = ms(i * dt)
+		await onFrame(t, i)
+	}
+}
+

package/s/timeline/renderers/player/parts/playback.ts

@@ -0,0 +1,213 @@
+
+import {Fps} from '../../../../units/fps.js'
+import {ms, Ms} from '../../../../units/ms.js'
+import {Driver} from '../../../../driver/driver.js'
+import {realtime} from '../../parts/schedulers.js'
+import {seconds} from '../../../../units/seconds.js'
+import {TimelineFile} from '../../../parts/basics.js'
+import {computeItemDuration} from '../../parts/handy.js'
+import {CursorVisualSampler, ReverseCursorVisualSampler} from '../../export/parts/cursor.js'
+import {DecoderSource} from '../../../../driver/fns/schematic.js'
+import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
+import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
+
+export class Playback {
+	audioSampler
+	seekVisualSampler
+	playVisualSampler: CursorVisualSampler | null = null
+	reversePlayVisualSampler: ReverseCursorVisualSampler | null = null
+
+	#playbackStart = ms(0)
+	#audioStartSec: number | null = null
+	#playbackRate = 1
+
+	#controller = realtime()
+	onTick = this.#controller.onTick
+
+	audioContext = new AudioContext({sampleRate: 48000})
+	audioGain = this.audioContext.createGain()
+	audioNodes = new Set<AudioBufferSourceNode>()
+	#audioAbort: AbortController | null = null
+
+	constructor(
+		private driver: Driver,
+		private timeline: TimelineFile,
+		private resolveMedia: (hash: string) => DecoderSource
+	) {
+		this.audioGain.connect(this.audioContext.destination)
+		this.audioGain.gain.value = 0.7 ** 2
+		this.seekVisualSampler = createVisualSampler(this.resolveMedia)
+		this.audioSampler = createAudioSampler(this.resolveMedia)
+		this.#samples()
+	}
+
+	update(timeline: TimelineFile) {
+		this.timeline = timeline
+	}
+
+	get isPlaying() {
+		return this.#controller.isPlaying()
+	}
+
+	async #samples() {
+		for await (const _ of this.#controller.ticks()) {
+			const time = this.currentTime
+			const layers = this.#playbackRate >= 0
+				? await this.playVisualSampler?.next(time) ?? []
+				: await this.reversePlayVisualSampler?.next(time) ?? []
+
+			const frame = await this.driver.composite(layers)
+			frame.close()
+
+			const hasEnded = this.#playbackRate >= 0
+				? time >= this.duration
+				: time <= 0
+
+			if (hasEnded)
+				this.pause()
+		}
+	}
+
+	async seek(time: Ms) {
+		this.pause()
+		this.#playbackStart = time
+		return await this.seekVisualSampler.sample(this.timeline, time)
+	}
+
+	async start() {
+		if(this.#controller.isPlaying())
+			return
+
+		await this.audioContext.resume()
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.audioContext.currentTime
+
+		this.#stopAudio()
+		this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+		this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+
+		this.#controller.play()
+		this.#startAudio()
+	}
+
+	pause() {
+		this.#playbackStart = this.currentTime
+		this.#controller.pause()
+		this.#stopAudio()
+
+		if (this.playVisualSampler) {
+			this.playVisualSampler.cancel()
+			this.playVisualSampler = null
+		}
+		if (this.reversePlayVisualSampler) {
+			this.reversePlayVisualSampler.cancel()
+			this.reversePlayVisualSampler = null
+		}
+
+	}
+
+	get duration() {
+		return computeItemDuration(
+			this.timeline.rootId,
+			this.timeline
+		)
+	}
+
+	get currentTime() {
+		if (!this.#controller.isPlaying() || this.#audioStartSec === null)
+			return this.#playbackStart
+
+		const elapsedMs = (this.audioContext.currentTime - this.#audioStartSec) * 1000
+		const current = this.#playbackStart + elapsedMs * this.#playbackRate
+		return ms(Math.max(0, Math.min(this.duration, current)))
+	}
+
+	get playbackRate() {
+		return this.#playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		if (!Number.isFinite(rate) || rate === 0)
+			throw new Error(`Invalid playback rate "${rate}".`)
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.#controller.isPlaying()
+			? this.audioContext.currentTime
+			: null
+		const wasReversed = this.#playbackRate < 0
+		this.#playbackRate = rate
+
+		if (this.#controller.isPlaying()) {
+			if (wasReversed && rate > 0) {
+				this.playVisualSampler?.cancel()
+				this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+			else if (!wasReversed && rate < 0) {
+				this.reversePlayVisualSampler?.cancel()
+				this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+
+			this.#syncAudio()
+		}
+	}
+
+	setFps(fps: Fps) {
+		this.#controller.setFPS(fps)
+	}
+
+	#syncAudio() {
+		this.#stopAudio()
+		this.#startAudio()
+	}
+
+	#stopAudio() {
+		this.#audioAbort?.abort()
+		this.#audioAbort = null
+
+		for (const node of this.audioNodes)
+			node.stop()
+
+		this.audioNodes.clear()
+	}
+
+	async #startAudio() {
+		if (this.#playbackRate !== 1)
+			return
+
+		const from = seconds(this.#playbackStart / 1000)
+		this.#audioAbort = new AbortController()
+		const signal = this.#audioAbort.signal
+
+		const ctx = this.audioContext
+
+		if (this.#audioStartSec === null)
+			return
+
+		for await (const {sample, timestamp} of this.audioSampler.sampleAudio(
+			this.timeline,
+			ms(from * 1000)
+		)) {
+
+			if (signal.aborted || !this.#controller.isPlaying())
+				return
+
+			while (timestamp - (ctx.currentTime - this.#audioStartSec + from) > 0.75)
+				await new Promise(r => setTimeout(r, 25))
+
+			const node = ctx.createBufferSource()
+			node.buffer = sample.toAudioBuffer()
+			node.connect(this.audioGain)
+			node.onended = () => this.audioNodes.delete(node)
+			this.audioNodes.add(node)
+
+			const startAt = this.#audioStartSec + timestamp - from
+
+			startAt >= ctx.currentTime
+				? node.start(startAt)
+				: node.start(ctx.currentTime, ctx.currentTime - startAt)
+
+		}
+	}
+}
+

package/s/timeline/renderers/player/player.ts

@@ -0,0 +1,86 @@
+
+import {ms} from "../../../units/ms.js"
+import {fps} from "../../../units/fps.js"
+import {Playback} from "./parts/playback.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+type ResolveMedia = (hash: string) => DecoderSource
+
+export class VideoPlayer {
+	canvas: HTMLCanvasElement
+	playback: Playback
+
+	#pendingSeek: number | null = null
+	#flushTask: Promise<void> | null = null
+
+	constructor(
+		private driver: Driver,
+		resolveMedia: ResolveMedia,
+		timeline: TimelineFile
+	) {
+		this.playback = new Playback(driver, timeline, resolveMedia)
+		this.canvas = driver.compositor.pixi.renderer.canvas
+	}
+
+	async play() {
+		await this.playback.start()
+	}
+
+	pause() {
+		this.playback.pause()
+	}
+
+	seek(timeMs: number) {
+		this.#pendingSeek = timeMs
+		return this.#flushTask ??= this.#flushSeeks().finally(() => this.#flushTask = null)
+	}
+
+	setFPS(value: number) {
+		this.playback.setFps(fps(value))
+	}
+
+	get isSeeking() {
+		return this.#flushTask !== null
+	}
+
+	get isPlaying() {
+		return this.playback.isPlaying
+	}
+
+	get duration() {
+		return this.playback.duration
+	}
+
+	get currentTime() {
+		return this.playback.currentTime
+	}
+
+	get playbackRate() {
+		return this.playback.playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		this.playback.playbackRate = rate
+	}
+
+	/**
+	 call this whenever your timeline state changes
+	*/
+	async update(timeline: TimelineFile) {
+		this.playback.update(timeline)
+		if (!this.isPlaying)
+			await this.seek(this.currentTime)
+	}
+
+	async #flushSeeks() {
+		while (this.#pendingSeek !== null) {
+			const next = this.#pendingSeek
+			this.#pendingSeek = null
+			const layers = await this.playback.seek(ms(next))
+			const frame = await this.driver.composite(layers)
+			frame.close()
+		}
+	}
+}