@omnimedia/omnitool 1.1.0-88 → 1.1.0-89

package/README.md

@@ -312,11 +312,14 @@ const driver = await Driver.setup({workerUrl})
 const player = await omni.playback(timeline)
 
 document.body.appendChild(player.canvas)
-player.play()
+await player.play()
+player.playbackRate = 0.5 // slow motion
+player.playbackRate = -1 // reverse
 ```
 
 Notes:
 - Call `await player.update(timeline)` if you update the timeline.
+- `playbackRate` supports slower, faster, and reverse visual playback. Audio currently plays only at `1`.
 
 ## 📤 Export
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-88",
+	"version": "1.1.0-89",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",

package/s/timeline/renderers/export/parts/cursor.ts

@@ -1,25 +1,28 @@
 
+import {ALL_FORMATS, Input, VideoSampleSink} from "mediabunny"
+
 import {ms, Ms} from "../../../../units/ms.js"
 import {Driver} from "../../../../driver/driver.js"
 import {TimelineFile} from "../../../parts/basics.js"
 import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../../../driver/utils/load-decoder-source.js"
 import {createVisualSampler} from "../../parts/samplers/visual/sampler.js"
 
-/**
- * forward-only frame cursor optimized for export purposes.
- * it uses mediabunny internally so the support for non-clients
- * should be done from mediabunny custom decoder/encoder
- */
+type StreamCursor<T> = {
+	next(target: number): Promise<T | undefined>
+	cancel(): Promise<void>
+}
 
-export class CursorVisualSampler {
-	#lastTimecode = -Infinity
-	#videoCursors = new Map<number, VideoFrameCursor>()
-	#sampler
+type VideoFrameCursor = StreamCursor<VideoFrame>
+
+abstract class BaseVisualSampler {
+	readonly #videoCursors = new Map<number, VideoFrameCursor>()
+	readonly #sampler
 
 	constructor(
-		private driver: Driver,
-		private resolveMedia: (hash: string) => DecoderSource,
-		private timeline: TimelineFile
+		protected driver: Driver,
+		protected resolveMedia: (hash: string) => DecoderSource,
+		protected timeline: TimelineFile
 	) {
 		this.#sampler = createVisualSampler(this.resolveMedia, (item, time) => {
 			const targetUs = toUs(time)
@@ -28,7 +31,7 @@ export class CursorVisualSampler {
 			if (!cursor) {
 				const source = this.resolveMedia(item.mediaHash)
 				const endUs = toUs(ms(item.start + item.duration))
-				cursor = this.#createVideoCursor(source, targetUs, endUs)
+				cursor = this.createCursor(source, targetUs, endUs)
 				this.#videoCursors.set(item.id, cursor)
 			}
 
@@ -36,11 +39,9 @@ export class CursorVisualSampler {
 		})
 	}
 
-	next(timecode: Ms) {
-		if (timecode < this.#lastTimecode)
-			throw new Error(`Forward-only cursor regression: ${timecode}ms < ${this.#lastTimecode}ms`)
+	protected abstract createCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor
 
-		this.#lastTimecode = timecode
+	protected sample(timecode: Ms) {
 		return this.#sampler.sample(this.timeline, timecode)
 	}
 
@@ -48,8 +49,26 @@ export class CursorVisualSampler {
 		await Promise.all([...this.#videoCursors.values()].map(c => c.cancel()))
 		this.#videoCursors.clear()
 	}
+}
+
+/**
+ * forward-only frame cursor optimized for export purposes.
+ * it uses mediabunny internally so the support for non-clients
+ * should be done from mediabunny custom decoder/encoder
+ */
+
+export class CursorVisualSampler extends BaseVisualSampler {
+	#lastTimecode = -Infinity
+
+	next(timecode: Ms) {
+		if (timecode < this.#lastTimecode)
+			throw new Error(`Forward-only cursor regression: ${timecode}ms < ${this.#lastTimecode}ms`)
+
+		this.#lastTimecode = timecode
+		return this.sample(timecode)
+	}
 
-	#createVideoCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
+	protected createCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
 		const video = this.driver.decodeVideo({source, start: startUs / 1_000_000, end: endUs / 1_000_000})
 		const reader = video.readable.getReader()
 
@@ -118,12 +137,138 @@ export class CursorVisualSampler {
 	}
 }
 
-const toUs = (ms: Ms) => Math.round(ms * 1_000)
+export class ReverseCursorVisualSampler extends BaseVisualSampler {
+	#lastTimecode = Infinity
 
-type StreamCursor<T> = {
-	next(target: number): Promise<T | undefined>
-	cancel(): Promise<void>
-}
+	next(timecode: Ms) {
+		if (timecode > this.#lastTimecode)
+			throw new Error(`Reverse-only cursor regression: ${timecode}ms > ${this.#lastTimecode}ms`)
 
-type VideoFrameCursor = StreamCursor<VideoFrame>
+		this.#lastTimecode = timecode
+		return this.sample(timecode)
+	}
 
+	protected createCursor(source: DecoderSource, _initialTargetUs: number, endUs: number): VideoFrameCursor {
+		const startUs = 0
+		const windowUs = 1_000_000
+		const prefetchThreshold = windowUs * 0.5
+
+		let frames: VideoFrame[] = []
+		let windowStart = Infinity
+		let windowEnd = -Infinity
+		let input: Input | null = null
+		let sink: VideoSampleSink | null = null
+		let prefetchPromise: Promise<{frames: VideoFrame[], windowStart: number, windowEnd: number}> | null = null
+		let canceled = false
+
+		const clear = () => {
+			for (const frame of frames)
+				frame.close()
+			frames = []
+		}
+
+		const getSink = async () => {
+			if (sink) return sink
+
+			input = new Input({
+				source: await loadDecoderSource(source),
+				formats: ALL_FORMATS,
+			})
+
+			const track = await input.getPrimaryVideoTrack()
+			sink = track && await track.canDecode()
+				? new VideoSampleSink(track)
+				: null
+
+			return sink
+		}
+
+		const fetchFrames = async (targetUs: number) => {
+			const wEnd = Math.min(endUs, targetUs + 1)
+			const wStart = Math.max(startUs, wEnd - windowUs)
+			const newFrames: VideoFrame[] = []
+
+			const videoSink = await getSink()
+			if (videoSink) {
+				for await (const sample of videoSink.samples(wStart / 1_000_000, wEnd / 1_000_000)) {
+					newFrames.push(sample.toVideoFrame())
+					sample.close()
+				}
+			}
+
+			return {frames: newFrames, windowStart: wStart, windowEnd: wEnd}
+		}
+
+		const loadWindow = async (targetUs: number) => {
+			clear()
+			const result = await fetchFrames(targetUs)
+			frames = result.frames
+			windowStart = result.windowStart
+			windowEnd = result.windowEnd
+		}
+
+		return {
+			async next(targetUs: number): Promise<VideoFrame | undefined> {
+				if (canceled)
+					return undefined
+
+				if (targetUs < windowStart || targetUs > windowEnd) {
+					if (prefetchPromise) {
+						const prefetched = await prefetchPromise
+						prefetchPromise = null
+
+						if (canceled) {
+							for (const f of prefetched.frames) f.close()
+							return undefined
+						}
+
+						if (targetUs >= prefetched.windowStart && targetUs <= prefetched.windowEnd) {
+							clear()
+							frames = prefetched.frames
+							windowStart = prefetched.windowStart
+							windowEnd = prefetched.windowEnd
+						} else {
+							for (const f of prefetched.frames) f.close()
+							await loadWindow(targetUs)
+						}
+					} else {
+						await loadWindow(targetUs)
+					}
+				}
+
+				if (!prefetchPromise && targetUs < windowStart + prefetchThreshold && windowStart > startUs)
+					prefetchPromise = fetchFrames(windowStart - 1)
+
+				let best: VideoFrame | undefined
+				let bestDistance = Infinity
+
+				for (const frame of frames) {
+					const distance = Math.abs((frame.timestamp ?? targetUs) - targetUs)
+					if (distance < bestDistance) {
+						best = frame
+						bestDistance = distance
+					}
+				}
+
+				return best ? new VideoFrame(best) : undefined
+			},
+
+			async cancel() {
+				canceled = true
+				const pending = prefetchPromise
+				prefetchPromise = null
+
+				const prefetched = await pending?.catch(() => null)
+				if (prefetched)
+					for (const f of prefetched.frames) f.close()
+
+				clear()
+				input?.dispose()
+				input = null
+				sink = null
+			}
+		}
+	}
+}
+
+const toUs = (ms: Ms) => Math.round(ms * 1_000)

package/s/timeline/renderers/player/parts/playback.ts

@@ -3,10 +3,10 @@ import {Fps} from '../../../../units/fps.js'
 import {ms, Ms} from '../../../../units/ms.js'
 import {Driver} from '../../../../driver/driver.js'
 import {realtime} from '../../parts/schedulers.js'
+import {seconds} from '../../../../units/seconds.js'
 import {TimelineFile} from '../../../parts/basics.js'
 import {computeItemDuration} from '../../parts/handy.js'
-import {seconds, Seconds} from '../../../../units/seconds.js'
-import {CursorVisualSampler} from '../../export/parts/cursor.js'
+import {CursorVisualSampler, ReverseCursorVisualSampler} from '../../export/parts/cursor.js'
 import {DecoderSource} from '../../../../driver/fns/schematic.js'
 import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
 import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
@@ -15,9 +15,11 @@ export class Playback {
 	audioSampler
 	seekVisualSampler
 	playVisualSampler: CursorVisualSampler | null = null
+	reversePlayVisualSampler: ReverseCursorVisualSampler | null = null
 
 	#playbackStart = ms(0)
 	#audioStartSec: number | null = null
+	#playbackRate = 1
 
 	#controller = realtime()
 	onTick = this.#controller.onTick
@@ -49,12 +51,19 @@ export class Playback {
 
 	async #samples() {
 		for await (const _ of this.#controller.ticks()) {
-			const layers = await this.playVisualSampler?.next(this.currentTime) ?? []
+			const time = this.currentTime
+			const layers = this.#playbackRate >= 0
+				? await this.playVisualSampler?.next(time) ?? []
+				: await this.reversePlayVisualSampler?.next(time) ?? []
 
 			const frame = await this.driver.composite(layers)
 			frame.close()
 
-			if (this.currentTime >= this.duration)
+			const hasEnded = this.#playbackRate >= 0
+				? time >= this.duration
+				: time <= 0
+
+			if (hasEnded)
 				this.pause()
 		}
 	}
@@ -74,34 +83,27 @@ export class Playback {
 		this.#playbackStart = this.currentTime
 		this.#audioStartSec = this.audioContext.currentTime
 
-		this.#audioAbort?.abort()
-		this.#audioAbort = new AbortController()
-
-		for (const node of this.audioNodes)
-			node.stop()
-
-		this.audioNodes.clear()
-
+		this.#stopAudio()
 		this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+		this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
 
 		this.#controller.play()
-		this.#startAudio(this.#audioAbort.signal, seconds(this.#playbackStart / 1000))
+		this.#startAudio()
 	}
 
 	pause() {
 		this.#playbackStart = this.currentTime
 		this.#controller.pause()
-		this.#audioAbort?.abort()
-
-		for (const node of this.audioNodes)
-			node.stop()
-
-		this.audioNodes.clear()
+		this.#stopAudio()
 
 		if (this.playVisualSampler) {
 			this.playVisualSampler.cancel()
 			this.playVisualSampler = null
 		}
+		if (this.reversePlayVisualSampler) {
+			this.reversePlayVisualSampler.cancel()
+			this.reversePlayVisualSampler = null
+		}
 
 	}
 
@@ -117,14 +119,66 @@ export class Playback {
 			return this.#playbackStart
 
 		const elapsedMs = (this.audioContext.currentTime - this.#audioStartSec) * 1000
-		return ms(this.#playbackStart + elapsedMs)
+		const current = this.#playbackStart + elapsedMs * this.#playbackRate
+		return ms(Math.max(0, Math.min(this.duration, current)))
+	}
+
+	get playbackRate() {
+		return this.#playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		if (!Number.isFinite(rate) || rate === 0)
+			throw new Error(`Invalid playback rate "${rate}".`)
+
+		this.#playbackStart = this.currentTime
+		this.#audioStartSec = this.#controller.isPlaying()
+			? this.audioContext.currentTime
+			: null
+		const wasReversed = this.#playbackRate < 0
+		this.#playbackRate = rate
+
+		if (this.#controller.isPlaying()) {
+			if (wasReversed && rate > 0) {
+				this.playVisualSampler?.cancel()
+				this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+			else if (!wasReversed && rate < 0) {
+				this.reversePlayVisualSampler?.cancel()
+				this.reversePlayVisualSampler = new ReverseCursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+			}
+
+			this.#syncAudio()
+		}
 	}
 
 	setFps(fps: Fps) {
 		this.#controller.setFPS(fps)
 	}
 
-	async #startAudio(signal: AbortSignal, from: Seconds) {
+	#syncAudio() {
+		this.#stopAudio()
+		this.#startAudio()
+	}
+
+	#stopAudio() {
+		this.#audioAbort?.abort()
+		this.#audioAbort = null
+
+		for (const node of this.audioNodes)
+			node.stop()
+
+		this.audioNodes.clear()
+	}
+
+	async #startAudio() {
+		if (this.#playbackRate !== 1)
+			return
+
+		const from = seconds(this.#playbackStart / 1000)
+		this.#audioAbort = new AbortController()
+		const signal = this.#audioAbort.signal
+
 		const ctx = this.audioContext
 
 		if (this.#audioStartSec === null)

package/s/timeline/renderers/player/player.ts

@@ -57,6 +57,14 @@ export class VideoPlayer {
 		return this.playback.currentTime
 	}
 
+	get playbackRate() {
+		return this.playback.playbackRate
+	}
+
+	set playbackRate(rate: number) {
+		this.playback.playbackRate = rate
+	}
+
 	/**
 	 call this whenever your timeline state changes
 	*/