@omnimedia/omnitool 1.1.0-52 → 1.1.0-55

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@omnimedia/omnitool",
-	"version": "1.1.0-52",
+	"version": "1.1.0-55",
 	"description": "open source video processing tools",
 	"license": "MIT",
 	"author": "Przemysław Gałęzki",

package/s/demo/routines/transcode-test.ts

@@ -38,8 +38,8 @@ export function setupTranscodeTest(driver: Driver, source: DecoderSource) {
 		const audio = driver.decodeAudio({source})
 
 		const {readable, done} = driver.encode({
-			video,
-			audio,
+			video: video.readable,
+			audio: audio.readable,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/demo/routines/transitions-test.ts

@@ -30,8 +30,8 @@ export async function setupTransitionsTest(driver: Driver, source: DecoderSource
 			}
 		})
 
-		await driver.encode({
-			video,
+		driver.encode({
+			video: video.readable,
 			config: {
 				audio: {codec: "opus", bitrate: 128000},
 				video: {codec: "vp9", bitrate: 1000000}

package/s/driver/driver.ts

@@ -58,33 +58,54 @@ export class Driver {
 
 	decodeVideo(input: DecoderInput) {
 		let lastFrame: VideoFrame | null = null
+		const {port1, port2} = new MessageChannel()
 		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
 			async transform(chunk, controller) {
 				const frame = await input.onFrame?.(chunk) ?? chunk
-				// below code is to prevent mem leaks and hardware accelerated decoder stall
 				lastFrame?.close()
 				controller.enqueue(frame)
 				lastFrame = frame
 			}
 		})
-		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable]})({
+		this.thread.work.decodeVideo[tune]({transfer: [videoTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			video: videoTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return videoTransform.readable
+		return {
+			readable: videoTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
 	decodeAudio(input: DecoderInput) {
 		const audioTransform = new TransformStream<AudioData, AudioData>()
-		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable]})({
+		const {port1, port2} = new MessageChannel()
+		this.thread.work.decodeAudio[tune]({transfer: [audioTransform.writable, port2]})({
 			source: input.source,
+			cancel: port2,
 			audio: audioTransform.writable,
 			start: input.start,
 			end: input.end
 		})
-		return audioTransform.readable
+		return {
+			readable: audioTransform.readable,
+			/**
+			 * use this to stop decoding (premature interruption)
+			 * */
+			cancel() {
+				port1.postMessage("close")
+				port1.close()
+			}
+		}
 	}
 
 	encode({audio, video, config}: EncoderInput) {

package/s/driver/fns/schematic.ts

@@ -14,6 +14,7 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeAudio(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			audio: WritableStream<AudioData>
 			start?: number
 			end?: number
@@ -21,6 +22,7 @@ export type DriverSchematic = AsSchematic<{
 
 		decodeVideo(input: {
 			source: DecoderSource
+			cancel: MessagePort
 			video: WritableStream<VideoFrame>
 			start?: number
 			end?: number

package/s/driver/fns/work.ts

@@ -20,7 +20,7 @@ export const setupDriverWork = (
 			await shell.host.world()
 		},
 
-		async decodeAudio({source, audio, start, end}) {
+		async decodeAudio({source, audio, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -30,19 +30,29 @@ export const setupDriverWork = (
 			const audioDecodable = await audioTrack?.canDecode()
 			const audioWriter = audio.getWriter()
 
-			if (audioDecodable && audioTrack) {
-				const sink = new AudioSampleSink(audioTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toAudioData()
-					await audioWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await audioWriter.close()
+			if(!audioDecodable || !audioTrack)
+				return
+
+			const sink = new AudioSampleSink(audioTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
 			}
+
+			for await (const sample of samples) {
+				const frame = sample.toAudioData()
+				sample.close()
+				await audioWriter.write(frame)
+				frame.close()
+			}
+
+			await audioWriter.close()
 		},
 
-		async decodeVideo({source, video, start, end}) {
+		async decodeVideo({source, video, start, end, cancel}) {
 			const input = new Input({
 				source: await loadSource(source),
 				formats: ALL_FORMATS
@@ -52,16 +62,26 @@ export const setupDriverWork = (
 			const videoDecodable = await videoTrack?.canDecode()
 			const videoWriter = video.getWriter()
 
-			if (videoDecodable && videoTrack) {
-				const sink = new VideoSampleSink(videoTrack)
-				for await (const sample of sink.samples(start, end)) {
-					const frame = sample.toVideoFrame()
-					await videoWriter.write(frame)
-					sample.close()
-					frame.close()
-				}
-				await videoWriter.close()
+			if(!videoDecodable || !videoTrack)
+				return
+
+			const sink = new VideoSampleSink(videoTrack)
+			const samples = sink.samples(start, end)
+
+			cancel.onmessage = async () => {
+				samples.return()
+				input.dispose()
+				cancel.close()
 			}
+
+			for await (const sample of samples) {
+				const frame = sample.toVideoFrame()
+				sample.close()
+				await videoWriter.write(frame)
+				frame.close()
+			}
+
+			await videoWriter.close()
 		},
 
 		async encode({video, audio, config, writable}) {
@@ -78,6 +98,7 @@ export const setupDriverWork = (
 					const sample = new VideoSample(frame)
 					await videoSource.add(sample)
 					sample.close()
+					frame.close()
 				}
 			}
 

package/s/timeline/parts/waveform.ts

@@ -20,7 +20,7 @@ export class Waveform {
 	}
 
 	static async init(driver: Driver, source: DecoderSource, container: HTMLElement) {
-		const reader = driver.decodeAudio({source}).getReader()
+		const reader = driver.decodeAudio({source}).readable.getReader()
 
 		const peaks: number[] = []
 		let buffer: number[] = []

package/s/timeline/renderers/export/parts/cursor.ts

@@ -1,6 +1,5 @@
 
 import {ms, Ms} from "../../../../units/ms.js"
-import {Item} from "../../../parts/item.js"
 import {Driver} from "../../../../driver/driver.js"
 import {TimelineFile} from "../../../parts/basics.js"
 import {DecoderSource} from "../../../../driver/fns/schematic.js"
@@ -13,87 +12,107 @@ import {createVisualSampler} from "../../parts/samplers/visual/sampler.js"
  */
 
 export class CursorVisualSampler {
-	#sampler
+	#lastTimecode = -Infinity
 	#videoCursors = new Map<number, VideoFrameCursor>()
+	#sampler
 
 	constructor(
 		private driver: Driver,
-		private resolveMedia: (hash: string) => DecoderSource
+		private resolveMedia: (hash: string) => DecoderSource,
+		private timeline: TimelineFile
 	) {
-		this.#sampler = createVisualSampler(resolveMedia, (item, time) => {
-			const mediaTime = toUs(ms(item.start + time))
-			const cursor = this.#getCursorForVideo(item)
-			return cursor.next(mediaTime)
+		this.#sampler = createVisualSampler(this.resolveMedia, (item, time) => {
+			const targetUs = toUs(time)
+			let cursor = this.#videoCursors.get(item.id)
+
+			if (!cursor) {
+				const source = this.resolveMedia(item.mediaHash)
+				const endUs = toUs(ms(item.start + item.duration))
+				cursor = this.#createVideoCursor(source, targetUs, endUs)
+				this.#videoCursors.set(item.id, cursor)
+			}
+
+			return cursor.next(targetUs)
 		})
 	}
 
-	cursor(timeline: TimelineFile) {
-		let lastTimecode = Number.NEGATIVE_INFINITY
-		return {
-			next: (timecode: Ms) => {
-				if (timecode < lastTimecode)
-					throw new Error(`CursorVisualSampler is forward-only: requested ${timecode}ms after ${lastTimecode}ms`)
-				lastTimecode = timecode
-				return this.#sampler.sample(timeline, timecode)
-			},
-			cancel: () => this.#cancel(),
-		}
-	}
-
-	#getCursorForVideo(videoItem: Item.Video) {
-		const existing = this.#videoCursors.get(videoItem.id)
-		if (existing)
-			return existing
-
-		const source = this.resolveMedia(videoItem.mediaHash)
-		const video = this.driver.decodeVideo({ source })
-		const cursor = this.#cursor(video.getReader())
+	next(timecode: Ms) {
+		if (timecode < this.#lastTimecode)
+			throw new Error(`Forward-only cursor regression: ${timecode}ms < ${this.#lastTimecode}ms`)
 
-		this.#videoCursors.set(videoItem.id, cursor)
-		return cursor
+		this.#lastTimecode = timecode
+		return this.#sampler.sample(this.timeline, timecode)
 	}
 
-	async #cancel() {
-		await Promise.all(
-			[...this.#videoCursors.values()].map(cursor => cursor.cancel())
-		)
+	async cancel() {
+		await Promise.all([...this.#videoCursors.values()].map(c => c.cancel()))
 		this.#videoCursors.clear()
 	}
 
-	// forward only
-	#cursor(reader: ReadableStreamDefaultReader<VideoFrame>) {
+	#createVideoCursor(source: DecoderSource, startUs: number, endUs: number): VideoFrameCursor {
+		const video = this.driver.decodeVideo({source, start: startUs / 1_000_000, end: endUs / 1_000_000})
+		const reader = video.readable.getReader()
+
+		let current: VideoFrame | null = null
+		let nextPromise: Promise<VideoFrame | null> | null = null
+		let ended = false
+
+		const readNext = async () => {
+			if (ended) return null
+			const {done, value} = await reader.read()
+			if (done) return (ended = true, null)
+
+			const frame = new VideoFrame(value)
+			value.close()
+			return frame
+		}
+
 		return {
 			async next(targetUs: number): Promise<VideoFrame | undefined> {
-				let prev: VideoFrame | null = null
+				current ??= await readNext()
+				if (!current) return undefined
+
 				while (true) {
-					const { done, value: hit } = await reader.read()
+					nextPromise ??= readNext()
+					const nextFrame = await nextPromise
 
-					if (done) {
-						const out = prev ? new VideoFrame(prev) : undefined
-						prev?.close()
-						return out
-					}
+					if (!nextFrame) return new VideoFrame(current)
 
-					const hitUs = hit.timestamp ?? 0
-					if (hitUs >= targetUs) {
-						const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
-						const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
+					const currentUs = current.timestamp ?? -Infinity
+					const nextUs = nextFrame.timestamp ?? currentUs
 
-						const chosen = usePrev ? prev! : hit
-						const other = usePrev ? hit : prev
+					if (nextUs < targetUs) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
+					}
+
+					const useNext = Math.abs(nextUs - targetUs) < Math.abs(currentUs - targetUs)
 
-						const copy = new VideoFrame(chosen)
-						chosen.close()
-						other?.close()
-						return copy
+					if (useNext) {
+						current.close()
+						current = nextFrame
+						nextPromise = null
+						continue
 					}
 
-					prev?.close()
-					prev = hit
+					return new VideoFrame(current)
 				}
 			},
 
-			cancel: async () => await reader.cancel()
+			async cancel() {
+				const pending = nextPromise
+				nextPromise = null
+
+				current?.close()
+				current = null
+
+				video.cancel()
+
+				const buffered = await pending?.catch(() => null)
+				buffered?.close()
+			}
 		}
 	}
 }

package/s/timeline/renderers/export/parts/produce-video.ts

@@ -21,14 +21,13 @@ export function produceVideo({
 
 	const stream = new TransformStream<VideoFrame, VideoFrame>()
 	const writer = stream.writable.getWriter()
-	const sampler = new CursorVisualSampler(driver, resolveMedia)
-	const cursor = sampler.cursor(timeline)
+	const sampler = new CursorVisualSampler(driver, resolveMedia, timeline)
 	const dt = 1 / fps
 	const duration = computeItemDuration(timeline.rootId, timeline)
 
 	async function produce() {
 		await fixedStep({fps, duration}, async (timecode, i) => {
-			const layers = await cursor.next(timecode)
+			const layers = await sampler.next(timecode)
 			const composed = await driver.composite(layers)
 
 			const frame = new VideoFrame(composed, {

package/s/timeline/renderers/player/parts/playback.ts

@@ -1,17 +1,20 @@
 
-import {pub} from '@e280/stz'
 import {Fps} from '../../../../units/fps.js'
 import {ms, Ms} from '../../../../units/ms.js'
+import {Driver} from '../../../../driver/driver.js'
 import {realtime} from '../../parts/schedulers.js'
 import {TimelineFile} from '../../../parts/basics.js'
+import {computeItemDuration} from '../../parts/handy.js'
 import {seconds, Seconds} from '../../../../units/seconds.js'
+import {CursorVisualSampler} from '../../export/parts/cursor.js'
 import {DecoderSource} from '../../../../driver/fns/schematic.js'
 import {createAudioSampler} from '../../parts/samplers/audio/sampler.js'
 import {createVisualSampler} from '../../parts/samplers/visual/sampler.js'
 
 export class Playback {
-	visualSampler
 	audioSampler
+	seekVisualSampler
+	playVisualSampler: CursorVisualSampler | null = null
 
 	#playbackStart = ms(0)
 	#audioStartSec: number | null = null
@@ -25,28 +28,39 @@ export class Playback {
 	#audioAbort: AbortController | null = null
 
 	constructor(
+		private driver: Driver,
 		private timeline: TimelineFile,
 		private resolveMedia: (hash: string) => DecoderSource
 	) {
 		this.audioGain.connect(this.audioContext.destination)
 		this.audioGain.gain.value = 0.7 ** 2
-		this.visualSampler = createVisualSampler(this.resolveMedia)
+		this.seekVisualSampler = createVisualSampler(this.resolveMedia)
 		this.audioSampler = createAudioSampler(this.resolveMedia)
+		this.#samples()
 	}
 
-	async *samples() {
+	async #samples() {
 		for await (const _ of this.#controller.ticks()) {
-			yield this.visualSampler.sample(this.timeline, this.currentTime)
+			const layers = await this.playVisualSampler?.next(this.currentTime) ?? []
+
+			const frame = await this.driver.composite(layers)
+			frame.close()
+
+			if (this.currentTime >= this.duration)
+				this.pause()
 		}
 	}
 
 	async seek(time: Ms) {
 		this.pause()
 		this.#playbackStart = time
-		return await this.visualSampler.sample(this.timeline, time)
+		return await this.seekVisualSampler.sample(this.timeline, time)
 	}
 
 	async start(timeline: TimelineFile) {
+		if(this.#controller.isPlaying())
+			return
+
 		this.timeline = timeline
 		await this.audioContext.resume()
 
@@ -61,6 +75,9 @@ export class Playback {
 
 		this.audioNodes.clear()
 
+		this.playVisualSampler?.cancel()
+		this.playVisualSampler = new CursorVisualSampler(this.driver, this.resolveMedia, this.timeline)
+
 		this.#controller.play()
 		this.#startAudio(this.#audioAbort.signal, seconds(this.#playbackStart / 1000))
 	}
@@ -74,6 +91,16 @@ export class Playback {
 			node.stop()
 
 		this.audioNodes.clear()
+
+		this.playVisualSampler?.cancel()
+		this.playVisualSampler = null
+	}
+
+	get duration() {
+		return computeItemDuration(
+			this.timeline.rootId,
+			this.timeline
+		)
 	}
 
 	get currentTime() {

package/s/timeline/renderers/player/player.ts

@@ -4,7 +4,6 @@ import {fps} from "../../../units/fps.js"
 import {Playback} from "./parts/playback.js"
 import {Driver} from "../../../driver/driver.js"
 import {TimelineFile} from "../../parts/basics.js"
-import {computeItemDuration} from "../parts/handy.js"
 import {DecoderSource} from "../../../driver/fns/schematic.js"
 
 type ResolveMedia = (hash: string) => DecoderSource
@@ -18,20 +17,12 @@ export class VideoPlayer {
 		resolveMedia: ResolveMedia,
 		private timeline: TimelineFile,
 	) {
-		this.playback = new Playback(timeline, resolveMedia)
+		this.playback = new Playback(driver, timeline, resolveMedia)
 		this.canvas = driver.compositor.pixi.renderer.canvas
 	}
 
 	async play() {
 		await this.playback.start(this.timeline)
-
-		for await (const layers of this.playback.samples()) {
-			const frame = await this.driver.composite(layers)
-			frame.close()
-
-			if (this.currentTime >= this.duration)
-				this.pause()
-		}
 	}
 
 	pause() {
@@ -49,10 +40,7 @@ export class VideoPlayer {
 	}
 
 	get duration() {
-		return computeItemDuration(
-			this.timeline.rootId,
-			this.timeline
-		)
+		return this.playback.duration
 	}
 
 	get currentTime() {

package/s/timeline/renderers/renderers.test.ts

@@ -49,8 +49,7 @@ export default Science.suite({
 			o.video(videoA, {duration: 2000}),
 			o.audio(videoA, {duration: 500}),
 		))})
-		const sampler = new CursorVisualSampler(driver, resolveMedia)
-		const cursor = sampler.cursor(timeline)
+		const cursor = new CursorVisualSampler(driver, resolveMedia, timeline)
 		await cursor.next(ms(1000))
 		await expect(async () => await cursor.next(ms(100))).throwsAsync()
 	}),
@@ -384,4 +383,3 @@ export default Science.suite({
 		expect(totalFrames).is(240000)
 	})
 })
-

package/s/timeline/sugar/o.ts

@@ -20,7 +20,7 @@ export class O {
 		return this.state.timeline
 	}
 
-	#getId() {
+	getId() {
 		return hex.toInteger(hex.random())
 	}
 
@@ -37,7 +37,7 @@ export class O {
 
   textStyle = (style: TextStyleOptions): Item.TextStyle => {
 		const item = {
-			id: this.#getId(),
+			id: this.getId(),
 			kind: Kind.TextStyle,
 			style
 		} as Item.TextStyle
@@ -47,7 +47,7 @@ export class O {
 
   spatial = (transform: Transform): Item.Spatial => {
   	const item: Item.Spatial = {
-  		id: this.#getId(),
+  		id: this.getId(),
   		kind: Kind.Spatial,
   		transform,
   		enabled: true
@@ -58,7 +58,7 @@ export class O {
 
 	sequence = (...items: Item.Any[]): Item.Sequence => {
 		const item =  {
-			id: this.#getId(),
+			id: this.getId(),
 			kind: Kind.Sequence,
 			childrenIds: items.map(item => item.id)
 		} as Item.Sequence
@@ -69,7 +69,7 @@ export class O {
 	stack = (...items: Item.Any[]): Item.Stack => {
 		const item = {
 			kind: Kind.Stack,
-			id: this.#getId(),
+			id: this.getId(),
 			childrenIds: items.map(item => item.id)
 		} as Item.Stack
 		this.register(item)
@@ -88,7 +88,7 @@ export class O {
 
 		const item: Item.Video = {
 			kind: Kind.Video,
-			id: this.#getId(),
+			id: this.getId(),
 			mediaHash: media.datafile.checksum.hash,
 			start: options?.start ?? 0,
 			duration: options?.duration ?? media.duration
@@ -110,7 +110,7 @@ export class O {
 
 		const item: Item.Audio = {
 			kind: Kind.Audio,
-			id: this.#getId(),
+			id: this.getId(),
 			mediaHash: media.datafile.checksum.hash,
 			start: options?.start ?? 0,
 			duration: options?.duration ?? media.duration,
@@ -126,7 +126,7 @@ export class O {
 		}): Item.Text => {
 
 		const item = {
-			id: this.#getId(),
+			id: this.getId(),
 			content,
 			kind: Kind.Text,
 			duration: options?.duration ?? 2000
@@ -141,7 +141,7 @@ export class O {
 
 	gap = (duration: number): Item.Gap => {
 		const item = {
-			id: this.#getId(),
+			id: this.getId(),
 			kind: Kind.Gap,
 			duration
 		} as Item.Gap
@@ -152,7 +152,7 @@ export class O {
 	transition = {
 		crossfade: (duration: number): Item.Transition => {
 			const item = {
-				id: this.#getId(),
+				id: this.getId(),
 				kind: Kind.Transition,
 				effect: Effect.Crossfade,
 				duration,