@omnimedia/omnitool 1.1.0-5 → 1.1.0-51

package/s/timeline/parts/media.ts

@@ -1,14 +1,45 @@
 
+import {ALL_FORMATS, Input} from "mediabunny"
+
 import {Datafile} from "../utils/datafile.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+import {loadDecoderSource} from "../../driver/utils/load-decoder-source.js"
 
 export class Media {
 	duration = 0
+	hasVideo = false
+	hasAudio = false
+
 	constructor(public datafile: Datafile) {}
 
 	static async analyze(datafile: Datafile) {
 		const media = new this(datafile)
-		media.duration = 10
+		const duration = (await this.duration(datafile.url)) * 1000
+		media.duration = duration
+		const {video, audio} = await this.#has(datafile.url)
+		media.hasAudio = audio
+		media.hasVideo = video
 		return media
 	}
+
+	static async duration(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		const duration =  await input.computeDuration()
+		return Number(duration.toFixed(5)) // fix weird floating points
+	}
+
+	static async #has(source: DecoderSource) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		return {
+			audio: !!(await input.getPrimaryAudioTrack()),
+			video: !!(await input.getPrimaryVideoTrack())
+		}
+	}
 }
 

package/s/timeline/parts/resource-pool.ts

@@ -1,27 +1,30 @@
 
-import {MapG} from "@e280/stz"
+import {GMap} from "@e280/stz"
 import {Hash} from "./basics.js"
 import {Media} from "./media.js"
 import {Resource} from "./resource.js"
 import {Datafile} from "../utils/datafile.js"
 
 export class ResourcePool {
-	#map = new MapG<Hash, Resource.Any>
+	#map = new GMap<Hash, Resource.Any>
 
 	/** store a media file (avoids duplicates via hash) */
 	async store(datafile: Datafile) {
 		const media = await Media.analyze(datafile)
 		const {hash} = media.datafile.checksum
-		const {filename, bytes} = media.datafile
+		const {filename, bytes, url, blob} = media.datafile
 
 		if (this.#map.has(hash)) {
 			const alreadyExists = this.#map.require(hash)
 			alreadyExists.filename = filename
 		}
 		else
-			this.#map.set(hash, {kind: "media", filename, bytes})
+			this.#map.set(hash, {kind: "media", filename, bytes, url, blob})
 
 		return media
 	}
-}
 
+	require(hash: Hash) {
+		return this.#map.require(hash)
+	}
+}

package/s/timeline/parts/resource.ts

@@ -4,6 +4,8 @@ export namespace Resource {
 		kind: "media"
 		filename: string
 		bytes: Uint8Array
+		blob: Blob
+		url: string
 	}
 
 	export type Any = Media

package/s/timeline/parts/waveform.ts

@@ -1,6 +1,6 @@
 import WaveSurfer from "wavesurfer.js"
 
-import {context} from "../../context.js"
+import {Driver} from "../../driver/driver.js"
 import {DecoderSource} from "../../driver/fns/schematic.js"
 
 export class Waveform {
@@ -19,9 +19,8 @@ export class Waveform {
 		})
 	}
 
-	static async init(source: DecoderSource, container: HTMLElement) {
-		const driver = await context.driver
-		const reader = driver.decode({ source }).audio.getReader()
+	static async init(driver: Driver, source: DecoderSource, container: HTMLElement) {
+		const reader = driver.decodeAudio({source}).getReader()
 
 		const peaks: number[] = []
 		let buffer: number[] = []

package/s/timeline/renderers/export/parts/audio-gain.ts

@@ -0,0 +1,17 @@
+// this is mutating fn
+export const applyGainToPlanar = (
+	planes: Float32Array[],
+	gain: number
+) => {
+	if (gain === 1)
+		return
+	// planes
+
+	for (const plane of planes) {
+		for (let i = 0; i < plane.length; i++) {
+			plane[i] *= gain
+		}
+	}
+
+	// return planes
+}

package/s/timeline/renderers/export/parts/audio-mix.ts

@@ -0,0 +1,133 @@
+
+export type AudioPlanesInput = {
+	planes: Float32Array[]
+	sampleRate: number
+	timestamp: number
+}
+
+export type MixedChunk = {
+	planar: Float32Array
+	sampleRate: number
+	channels: number
+	frames: number
+	startFrame: number
+}
+
+type ActiveSample = {
+	startFrame: number
+	endFrame: number
+	data: Float32Array[]
+}
+
+export type AudioMixOptions = {
+	chunkFrames?: number
+	clamp?: boolean
+}
+
+export class AudioMix {
+	readonly #chunkFrames: number
+	readonly #clamp: boolean
+
+	constructor(options: AudioMixOptions = {}) {
+		this.#chunkFrames = options.chunkFrames ?? 1024
+		this.#clamp = options.clamp ?? true
+	}
+
+	async *mix(samples: AsyncIterable<AudioPlanesInput>): AsyncGenerator<MixedChunk> {
+		const chunkFrames = this.#chunkFrames
+		let sampleRate: number | null = null
+		let channels: number | null = null
+		const active: ActiveSample[] = []
+		let nextFrame = 0
+		let maxEnd = 0
+
+		for await (const input of samples) {
+			if (channels === null) {
+				channels = input.planes.length
+				sampleRate = input.sampleRate
+			} else {
+				if (input.planes.length !== channels) throw new Error(`Channel count changed`)
+				if (input.sampleRate !== sampleRate) throw new Error(`Sample rate changed`)
+			}
+
+			const inputStart = Math.round(input.timestamp * sampleRate)
+			const frames = input.planes[0]?.length ?? 0
+
+			while (nextFrame + chunkFrames <= inputStart) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+
+			active.push({
+				startFrame: inputStart,
+				endFrame: inputStart + frames,
+				data: input.planes
+			})
+
+			maxEnd = Math.max(maxEnd, inputStart + frames)
+		}
+
+		if (channels !== null && sampleRate !== null) {
+			while (nextFrame < maxEnd) {
+				yield this.#processChunk(active, nextFrame, channels, sampleRate)
+				nextFrame += chunkFrames
+			}
+		}
+	}
+
+	#processChunk(
+		active: ActiveSample[],
+		currentStartFrame: number,
+		channels: number,
+		sampleRate: number
+	): MixedChunk {
+		const chunkFrames = this.#chunkFrames
+		const outputBuffer = new Float32Array(channels * chunkFrames)
+		const chunkEnd = currentStartFrame + chunkFrames
+
+		for (let ch = 0; ch < channels; ch++) {
+			const channelOffset = ch * chunkFrames
+			const outChannelView = outputBuffer.subarray(channelOffset, channelOffset + chunkFrames)
+
+			for (const sample of active) {
+				const data = sample.data[ch]
+				if (!data) continue
+
+				const start = Math.max(currentStartFrame, sample.startFrame)
+				const end = Math.min(chunkEnd, sample.endFrame)
+
+				if (start >= end) continue
+
+				const dstIdx = start - currentStartFrame
+				const srcIdx = start - sample.startFrame
+				const len = end - start
+
+				for (let i = 0; i < len; i++) {
+					outChannelView[dstIdx + i] += data[srcIdx + i]
+				}
+			}
+
+			if (this.#clamp) {
+				for (let i = 0; i < chunkFrames; i++) {
+					const v = outChannelView[i]
+					outChannelView[i] = v < -1.0 ? -1.0 : (v > 1.0 ? 1.0 : v)
+				}
+			}
+		}
+
+		for (let i = active.length - 1; i >= 0; i--) {
+			if (active[i].endFrame <= chunkEnd) {
+				active.splice(i, 1)
+			}
+		}
+
+		return {
+			planar: outputBuffer,
+			sampleRate,
+			channels,
+			frames: chunkFrames,
+			startFrame: currentStartFrame
+		}
+	}
+}
+

package/s/timeline/renderers/export/parts/cursor.ts

@@ -0,0 +1,109 @@
+
+import {ms, Ms} from "../../../../units/ms.js"
+import {Item} from "../../../parts/item.js"
+import {Driver} from "../../../../driver/driver.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createVisualSampler} from "../../parts/samplers/visual/sampler.js"
+
+/**
+ * forward-only frame cursor optimized for export purposes.
+ * it uses mediabunny internally so the support for non-clients
+ * should be done from mediabunny custom decoder/encoder
+ */
+
+export class CursorVisualSampler {
+	#sampler
+	#videoCursors = new Map<number, VideoFrameCursor>()
+
+	constructor(
+		private driver: Driver,
+		private resolveMedia: (hash: string) => DecoderSource
+	) {
+		this.#sampler = createVisualSampler(resolveMedia, (item, time) => {
+			const mediaTime = toUs(ms(item.start + time))
+			const cursor = this.#getCursorForVideo(item)
+			return cursor.next(mediaTime)
+		})
+	}
+
+	cursor(timeline: TimelineFile) {
+		let lastTimecode = Number.NEGATIVE_INFINITY
+		return {
+			next: (timecode: Ms) => {
+				if (timecode < lastTimecode)
+					throw new Error(`CursorVisualSampler is forward-only: requested ${timecode}ms after ${lastTimecode}ms`)
+				lastTimecode = timecode
+				return this.#sampler.sample(timeline, timecode)
+			},
+			cancel: () => this.#cancel(),
+		}
+	}
+
+	#getCursorForVideo(videoItem: Item.Video) {
+		const existing = this.#videoCursors.get(videoItem.id)
+		if (existing)
+			return existing
+
+		const source = this.resolveMedia(videoItem.mediaHash)
+		const video = this.driver.decodeVideo({ source })
+		const cursor = this.#cursor(video.getReader())
+
+		this.#videoCursors.set(videoItem.id, cursor)
+		return cursor
+	}
+
+	async #cancel() {
+		await Promise.all(
+			[...this.#videoCursors.values()].map(cursor => cursor.cancel())
+		)
+		this.#videoCursors.clear()
+	}
+
+	// forward only
+	#cursor(reader: ReadableStreamDefaultReader<VideoFrame>) {
+		return {
+			async next(targetUs: number): Promise<VideoFrame | undefined> {
+				let prev: VideoFrame | null = null
+				while (true) {
+					const { done, value: hit } = await reader.read()
+
+					if (done) {
+						const out = prev ? new VideoFrame(prev) : undefined
+						prev?.close()
+						return out
+					}
+
+					const hitUs = hit.timestamp ?? 0
+					if (hitUs >= targetUs) {
+						const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
+						const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
+
+						const chosen = usePrev ? prev! : hit
+						const other = usePrev ? hit : prev
+
+						const copy = new VideoFrame(chosen)
+						chosen.close()
+						other?.close()
+						return copy
+					}
+
+					prev?.close()
+					prev = hit
+				}
+			},
+
+			cancel: async () => await reader.cancel()
+		}
+	}
+}
+
+const toUs = (ms: Ms) => Math.round(ms * 1_000)
+
+type StreamCursor<T> = {
+	next(target: number): Promise<T | undefined>
+	cancel(): Promise<void>
+}
+
+type VideoFrameCursor = StreamCursor<VideoFrame>
+

package/s/timeline/renderers/export/parts/produce-audio.ts

@@ -0,0 +1,64 @@
+
+import {AudioMix} from "./audio-mix.js"
+import {ms} from "../../../../units/ms.js"
+import {resampleToPlanar} from "./resamplers.js"
+import {applyGainToPlanar} from "./audio-gain.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+import {createAudioSampler} from "../../parts/samplers/audio/sampler.js"
+
+export function produceAudio({
+	timeline,
+	resolveMedia
+}: {
+	timeline: TimelineFile,
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+	const mixer = new AudioMix()
+	const audio = streamAudio(timeline, resolveMedia)
+	const stream = new TransformStream<AudioData, AudioData>()
+	const writer = stream.writable.getWriter()
+
+	async function produce() {
+		for await (const chunk of mixer.mix(audio)) {
+			const data = new AudioData({
+				format: 'f32-planar',
+				sampleRate: chunk.sampleRate,
+				numberOfFrames: chunk.frames,
+				numberOfChannels: chunk.channels,
+				timestamp: Math.round(
+					(chunk.startFrame / chunk.sampleRate) * 1_000_000
+				),
+				data: new Float32Array(chunk.planar)
+			})
+
+			await writer.write(data)
+		}
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+
+async function *streamAudio(timeline: TimelineFile, resolveMedia: (hash: string) => DecoderSource) {
+	const audioSampler = createAudioSampler(resolveMedia)
+
+	for await (const {sample, timestamp, gain}
+		of audioSampler.sampleAudio(timeline, ms(0))) {
+
+		const {data} = resampleToPlanar(sample, 48000)
+		applyGainToPlanar(data, gain)
+
+		yield {
+			planes: data,
+			sampleRate: 48000,
+			timestamp
+		}
+
+		sample.close()
+	}
+}
+

package/s/timeline/renderers/export/parts/produce-video.ts

@@ -0,0 +1,50 @@
+
+import {Fps} from "../../../../units/fps.js"
+import {CursorVisualSampler} from "./cursor.js"
+import {Driver} from "../../../../driver/driver.js"
+import {fixedStep} from "../../parts/schedulers.js"
+import {TimelineFile} from "../../../parts/basics.js"
+import {computeItemDuration} from "../../parts/handy.js"
+import {DecoderSource} from "../../../../driver/fns/schematic.js"
+
+export function produceVideo({
+	timeline,
+	fps,
+	driver,
+	resolveMedia,
+}: {
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+	timeline: TimelineFile
+	fps: Fps
+}) {
+
+	const stream = new TransformStream<VideoFrame, VideoFrame>()
+	const writer = stream.writable.getWriter()
+	const sampler = new CursorVisualSampler(driver, resolveMedia)
+	const cursor = sampler.cursor(timeline)
+	const dt = 1 / fps
+	const duration = computeItemDuration(timeline.rootId, timeline)
+
+	async function produce() {
+		await fixedStep({fps, duration}, async (timecode, i) => {
+			const layers = await cursor.next(timecode)
+			const composed = await driver.composite(layers)
+
+			const frame = new VideoFrame(composed, {
+				timestamp: Math.round(i * dt * 1_000_000),
+				duration: Math.round(dt * 1_000_000)
+			})
+
+			await writer.write(frame)
+			composed.close()
+		})
+
+		await writer.close()
+	}
+
+	produce()
+
+	return stream.readable
+}
+

package/s/timeline/renderers/export/parts/resamplers.ts

@@ -0,0 +1,48 @@
+
+const resampleLinear = (
+	src: Float32Array,
+	fromRate: number,
+	targetRate: number
+) => {
+	if (fromRate === targetRate)
+		return src
+
+	const ratio = targetRate / fromRate
+	const outFrames = Math.max(1, Math.round(src.length * ratio))
+	const out = new Float32Array(outFrames)
+
+	for (let i = 0; i < outFrames; i++) {
+		const t = i / ratio
+		const i0 = Math.floor(t)
+		const i1 = Math.min(i0 + 1, src.length - 1)
+		const frac = t - i0
+		out[i] = src[i0] * (1 - frac) + src[i1] * frac
+	}
+
+	return out
+}
+
+export const resampleToPlanar = (
+	sample: {
+		numberOfFrames: number
+		numberOfChannels: number
+		sampleRate: number
+		copyTo: (dest: Float32Array, options: {planeIndex: number; format: 'f32-planar'}) => void
+	},
+	targetRate: number
+): {data: Float32Array[]; frames: number} => {
+	const channels = sample.numberOfChannels
+	const data = new Array<Float32Array>(channels)
+	let frames = 0
+
+	for (let ch = 0; ch < channels; ch++) {
+		const plane = new Float32Array(sample.numberOfFrames)
+		sample.copyTo(plane, {planeIndex: ch, format: 'f32-planar'})
+		const resampled = resampleLinear(plane, sample.sampleRate, targetRate)
+		data[ch] = resampled
+		frames = resampled.length
+	}
+
+	return {data, frames}
+}
+

package/s/timeline/renderers/export/produce.ts

@@ -0,0 +1,28 @@
+
+import {Fps} from "../../../units/fps.js"
+import {Driver} from "../../../driver/driver.js"
+import {TimelineFile} from "../../parts/basics.js"
+import {produceAudio} from "./parts/produce-audio.js"
+import {produceVideo} from "./parts/produce-video.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+
+export function produce(opts: {
+	timeline: TimelineFile
+	fps: Fps
+	driver: Driver
+	resolveMedia: (hash: string) => DecoderSource
+}) {
+
+	const audio = produceAudio({...opts})
+	const video = produceVideo({...opts})
+
+	return opts.driver.encode({
+		video,
+		audio,
+		config: {
+			audio: {codec: 'opus', bitrate: 128000},
+			video: {codec: 'vp9', bitrate: 1000000}
+		}
+	})
+}
+