@omnimedia/omnitool 1.0.0 → 1.1.0-3

package/s/driver/driver.ts

@@ -0,0 +1,116 @@
+import {Comrade, tune, Thread} from "@e280/comrade"
+import {ALL_FORMATS, Input, type StreamTargetChunk} from "mediabunny"
+
+import {Machina} from "./parts/machina.js"
+import {setupDriverHost} from "./fns/host.js"
+import {loadDecoderSource} from "./utils/load-decoder-source.js"
+import {DecoderInput, DriverSchematic, Composition, EncoderInput, DecoderSource} from "./fns/schematic.js"
+
+export type DriverOptions = {
+	workerUrl: URL | string
+}
+
+export class Driver {
+	static async setup(options: DriverOptions) {
+		const machina = new Machina()
+		const thread = await Comrade.thread<DriverSchematic>({
+			label: "OmnitoolDriver",
+			workerUrl: options.workerUrl,
+			setupHost: setupDriverHost(machina),
+		})
+		return new this(machina, thread)
+	}
+
+	constructor(
+		public machina: Machina,
+		public thread: Thread<DriverSchematic>
+	) {}
+
+	async hello() {
+		return this.thread.work.hello()
+	}
+
+	async getAudioDuration(source: DecoderSource) {
+		const input = new Input({
+			source: await loadDecoderSource(source),
+			formats: ALL_FORMATS
+		})
+
+		const audioTrack = await input.getPrimaryAudioTrack()
+		return await audioTrack?.computeDuration()
+	}
+
+	async getVideoDuration(source: DecoderSource) {
+		const input = new Input({
+			source: await loadDecoderSource(source),
+			formats: ALL_FORMATS
+		})
+
+		const videoTrack = await input.getPrimaryVideoTrack()
+		return await videoTrack?.computeDuration()
+	}
+
+	decode(input: DecoderInput) {
+		let lastFrame: VideoFrame | null = null
+		const videoTransform = new TransformStream<VideoFrame, VideoFrame>({
+			async transform(chunk, controller) {
+				const frame = await input.onFrame?.(chunk) ?? chunk
+				// below code is to prevent mem leaks and hardware accelerated decoder stall
+				lastFrame?.close()
+				controller.enqueue(frame)
+				lastFrame = frame
+			}
+		})
+		const audioTransform = new TransformStream<AudioData, AudioData>()
+		this.thread.work.decode[tune]({transfer: [videoTransform.writable, audioTransform.writable]})({
+			source: input.source,
+			video: videoTransform.writable,
+			audio: audioTransform.writable,
+		})
+		return {
+			audio: audioTransform.readable,
+			video: videoTransform.readable
+		}
+	}
+
+	async encode({readables, config}: EncoderInput) {
+		const handle = await window.showSaveFilePicker()
+		const writable = await handle.createWritable()
+		// making bridge because file picker writable is not transferable
+		const bridge = new WritableStream<StreamTargetChunk>({
+			async write(chunk) {
+				await writable.write(chunk)
+			},
+			async close() {
+				await writable.close()
+			}
+		})
+		return await this.thread.work.encode[tune]({transfer: [readables.audio, readables.video, bridge]})({readables, config, bridge})
+	}
+
+	async composite(
+		composition: Composition,
+	) {
+		const transfer = this.#collectTransferablesFromComposition(composition)
+		return await this.thread.work.composite[tune]({transfer})(composition)
+	}
+
+	#collectTransferablesFromComposition(composition: Composition) {
+		const transferables: Transferable[] = []
+
+		const visit = (node: Composition) => {
+			if (Array.isArray(node)) {
+				for (const child of node)
+					visit(child)
+			}
+			else if (node && typeof node === 'object' && 'kind' in node) {
+				if (node.kind === 'image' && node.frame instanceof VideoFrame)
+					transferables.push(node.frame)
+			}
+		}
+
+		visit(composition)
+		return transferables
+	}
+}
+

package/s/driver/driver.worker.bundle.ts

@@ -0,0 +1,7 @@
+
+import {Comrade} from "@e280/comrade"
+import {setupDriverWork} from "./fns/work.js"
+import {DriverSchematic} from "./fns/schematic.js"
+
+await Comrade.worker<DriverSchematic>(setupDriverWork)
+

package/s/driver/fns/host.ts

@@ -0,0 +1,12 @@
+
+import {Comrade} from "@e280/comrade"
+import {Machina} from "../parts/machina.js"
+import {DriverSchematic} from "./schematic.js"
+
+export const setupDriverHost = (machina: Machina) => Comrade.host<DriverSchematic>(({work}, rig) => ({
+
+	async world() {
+		machina.count++
+	}
+}))
+

package/s/driver/fns/schematic.ts

@@ -0,0 +1,83 @@
+
+import {AsSchematic} from "@e280/comrade"
+import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"
+
+export type DriverSchematic = AsSchematic<{
+
+	// happens on the web worker
+	work: {
+		hello(): Promise<void>
+
+		decode(input: {
+			source: DecoderSource
+			video: WritableStream<VideoFrame>
+			audio: WritableStream<AudioData>
+		}): Promise<void>
+
+		encode(input: EncoderInput & {bridge: WritableStream<StreamTargetChunk>}): Promise<void>
+
+		composite(input: Composition): Promise<VideoFrame>
+	}
+
+	// happens on the main thread
+	host: {
+		world(): Promise<void>
+	}
+}>
+
+export interface EncoderInput {
+	readables: {
+		video: ReadableStream<VideoFrame>
+		audio: ReadableStream<AudioData>
+	},
+	config: {
+		video: VideoEncodingConfig
+		audio: AudioEncodingConfig
+	}
+}
+
+export type DecoderSource = FileSystemFileHandle | string
+
+export interface DecoderInput {
+	source: DecoderSource
+	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
+}
+
+export interface MuxOpts {
+	config: {
+		video: {
+			width: number
+			height: number
+		},
+		audio?: {
+			codec: "opus" | "aac"
+			numberOfChannels: number
+			sampleRate: number
+		}
+	}
+}
+
+export type Composition = Layer | (Layer | Composition)[]
+
+export type Transform = {
+	x?: number
+	y?: number
+	scale?: number
+	opacity?: number
+	anchor?: number
+}
+
+export type TextLayer = {
+	kind: 'text'
+	content: string
+	fontSize?: number
+	color?: string
+} & Transform
+
+export type ImageLayer = {
+	kind: 'image'
+	frame: VideoFrame
+} & Transform
+
+export type Layer = TextLayer | ImageLayer
+

package/s/driver/fns/work.ts

@@ -0,0 +1,237 @@
+import {Comrade} from "@e280/comrade"
+import {
+	Input, ALL_FORMATS, VideoSampleSink, Output, Mp4OutputFormat, VideoSampleSource, VideoSample,
+	AudioSampleSink, AudioSampleSource, AudioSample, StreamTarget, BlobSource, UrlSource
+} from "mediabunny"
+import {autoDetectRenderer, Container, Renderer, Sprite, Text, Texture, DOMAdapter, WebWorkerAdapter} from "pixi.js"
+
+import {Composition, DriverSchematic, Layer, Transform} from "./schematic.js"
+
+DOMAdapter.set(WebWorkerAdapter)
+
+export const setupDriverWork = Comrade.work<DriverSchematic>(({host}, rig) => ({
+
+	async hello() {
+		await host.world()
+	},
+
+	async decode({source, video, audio}) {
+		const loadSource = async () => {
+			if(source instanceof FileSystemFileHandle) {
+				const file = await source.getFile()
+				return new BlobSource(file)
+			} else {
+				return new UrlSource(source)
+			}
+		}
+		const input = new Input({
+			source: await loadSource(),
+			formats: ALL_FORMATS
+		})
+
+		const [videoTrack, audioTrack] = await Promise.all([
+			input.getPrimaryVideoTrack(),
+			input.getPrimaryAudioTrack()
+		])
+
+		const videoDecodable = await videoTrack?.canDecode()
+		const audioDecodable = await audioTrack?.canDecode()
+
+		const videoWriter = video.getWriter()
+		const audioWriter = audio.getWriter()
+
+		await Promise.all([
+			(async () => {
+				if (videoDecodable && videoTrack) {
+					const sink = new VideoSampleSink(videoTrack)
+					for await (const sample of sink.samples()) {
+						const frame = sample.toVideoFrame()
+						await videoWriter.write(frame)
+						sample.close()
+						frame.close()
+					}
+					await videoWriter.close()
+				}
+			})(),
+			(async () => {
+				if (audioDecodable && audioTrack) {
+					const sink = new AudioSampleSink(audioTrack)
+					for await (const sample of sink.samples()) {
+						const frame = sample.toAudioData()
+						await audioWriter.write(frame)
+						sample.close()
+						frame.close()
+					}
+					await audioWriter.close()
+				}
+			})()
+		])
+	},
+
+	async encode({readables, config, bridge}) {
+		const output = new Output({
+			format: new Mp4OutputFormat(),
+			target: new StreamTarget(bridge, {chunked: true})
+		})
+		const videoSource = new VideoSampleSource(config.video)
+		output.addVideoTrack(videoSource)
+		// since AudioSample is not transferable it fails to transfer encoder bitrate config
+		// so it needs to be hardcoded not set through constants eg QUALITY_LOW
+		const audioSource = new AudioSampleSource(config.audio)
+		output.addAudioTrack(audioSource)
+
+		await output.start()
+
+		const videoReader = readables.video.getReader()
+		const audioReader = readables.audio.getReader()
+
+		await Promise.all([
+			(async () => {
+				while (true) {
+					const {done, value} = await videoReader.read()
+					if (done) break
+					const sample = new VideoSample(value)
+					await videoSource.add(sample)
+					sample.close()
+				}
+			})(),
+			(async () => {
+				while (true) {
+					const {done, value} = await audioReader.read()
+					if (done) break
+					const sample = new AudioSample(value)
+					await audioSource.add(sample)
+					sample.close()
+					value.close()
+				}
+			})()
+		])
+
+		await output.finalize()
+	},
+
+	async composite(composition) {
+		const {stage, renderer} = await renderPIXI(1920, 1080)
+		stage.removeChildren()
+
+		const {baseFrame, disposables} = await renderLayer(composition, stage)
+		renderer.render(stage)
+
+		// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
+		// if its very slow on eg edge try chrome
+		const frame = new VideoFrame(renderer.canvas, {
+			timestamp: baseFrame?.timestamp,
+			duration: baseFrame?.duration ?? undefined,
+		})
+
+		baseFrame?.close()
+		renderer.clear()
+
+		for (const disposable of disposables) {
+			disposable.destroy(true)
+		}
+
+		rig.transfer = [frame]
+		return frame
+	}
+}))
+
+
+let pixi: {
+	renderer: Renderer
+	stage: Container
+} | null = null
+
+async function renderPIXI(width: number, height: number) {
+	if (pixi)
+		return pixi
+
+	const renderer = await autoDetectRenderer({
+		width,
+		height,
+		preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
+		background: "black",
+		preferWebGLVersion: 2
+	})
+
+	const stage = new Container()
+	pixi = {renderer, stage}
+
+	return pixi
+}
+
+type RenderableObject = Sprite | Text | Texture
+
+async function renderLayer(
+	layer: Layer | Composition,
+	parent: Container,
+	disposables: RenderableObject[] = []
+) {
+	if (Array.isArray(layer)) {
+		let baseFrame: VideoFrame | undefined
+		for (const child of layer) {
+			const result = await renderLayer(child, parent, disposables)
+			baseFrame ??= result.baseFrame
+		}
+		return {baseFrame, disposables}
+	}
+
+	if (!isRenderableLayer(layer)) {
+		console.warn('Invalid layer', layer)
+		return {disposables}
+	}
+
+	switch (layer.kind) {
+		case 'text':
+			return renderTextLayer(layer, parent, disposables)
+		case 'image':
+			return renderImageLayer(layer, parent, disposables)
+		default:
+			console.warn('Unknown layer kind', (layer as any).kind)
+			return {disposables}
+	}
+}
+
+function isRenderableLayer(layer: any): layer is Layer {
+	return !!layer && typeof layer === 'object' && typeof layer.kind === 'string'
+}
+
+function renderTextLayer(
+	layer: Extract<Layer, {kind: 'text'}>,
+	parent: Container,
+	disposables: RenderableObject[]
+) {
+	const text = new Text({
+		text: layer.content,
+		style: {
+			fontFamily: 'sans-serif',
+			fontSize: layer.fontSize ?? 48,
+			fill: layer.color ?? 'white'
+		}
+	})
+	applyTransform(text, layer)
+	parent.addChild(text)
+	disposables.push(text)
+	return {disposables}
+}
+
+function renderImageLayer(
+	layer: Extract<Layer, {kind: 'image'}>,
+	parent: Container,
+	disposables: RenderableObject[]
+) {
+	const texture = Texture.from(layer.frame)
+	const sprite = new Sprite(texture)
+	applyTransform(sprite, layer)
+	parent.addChild(sprite)
+	disposables.push(sprite, texture)
+	return {baseFrame: layer.frame, disposables}
+}
+
+function applyTransform(target: Sprite | Text, t: Transform = {}) {
+	if(t.x) target.x = t.x
+	if(t.y) target.y = t.y
+	if(t.scale) target.scale.set(t.scale)
+	if(t.opacity) target.alpha = t.opacity
+	if(t.anchor && 'anchor' in target) target.anchor.set(t.anchor)
+}

package/s/driver/parts/constants.ts

@@ -0,0 +1,17 @@
+// for later: https://github.com/gpac/mp4box.js/issues/243
+export const encoderDefaultConfig: VideoEncoderConfig = {
+	codec: "avc1.640034",
+	avc: {format: "annexb"},
+	width: 1280,
+	height: 720,
+	bitrate: 9_000_000, // 9 Mbps
+	framerate: 60,
+	bitrateMode: "variable",
+	hardwareAcceleration: "no-preference" // prefer-hardware seems like 2x slower from what i been testing
+}
+
+export const audioEncoderDefaultConfig: AudioEncoderConfig = {
+	codec: "opus",
+	numberOfChannels: 2,
+	sampleRate: 44100
+}

package/s/driver/parts/machina.ts

@@ -0,0 +1,27 @@
+import {WebMediaInfo} from "web-demuxer"
+
+type Events =
+	| {type: "config", config: {audio: AudioDecoderConfig, video: VideoDecoderConfig}}
+	| {type: "info", data: WebMediaInfo}
+	| {type: "encoderQueueSize", size: number}
+
+type Handler = (event: Events) => void
+
+export class Machina {
+	count = 0
+
+	#handlers = new Map<number, Handler>()
+
+	register(id: number, handler: Handler) {
+		this.#handlers.set(id, handler)
+	}
+
+	unregister(id: number) {
+		this.#handlers.delete(id)
+	}
+
+	dispatch(id: number, event: Events) {
+		this.#handlers.get(id)?.(event)
+	}
+}
+

package/s/driver/utils/load-decoder-source.ts

@@ -0,0 +1,13 @@
+import {BlobSource, UrlSource} from "mediabunny"
+import {DecoderSource} from "../fns/schematic.js"
+
+// only streamable sources
+export async function loadDecoderSource(source: DecoderSource) {
+	if(source instanceof FileSystemFileHandle) {
+		const file = await source.getFile()
+		return new BlobSource(file)
+	} else {
+		return new UrlSource(source)
+	}
+}
+

package/s/driver/utils/sleep.ts

@@ -0,0 +1,3 @@
+export function sleep(ms: number) {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}

package/s/index.html.ts

@@ -0,0 +1,53 @@
+
+import {ssg, html} from "@e280/scute"
+
+const title = "omnitool"
+const domain = "omnitool.omniclip.app"
+const favicon = "/assets/favicon.png"
+
+export default ssg.page(import.meta.url, async orb => ({
+	title,
+	// favicon,
+	dark: true,
+	css: "demo/demo.css",
+	js: "demo/demo.bundle.min.js",
+
+	head: html`
+		<link rel="preconnect" href="https://fonts.googleapis.com">
+		<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&display=swap" rel="stylesheet">
+	`,
+
+	socialCard: {
+		title,
+		description: "video processing toolkit",
+		themeColor: "#3cff9c",
+		siteName: domain,
+		image: `https://${domain}${favicon}`,
+	},
+
+	body: html`
+		<section>
+			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
+			<button class=fetch>fetch</button>
+			<button class="import">import</button>
+			<div class=results></div>
+			<div class=filmstrip-demo>
+				<label for="viewable-range">viewable range:</label>
+				<input type="range" min="0" max="100" step="1" value="10" class="range" id="viewable-range" name="viewable-range">
+				<div class="range-view"></div>
+				<label for="range-size">viewable range size:</label>
+				<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5" id="range-size" name="range-size">
+				<label for="frequency">frequency:</label>
+				<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10" id="frequency" name="frequency">
+				<div class="frequency-view">10 (fps)</div>
+				<div id=filmstrip></div>
+			</div>
+			<div class=waveform-demo>
+				<label for="width">width:</label>
+				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
+			</div>
+		</section>
+	`,
+}))
+

package/s/index.ts

@@ -1,41 +1,4 @@
-import {MP4Demuxer} from "./tools/mp4boxjs/demuxer.js"
-
-import {Clip, LoadedVideoClip} from "./types.js"
-import {generateId} from "./tools/generate-id.js"
-import { OmniVideoDecoder } from "./parts/video-decoder.js"
-
-export class OmniTool {
-	#clips: Map<Clip, LoadedVideoClip> = new Map()
-	#getLoadedClip = (clip: Clip) => this.#clips.get(clip)
-	
-	videoDecoder = new OmniVideoDecoder(this.#getLoadedClip)
-
-	// ffmpeg node or web version
-	constructor(ffmpeg) {}
-
-
-	load(file: File): Promise<Clip> {
-		return new Promise((resolve) => {
-			const demuxer = new MP4Demuxer(file, {
-				onChunk: () => {},
-				onConfig: () => {},
-				framesCount: () => {},
-				setStatus: () => {},
-				OnReady: () => {
-					const demuxed = {id: generateId(), demuxer}
-					this.#clips.set(demuxed, demuxed)
-					resolve(demuxed)
-				}
-			})
-		})
-
-	}
-
-	async getFramesCount() {}
-}
-
-const tool = new OmniTool('ffmpeg')
-// const file = tool.load()
-// tool.videoDecoder.decode(file)
 
+export * from "./driver/driver.js"
+export * from "./timeline/index.js"
 

package/s/tests.test.ts

@@ -0,0 +1,8 @@
+
+import {Science} from "@e280/science"
+import driver from "./driver/driver.test.js"
+
+await Science.run({
+	driver,
+})
+

package/s/timeline/index.ts

@@ -0,0 +1,14 @@
+
+export * from "./parts/basics.js"
+export * from "./parts/item.js"
+export * from "./parts/media.js"
+export * from "./parts/resource-pool.js"
+export * from "./parts/resource.js"
+export * from "./parts/filmstrip.js"
+
+export * from "./sugar/o.js"
+export * from "./sugar/omni.js"
+
+export * from "./utils/checksum.js"
+export * from "./utils/datafile.js"
+

package/s/timeline/parts/basics.ts

@@ -0,0 +1,17 @@
+
+import {Item} from "./item.js"
+
+/** sha256 hash */
+export type Hash = string
+
+/** item identifier */
+export type Id = number
+
+export type TimelineFile = {
+	info: "https://omniclip.app/"
+	format: "timeline"
+	version: number
+	root: Id
+	items: Item.Any[]
+}
+