@omnimedia/omnitool 1.1.0-1 → 1.1.0-11

package/s/features/speech/transcribe/worker.bundle.ts

@@ -0,0 +1,40 @@
+
+import {defer, once} from "@e280/stz"
+import {Comrade, Host} from "@e280/comrade"
+import {Pipeline} from "@huggingface/transformers"
+
+import {loadPipe} from "./parts/load-pipe.js"
+import {transcribe} from "./parts/transcribe.js"
+import {TranscriberSchematic, TranscriberSpec} from "./types.js"
+
+const deferred = defer<{pipe: Pipeline, spec: TranscriberSpec}>()
+
+const makePrepare = (host: Host<TranscriberSchematic>) => once(async(spec: TranscriberSpec) => {
+	deferred.resolve({
+		spec,
+		pipe: await loadPipe({
+			spec,
+			onLoading: loading => host.loading(loading),
+		}),
+	})
+})
+
+await Comrade.worker<TranscriberSchematic>(shell => {
+	const prepare = makePrepare(shell.host)
+	return {
+		prepare,
+		async transcribe(request) {
+			const {pipe, spec} = await deferred.promise
+			return transcribe({
+				pipe,
+				spec,
+				request,
+				callbacks: {
+					onReport: report => shell.host.deliverReport(report),
+					onTranscription: transcription => shell.host.deliverTranscription(transcription),
+				},
+			})
+		}
+	}
+})
+

package/s/features/transition/parts/fragment.ts

@@ -0,0 +1,24 @@
+export const fragment = (glsl: string) => `
+	precision highp float;
+	varying vec2 vTextureCoord;
+	varying vec2 _uv;
+	uniform sampler2D from, to;
+	uniform float progress, ratio, _fromR, _toR;
+	uniform float customUniform;
+
+	vec4 getFromColor(vec2 uv){
+		return texture2D(from, .5+(uv-.5)*vec2(max(ratio/_fromR,1.), max(_fromR/ratio,1.)));
+	}
+	vec4 getToColor(vec2 uv){
+		return texture2D(to, .5+(uv-.5)*vec2(max(ratio/_toR,1.), max(_toR/ratio,1.)));
+	}
+
+	// gl-transition code here
+	${glsl}
+	// gl-transition code end
+
+	void main(){
+		vec2 uv = vTextureCoord.xy;
+		gl_FragColor = transition(vTextureCoord);
+	}
+`

package/s/features/transition/parts/types.ts

@@ -0,0 +1,94 @@
+import {Renderer} from "pixi.js"
+
+export interface TransitionOptions {
+	name: Transition
+	renderer: Renderer
+}
+
+export interface TransitionRendererOptions {
+	from: VideoFrame
+	to: VideoFrame
+	progress: number
+	width: number
+	height: number
+}
+
+export interface GLTransition {
+	author: string
+	createdAt: string
+	glsl: string
+	license: string
+	name: Transition
+	updatedAt: string
+	defaultParams: any
+	paramsTypes: any
+}
+
+export type Transition =
+  | "Bounce"
+  | "BowTieHorizontal"
+  | "BowTieVertical"
+  | "ButterflyWaveScrawler"
+  | "CircleCrop"
+  | "ColourDistance"
+  | "CrazyParametricFun"
+  | "CrossZoom"
+  | "Directional"
+  | "DoomScreenTransition"
+  | "Dreamy"
+  | "DreamyZoom"
+  | "GlitchDisplace"
+  | "GlitchMemories"
+  | "GridFlip"
+  | "InvertedPageCurl"
+  | "LinearBlur"
+  | "Mosaic"
+  | "PolkaDotsCurtain"
+  | "Radial"
+  | "SimpleZoom"
+  | "StereoViewer"
+  | "Swirl"
+  | "WaterDrop"
+  | "ZoomInCircles"
+  | "angular"
+  | "burn"
+  | "cannabisleaf"
+  | "circle"
+  | "circleopen"
+  | "colorphase"
+  | "crosshatch"
+  | "crosswarp"
+  | "cube"
+  | "directionalwarp"
+  | "directionalwipe"
+  | "displacement"
+  | "doorway"
+  | "fade"
+  | "fadecolor"
+  | "fadegrayscale"
+  | "flyeye"
+  | "heart"
+  | "hexagonalize"
+  | "kaleidoscope"
+  | "luma"
+  | "luminance_melt"
+  | "morph"
+  | "multiply_blend"
+  | "perlin"
+  | "pinwheel"
+  | "pixelize"
+  | "polar_function"
+  | "randomsquares"
+  | "ripple"
+  | "rotate_scale_fade"
+  | "squareswire"
+  | "squeeze"
+  | "swap"
+  | "undulatingBurnOut"
+  | "wind"
+  | "windowblinds"
+  | "windowslice"
+  | "wipeDown"
+  | "wipeLeft"
+  | "wipeRight"
+  | "wipeUp"

package/s/features/transition/parts/uniforms.ts

@@ -0,0 +1,29 @@
+import {GLTransition} from "./types.js"
+
+export const uniforms = {
+	custom: (transition: GLTransition) => Object.fromEntries(
+		Object.entries(transition.defaultParams).map(([name, value]) => [
+			name,
+			{
+				value,
+				type: getUniformType(transition.paramsTypes[name])
+			}
+		])
+	),
+	basics: {
+		_fromR: {value: 1, type: "f32"},
+		_toR: {value: 1, type: "f32"},
+		ratio: {value: 1, type: "f32"},
+		progress: {value: 0, type: "f32"},
+		customUniform: {value: 0, type: "f32"},
+	}
+}
+
+const getUniformType = (type: string) => {
+	if(type === "f32" || type === "i32") {
+		return type
+	} else if(type === "float") {
+		return "f32"
+	}
+	else return `${type}<f32>`
+}

package/s/features/transition/parts/vertex.ts

@@ -0,0 +1,31 @@
+export const vertex = `
+	in vec2 aPosition;
+	varying vec2 _uv;                          // gl-transition
+	uniform mat3 projectionMatrix;
+	uniform vec4 uInputSize;
+	uniform vec4 uOutputFrame;
+	out vec2 vTextureCoord;
+	uniform vec4 uOutputTexture;
+
+	vec4 filterVertexPosition( void )
+	{
+			vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+
+			position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+			position.y = position.y * (2.0*uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+
+			return vec4(position, 0.0, 1.0);
+	}
+
+	vec2 filterTextureCoord( void )
+	{
+	return aPosition * (uOutputFrame.zw * uInputSize.zw);
+	}
+
+	void main(void)
+	{
+	gl_Position = filterVertexPosition();
+	vTextureCoord = filterTextureCoord();
+	_uv = vec2(0.5, 0.5) * (aPosition +vec2(1.0, 1.0));    // gl-transition
+	}
+`

package/s/features/transition/transition.ts

@@ -0,0 +1,60 @@
+//@ts-ignore
+import transitions from "gl-transitions"
+import {Filter, GlProgram, Sprite, Texture, ImageSource} from "pixi.js"
+
+import {vertex} from "./parts/vertex.js"
+import {uniforms} from "./parts/uniforms.js"
+import {fragment} from "./parts/fragment.js"
+import {GLTransition, TransitionOptions, TransitionRendererOptions} from "./parts/types.js"
+
+export function makeTransition({name, renderer}: TransitionOptions) {
+	const transition = transitions.find((t: GLTransition) => t.name === name) as GLTransition
+	const transitionSprite = new Sprite()
+	const transitionTexture = new Texture()
+	const sourceFrom = new ImageSource({})
+	const sourceTo = new ImageSource({})
+
+	const filter = new Filter({
+		glProgram: new GlProgram({
+			vertex,
+			fragment: fragment(transition.glsl),
+		}),
+		resources: {
+			from: sourceFrom,
+			to: sourceTo,
+			uniforms: {
+				...uniforms.basics,
+				...uniforms.custom(transition)
+			}
+		}
+	})
+
+	transitionSprite.filters = [filter]
+
+	return {
+		render({width, height, from, to, progress}: TransitionRendererOptions) {
+			if(transitionSprite.width !== width || transitionSprite.height !== height) {
+				transitionSprite.setSize({width, height})
+				transitionTexture.source.resize(width, height)
+			}
+
+			sourceFrom.resource = from
+			sourceTo.resource = to
+			sourceFrom.update()
+			sourceTo.update()
+
+			filter.resources.uniforms.uniforms.progress = progress
+
+			renderer.render({
+				container: transitionSprite,
+				target: transitionTexture,
+				clear: false,
+				width,
+				height
+			})
+
+			return transitionTexture
+		}
+	}
+}
+

package/s/index.html.ts

@@ -30,7 +30,7 @@ export default ssg.page(import.meta.url, async orb => ({
 		<section>
 			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
 			<button class=fetch>fetch</button>
-			<button class="import">import</button>
+			<input type="file" class="file-input">
 			<div class=results></div>
 			<div class=filmstrip-demo>
 				<label for="viewable-range">viewable range:</label>
@@ -47,6 +47,11 @@ export default ssg.page(import.meta.url, async orb => ({
 				<label for="width">width:</label>
 				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
 			</div>
+			<div class=player>
+				<input class="seek" type="number" min="0">
+				<button class=play>play</button>
+				<button class=stop>stop</button>
+			</div>
 		</section>
 	`,
 }))

package/s/timeline/index.ts

@@ -5,6 +5,7 @@ export * from "./parts/media.js"
 export * from "./parts/resource-pool.js"
 export * from "./parts/resource.js"
 export * from "./parts/filmstrip.js"
+export * from "./parts/compositor/playback.js"
 
 export * from "./sugar/o.js"
 export * from "./sugar/omni.js"

package/s/timeline/parts/basics.ts

@@ -11,7 +11,7 @@ export type TimelineFile = {
 	info: "https://omniclip.app/"
 	format: "timeline"
 	version: number
-	root: Id
+	rootId: Id
 	items: Item.Any[]
 }
 

package/s/timeline/parts/compositor/export.ts

@@ -0,0 +1,77 @@
+import {TimelineFile} from "../basics.js"
+import {fixedStep} from "./parts/schedulers.js"
+import {Driver} from "../../../driver/driver.js"
+import {makeWebCodecsSampler} from "./samplers/webcodecs.js"
+import {DecoderSource} from "../../../driver/fns/schematic.js"
+import {buildWebCodecsNodeTree} from "./parts/webcodecs-tree.js"
+
+export class Export {
+	#sampler
+	constructor(
+		private driver: Driver,
+		private framerate = 30,
+		private resolveMedia: (hash: string) => DecoderSource = _hash => "/assets/temp/gl.mp4"
+	) {
+		this.#sampler = makeWebCodecsSampler(this.driver, this.resolveMedia)
+	}
+
+	async #build(timeline: TimelineFile) {
+		const rootItem = new Map(timeline.items.map(i => [i.id, i])).get(timeline.rootId)!
+		const items = new Map(timeline.items.map(i => [i.id, i]))
+		return await buildWebCodecsNodeTree(rootItem, items, this.#sampler)
+	}
+
+	async render(timeline: TimelineFile) {
+		const root = await this.#build(timeline)
+
+		const videoStream = new TransformStream<VideoFrame, VideoFrame>()
+		const audioStream = new TransformStream<AudioData, AudioData>()
+
+		const encodePromise = this.driver.encode({
+			video: videoStream.readable,
+			audio: audioStream.readable,
+			config: {
+				audio: {codec: "opus", bitrate: 128000},
+				video: {codec: "vp9", bitrate: 1000000},
+			},
+		})
+
+		const videoWriter = videoStream.writable.getWriter()
+		const audioWriter = audioStream.writable.getWriter()
+
+		const audioPromise = (async () => {
+			if (root.audio) {
+				for await (const chunk of root.audio.getStream()) {
+					await audioWriter.write(chunk)
+				}
+			}
+			await audioWriter.close()
+		})()
+
+		const videoPromise = (async () => {
+			let i = 0
+			const dt = 1 / this.framerate
+
+			await fixedStep(
+				{fps: this.framerate, duration: root.duration ?? 0},
+				async t => {
+					const layers = await root.visuals?.sampleAt(t) ?? []
+					const composed = await this.driver.composite(layers)
+					const vf = new VideoFrame(composed, {
+						timestamp: Math.round(i * dt * 1_000_000),
+						duration: Math.round(dt * 1_000_000),
+					})
+					await videoWriter.write(vf)
+					composed.close()
+					i++
+				}
+			)
+			await videoWriter.close()
+		})()
+
+		await audioPromise
+		await videoPromise
+		await encodePromise
+		// this.#sampler.dispose()
+	}
+}

package/s/timeline/parts/compositor/parts/html-tree.ts

@@ -0,0 +1,37 @@
+import {Item} from "../../item.js"
+import {AudioPlaybackComponent, HTMLSampler, Node, TreeBuilder} from "./tree-builder.js"
+
+class HTMLNodeBuilder extends TreeBuilder<AudioPlaybackComponent> {
+	constructor(protected items: Map<number, Item.Any>, protected sampler: HTMLSampler) {
+		super(items, sampler)
+	}
+
+	composeAudio_Stack(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				for (const child of children) {
+					if (child.audio) child.audio.onTimeUpdate(time)
+				}
+			}
+		}
+	}
+	composeAudio_Sequence(children: Node<AudioPlaybackComponent>[]) {
+		return {
+			onTimeUpdate: (time: number) => {
+				let localTime = time
+				for (const child of children) {
+					if (localTime < child.duration) {
+						if (child.audio) child.audio.onTimeUpdate(localTime)
+						break
+					}
+					localTime -= child.duration
+				}
+			}
+		}
+	}
+}
+
+export function buildHTMLNodeTree(root: Item.Any, items: Map<number, Item.Any>, sampler: HTMLSampler) {
+	const builder = new HTMLNodeBuilder(items, sampler)
+	return builder.build(root)
+}

package/s/timeline/parts/compositor/parts/schedulers.ts

@@ -0,0 +1,85 @@
+export type RealtimeController = {
+	play(): void
+	pause(): void
+	seek(t: number): void
+	dispose(): void
+	setFPS(v: number): void
+	isPlaying(): boolean
+}
+
+export const realtime = (onTick: (t: number) => void): RealtimeController => {
+  let playing = false
+  let rafId: number | null = null
+  let fps = 60
+
+  let frameDuration = 1000 / fps
+  let currentTimeS = 0
+  let lastTime = 0
+  let accumulator = 0
+
+  const tick = (now: number) => {
+    if (!playing) return
+
+    const deltaTime = now - lastTime
+    lastTime = now
+
+    accumulator += deltaTime
+
+    while (accumulator >= frameDuration) {
+      onTick(currentTimeS)
+      currentTimeS += frameDuration / 1000
+      accumulator -= frameDuration
+    }
+
+    rafId = requestAnimationFrame(tick)
+  }
+
+  return {
+    play() {
+      if (playing) return
+      playing = true
+      lastTime = performance.now()
+      rafId = requestAnimationFrame(tick)
+    },
+    pause() {
+      if (!playing) return
+      playing = false
+      if (rafId !== null) cancelAnimationFrame(rafId)
+      rafId = null
+    },
+    seek(t) {
+      currentTimeS = Math.max(0, t)
+      accumulator = 0
+    },
+    dispose() {
+      this.pause()
+    },
+    isPlaying() {
+      return playing
+    },
+    setFPS(v) {
+    	fps = v
+    	frameDuration = 1000 / fps
+    }
+  }
+}
+
+export type FixedStepOptions = {
+	fps: number
+	duration: number
+	abort?: AbortSignal
+}
+
+export const fixedStep = async (
+	opts: FixedStepOptions,
+	onFrame: (t: number, index: number) => Promise<void> | void
+) => {
+	const dt = 1 / opts.fps
+	const total = Math.ceil(opts.duration * opts.fps)
+
+	for (let i = 0; i < total; i++) {
+		if (opts.abort?.aborted) break
+		const t = i * dt
+		await onFrame(t, i)
+	}
+}