@omnimedia/omnitool 1.0.0 → 1.1.0-10

package/s/features/speech/transcribe/parts/transcribe.ts

@@ -0,0 +1,70 @@
+
+import {WhisperTextStreamer} from "@huggingface/transformers"
+import {TranscribeOptions} from "../types.js"
+
+export async function transcribe(options: TranscribeOptions) {
+	const {pipe, spec, request, callbacks} = options
+
+	if (!pipe.processor.feature_extractor)
+		throw new Error("no feature_extractor")
+
+	const timePrecision = (
+		pipe.processor.feature_extractor?.config.chunk_length /
+		// @ts-ignore
+		pipe.model.config.max_source_positions
+	)
+
+	let chunkCount = 0
+	let startTime: number | null = null
+	let tokenCount = 0
+	let tokensPerSecond = 0
+
+	const chunkDuration = spec.chunkLength - spec.strideLength
+
+	const calculateProgress = () => {
+		const audioProgressSeconds = chunkCount * chunkDuration
+		return Math.min(audioProgressSeconds / request.duration, 1)
+	}
+
+	// TODO type error on pipe.tokenizer
+	const tokenizer = pipe.tokenizer as any
+
+	const streamer = new WhisperTextStreamer(tokenizer, {
+		time_precision: timePrecision,
+		token_callback_function: () => {
+			startTime ??= performance.now()
+			if (++tokenCount > 1) {
+				tokensPerSecond = (tokenCount / (performance.now() - startTime)) * 1000
+			}
+		},
+		callback_function: (textChunk: any) => {
+			// TODO
+			callbacks.onTranscription(textChunk)
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+		on_finalize: () => {
+			startTime = null
+			tokenCount = 0
+			chunkCount++
+			callbacks.onReport({tokensPerSecond, progress: calculateProgress()})
+		},
+	})
+
+	const result = await pipe(new Float32Array(request.audio), {
+		top_k: 0,
+		do_sample: false,
+		chunk_length_s: spec.chunkLength,
+		stride_length_s: spec.strideLength,
+		language: request.language,
+		task: "transcribe",
+		return_timestamps: "word", // if using "word" the on_chunk_start & end is not called thus we cant retrieve timestamps, only after whole thing finishes
+		force_full_sequences: false,
+		streamer,
+	})
+
+	return {
+		text: result.text,
+		chunks: result.chunks
+	}
+}
+

package/s/features/speech/transcribe/transcriber.ts

@@ -0,0 +1,46 @@
+
+import {Comrade} from "@e280/comrade"
+import {coalesce, queue, sub} from "@e280/stz"
+
+import {prepAudio} from "./parts/prep-audio.js"
+import {TranscriberOptions, TranscriberSchematic, TranscriptionOptions, TranscriptionReport} from "./types.js"
+
+export async function makeTranscriber({driver, spec, workerUrl, onLoading}: TranscriberOptions) {
+	const onReport = sub<[report: TranscriptionReport]>()
+	const onTranscription = sub<[transcription: string]>()
+
+	const thread = await Comrade.thread<TranscriberSchematic>({
+		label: "OmnitoolSpeechTranscriber",
+		workerUrl,
+		setupHost: () => ({
+			loading: async loading => onLoading(loading),
+			deliverReport: async report => onReport.pub(report),
+			deliverTranscription: async transcription => onTranscription.pub(transcription),
+		}),
+	})
+
+	await thread.work.prepare(spec)
+
+	return {
+		transcribe: queue(async(info: TranscriptionOptions) => {
+			const {source, language} = info
+			const {audio, duration} = await prepAudio(driver, source)
+
+			const detachCallbacks = coalesce(
+				onReport(info.onReport),
+				onTranscription(info.onTranscription),
+			)
+
+			const result = await thread.work.transcribe({
+				duration,
+				language,
+				audio: audio.buffer,
+			})
+
+			detachCallbacks()
+			return result
+		}),
+		dispose: thread.terminate()
+	}
+}
+

package/s/features/speech/transcribe/types.ts

@@ -0,0 +1,82 @@
+
+import {AsSchematic} from "@e280/comrade"
+import {DataType, DeviceType, Pipeline} from "@huggingface/transformers"
+
+import {Driver} from "../../../driver/driver.js"
+
+export type TranscriberSchematic = AsSchematic<{
+	work: {
+		prepare(spec: TranscriberSpec): Promise<void>
+		transcribe(request: TranscriptionRequest): Promise<Transcription>
+	},
+
+	host: {
+		loading(load: Loading): Promise<void>
+		deliverReport(report: TranscriptionReport): Promise<void>
+		deliverTranscription(transcription: string): Promise<void>
+	}
+}>
+
+export type Loading = {
+	total: number
+	progress: number
+}
+
+export type TranscribeOptions = {
+	pipe: Pipeline
+	spec: TranscriberSpec
+	request: TranscriptionRequest
+	callbacks: TranscriptionCallbacks
+}
+
+export type TranscriberPipeOptions = {
+	spec: TranscriberSpec
+	onLoading: (loading: Loading) => void
+}
+
+export type SpeechTime = [start: number, end: number]
+
+export type Transcription = {
+	text: string
+	chunks: {
+		text: string
+		timestamp: SpeechTime
+	}[]
+}
+
+export type TranscriberSpec = {
+	model: string
+	dtype: DataType
+	device: DeviceType
+	chunkLength: number
+	strideLength: number
+}
+
+export type TranscriptionOptions = {
+	source: Blob
+	language: string | null
+} & TranscriptionCallbacks
+
+export type TranscriptionRequest = {
+	audio: ArrayBufferLike
+	language: string | null
+	duration: number
+}
+
+export type TranscriptionReport = {
+	progress: number
+	tokensPerSecond: number
+}
+
+export type TranscriptionCallbacks = {
+	onReport: (report: TranscriptionReport) => void
+	onTranscription: (transcription: string) => void
+}
+
+export type TranscriberOptions = {
+	driver: Driver
+	spec: TranscriberSpec
+	workerUrl: URL | string
+	onLoading: (loading: Loading) => void
+}
+

package/s/features/speech/transcribe/worker.bundle.ts

@@ -0,0 +1,40 @@
+
+import {defer, once} from "@e280/stz"
+import {Comrade, Host} from "@e280/comrade"
+import {Pipeline} from "@huggingface/transformers"
+
+import {loadPipe} from "./parts/load-pipe.js"
+import {transcribe} from "./parts/transcribe.js"
+import {TranscriberSchematic, TranscriberSpec} from "./types.js"
+
+const deferred = defer<{pipe: Pipeline, spec: TranscriberSpec}>()
+
+const makePrepare = (host: Host<TranscriberSchematic>) => once(async(spec: TranscriberSpec) => {
+	deferred.resolve({
+		spec,
+		pipe: await loadPipe({
+			spec,
+			onLoading: loading => host.loading(loading),
+		}),
+	})
+})
+
+await Comrade.worker<TranscriberSchematic>(shell => {
+	const prepare = makePrepare(shell.host)
+	return {
+		prepare,
+		async transcribe(request) {
+			const {pipe, spec} = await deferred.promise
+			return transcribe({
+				pipe,
+				spec,
+				request,
+				callbacks: {
+					onReport: report => shell.host.deliverReport(report),
+					onTranscription: transcription => shell.host.deliverTranscription(transcription),
+				},
+			})
+		}
+	}
+})
+

package/s/features/transition/parts/fragment.ts

@@ -0,0 +1,24 @@
+export const fragment = (glsl: string) => `
+	precision highp float;
+	varying vec2 vTextureCoord;
+	varying vec2 _uv;
+	uniform sampler2D from, to;
+	uniform float progress, ratio, _fromR, _toR;
+	uniform float customUniform;
+
+	vec4 getFromColor(vec2 uv){
+		return texture2D(from, .5+(uv-.5)*vec2(max(ratio/_fromR,1.), max(_fromR/ratio,1.)));
+	}
+	vec4 getToColor(vec2 uv){
+		return texture2D(to, .5+(uv-.5)*vec2(max(ratio/_toR,1.), max(_toR/ratio,1.)));
+	}
+
+	// gl-transition code here
+	${glsl}
+	// gl-transition code end
+
+	void main(){
+		vec2 uv = vTextureCoord.xy;
+		gl_FragColor = transition(vTextureCoord);
+	}
+`

package/s/features/transition/parts/types.ts

@@ -0,0 +1,94 @@
+import {Renderer} from "pixi.js"
+
+export interface TransitionOptions {
+	name: Transition
+	renderer: Renderer
+}
+
+export interface TransitionRendererOptions {
+	from: VideoFrame
+	to: VideoFrame
+	progress: number
+	width: number
+	height: number
+}
+
+export interface GLTransition {
+	author: string
+	createdAt: string
+	glsl: string
+	license: string
+	name: Transition
+	updatedAt: string
+	defaultParams: any
+	paramsTypes: any
+}
+
+export type Transition =
+  | "Bounce"
+  | "BowTieHorizontal"
+  | "BowTieVertical"
+  | "ButterflyWaveScrawler"
+  | "CircleCrop"
+  | "ColourDistance"
+  | "CrazyParametricFun"
+  | "CrossZoom"
+  | "Directional"
+  | "DoomScreenTransition"
+  | "Dreamy"
+  | "DreamyZoom"
+  | "GlitchDisplace"
+  | "GlitchMemories"
+  | "GridFlip"
+  | "InvertedPageCurl"
+  | "LinearBlur"
+  | "Mosaic"
+  | "PolkaDotsCurtain"
+  | "Radial"
+  | "SimpleZoom"
+  | "StereoViewer"
+  | "Swirl"
+  | "WaterDrop"
+  | "ZoomInCircles"
+  | "angular"
+  | "burn"
+  | "cannabisleaf"
+  | "circle"
+  | "circleopen"
+  | "colorphase"
+  | "crosshatch"
+  | "crosswarp"
+  | "cube"
+  | "directionalwarp"
+  | "directionalwipe"
+  | "displacement"
+  | "doorway"
+  | "fade"
+  | "fadecolor"
+  | "fadegrayscale"
+  | "flyeye"
+  | "heart"
+  | "hexagonalize"
+  | "kaleidoscope"
+  | "luma"
+  | "luminance_melt"
+  | "morph"
+  | "multiply_blend"
+  | "perlin"
+  | "pinwheel"
+  | "pixelize"
+  | "polar_function"
+  | "randomsquares"
+  | "ripple"
+  | "rotate_scale_fade"
+  | "squareswire"
+  | "squeeze"
+  | "swap"
+  | "undulatingBurnOut"
+  | "wind"
+  | "windowblinds"
+  | "windowslice"
+  | "wipeDown"
+  | "wipeLeft"
+  | "wipeRight"
+  | "wipeUp"

package/s/features/transition/parts/uniforms.ts

@@ -0,0 +1,29 @@
+import {GLTransition} from "./types.js"
+
+export const uniforms = {
+	custom: (transition: GLTransition) => Object.fromEntries(
+		Object.entries(transition.defaultParams).map(([name, value]) => [
+			name,
+			{
+				value,
+				type: getUniformType(transition.paramsTypes[name])
+			}
+		])
+	),
+	basics: {
+		_fromR: {value: 1, type: "f32"},
+		_toR: {value: 1, type: "f32"},
+		ratio: {value: 1, type: "f32"},
+		progress: {value: 0, type: "f32"},
+		customUniform: {value: 0, type: "f32"},
+	}
+}
+
+const getUniformType = (type: string) => {
+	if(type === "f32" || type === "i32") {
+		return type
+	} else if(type === "float") {
+		return "f32"
+	}
+	else return `${type}<f32>`
+}

package/s/features/transition/parts/vertex.ts

@@ -0,0 +1,31 @@
+export const vertex = `
+	in vec2 aPosition;
+	varying vec2 _uv;                          // gl-transition
+	uniform mat3 projectionMatrix;
+	uniform vec4 uInputSize;
+	uniform vec4 uOutputFrame;
+	out vec2 vTextureCoord;
+	uniform vec4 uOutputTexture;
+
+	vec4 filterVertexPosition( void )
+	{
+			vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+
+			position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+			position.y = position.y * (2.0*uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+
+			return vec4(position, 0.0, 1.0);
+	}
+
+	vec2 filterTextureCoord( void )
+	{
+	return aPosition * (uOutputFrame.zw * uInputSize.zw);
+	}
+
+	void main(void)
+	{
+	gl_Position = filterVertexPosition();
+	vTextureCoord = filterTextureCoord();
+	_uv = vec2(0.5, 0.5) * (aPosition +vec2(1.0, 1.0));    // gl-transition
+	}
+`

package/s/features/transition/transition.ts

@@ -0,0 +1,60 @@
+//@ts-ignore
+import transitions from "gl-transitions"
+import {Filter, GlProgram, Sprite, Texture, ImageSource} from "pixi.js"
+
+import {vertex} from "./parts/vertex.js"
+import {uniforms} from "./parts/uniforms.js"
+import {fragment} from "./parts/fragment.js"
+import {GLTransition, TransitionOptions, TransitionRendererOptions} from "./parts/types.js"
+
+export function makeTransition({name, renderer}: TransitionOptions) {
+	const transition = transitions.find((t: GLTransition) => t.name === name) as GLTransition
+	const transitionSprite = new Sprite()
+	const transitionTexture = new Texture()
+	const sourceFrom = new ImageSource({})
+	const sourceTo = new ImageSource({})
+
+	const filter = new Filter({
+		glProgram: new GlProgram({
+			vertex,
+			fragment: fragment(transition.glsl),
+		}),
+		resources: {
+			from: sourceFrom,
+			to: sourceTo,
+			uniforms: {
+				...uniforms.basics,
+				...uniforms.custom(transition)
+			}
+		}
+	})
+
+	transitionSprite.filters = [filter]
+
+	return {
+		render({width, height, from, to, progress}: TransitionRendererOptions) {
+			if(transitionSprite.width !== width || transitionSprite.height !== height) {
+				transitionSprite.setSize({width, height})
+				transitionTexture.source.resize(width, height)
+			}
+
+			sourceFrom.resource = from
+			sourceTo.resource = to
+			sourceFrom.update()
+			sourceTo.update()
+
+			filter.resources.uniforms.uniforms.progress = progress
+
+			renderer.render({
+				container: transitionSprite,
+				target: transitionTexture,
+				clear: false,
+				width,
+				height
+			})
+
+			return transitionTexture
+		}
+	}
+}
+

package/s/index.html.ts

@@ -0,0 +1,58 @@
+
+import {ssg, html} from "@e280/scute"
+
+const title = "omnitool"
+const domain = "omnitool.omniclip.app"
+const favicon = "/assets/favicon.png"
+
+export default ssg.page(import.meta.url, async orb => ({
+	title,
+	// favicon,
+	dark: true,
+	css: "demo/demo.css",
+	js: "demo/demo.bundle.min.js",
+
+	head: html`
+		<link rel="preconnect" href="https://fonts.googleapis.com">
+		<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+		<link href="https://fonts.googleapis.com/css2?family=Share+Tech&display=swap" rel="stylesheet">
+	`,
+
+	socialCard: {
+		title,
+		description: "video processing toolkit",
+		themeColor: "#3cff9c",
+		siteName: domain,
+		image: `https://${domain}${favicon}`,
+	},
+
+	body: html`
+		<section>
+			<h1>Omnitool <small>v${orb.packageVersion()}</small></h1>
+			<button class=fetch>fetch</button>
+			<input type="file" class="file-input">
+			<div class=results></div>
+			<div class=filmstrip-demo>
+				<label for="viewable-range">viewable range:</label>
+				<input type="range" min="0" max="100" step="1" value="10" class="range" id="viewable-range" name="viewable-range">
+				<div class="range-view"></div>
+				<label for="range-size">viewable range size:</label>
+				<input type="range" class="range-size" min="0.1" max="10" step="0.1" value="0.5" id="range-size" name="range-size">
+				<label for="frequency">frequency:</label>
+				<input type="range" class="frequency" min="0.1" max="120" step="0.1" value="10" id="frequency" name="frequency">
+				<div class="frequency-view">10 (fps)</div>
+				<div id=filmstrip></div>
+			</div>
+			<div class=waveform-demo>
+				<label for="width">width:</label>
+				<input class="width" id="width" name="width" type="range" min="100" max="1000000" value="1000" />
+			</div>
+			<div class=player>
+				<input class="seek" type="number" min="0">
+				<button class=play>play</button>
+				<button class=stop>stop</button>
+			</div>
+		</section>
+	`,
+}))
+

package/s/index.ts

@@ -1,41 +1,4 @@
-import {MP4Demuxer} from "./tools/mp4boxjs/demuxer.js"
-
-import {Clip, LoadedVideoClip} from "./types.js"
-import {generateId} from "./tools/generate-id.js"
-import { OmniVideoDecoder } from "./parts/video-decoder.js"
-
-export class OmniTool {
-	#clips: Map<Clip, LoadedVideoClip> = new Map()
-	#getLoadedClip = (clip: Clip) => this.#clips.get(clip)
-	
-	videoDecoder = new OmniVideoDecoder(this.#getLoadedClip)
-
-	// ffmpeg node or web version
-	constructor(ffmpeg) {}
-
-
-	load(file: File): Promise<Clip> {
-		return new Promise((resolve) => {
-			const demuxer = new MP4Demuxer(file, {
-				onChunk: () => {},
-				onConfig: () => {},
-				framesCount: () => {},
-				setStatus: () => {},
-				OnReady: () => {
-					const demuxed = {id: generateId(), demuxer}
-					this.#clips.set(demuxed, demuxed)
-					resolve(demuxed)
-				}
-			})
-		})
-
-	}
-
-	async getFramesCount() {}
-}
-
-const tool = new OmniTool('ffmpeg')
-// const file = tool.load()
-// tool.videoDecoder.decode(file)
 
+export * from "./driver/driver.js"
+export * from "./timeline/index.js"
 

package/s/tests.test.ts

@@ -0,0 +1,8 @@
+
+import {Science} from "@e280/science"
+import driver from "./driver/driver.test.js"
+
+await Science.run({
+	driver,
+})
+

package/s/timeline/index.ts

@@ -0,0 +1,15 @@
+
+export * from "./parts/basics.js"
+export * from "./parts/item.js"
+export * from "./parts/media.js"
+export * from "./parts/resource-pool.js"
+export * from "./parts/resource.js"
+export * from "./parts/filmstrip.js"
+export * from "./parts/compositor/playback.js"
+
+export * from "./sugar/o.js"
+export * from "./sugar/omni.js"
+
+export * from "./utils/checksum.js"
+export * from "./utils/datafile.js"
+

package/s/timeline/parts/basics.ts

@@ -0,0 +1,17 @@
+
+import {Item} from "./item.js"
+
+/** sha256 hash */
+export type Hash = string
+
+/** item identifier */
+export type Id = number
+
+export type TimelineFile = {
+	info: "https://omniclip.app/"
+	format: "timeline"
+	version: number
+	rootId: Id
+	items: Item.Any[]
+}
+