@omnimedia/omnitool 1.0.0 → 1.1.0-3

package/s/timeline/parts/filmstrip.ts

@@ -0,0 +1,159 @@
+import {
+	ALL_FORMATS,
+	CanvasSink,
+	CanvasSinkOptions,
+	Input,
+	InputVideoTrack,
+	WrappedCanvas,
+} from "mediabunny"
+
+import {DecoderSource} from '../../driver/fns/schematic.js'
+import {loadDecoderSource} from '../../driver/utils/load-decoder-source.js'
+
+export class Filmstrip {
+	#sink: CanvasSink
+	#cache: Map<number, WrappedCanvas> = new Map()
+	#activeRange: TimeRange = [0, 0]
+
+	private constructor(
+		private videoTrack: InputVideoTrack,
+		private options: Required<FilmstripOptions>
+	) {
+		this.#sink = new CanvasSink(videoTrack, options.canvasSinkOptions)
+	}
+
+	static async init(source: DecoderSource, options: FilmstripOptions) {
+		const input = new Input({
+			formats: ALL_FORMATS,
+			source: await loadDecoderSource(source)
+		})
+		const videoTrack = await input.getPrimaryVideoTrack()
+		if(videoTrack)
+			return new Filmstrip(
+				videoTrack, {
+					frequency: options.frequency ?? 1,
+					canvasSinkOptions: options.canvasSinkOptions ?? {width: 80, height: 50, fit: "fill"},
+					onChange: options.onChange
+			})
+		else throw new Error("Source has no video track")
+	}
+
+	/**
+ 	* Sets the frequency (granularity) of filmstrip thumbnails.
+ 	* Changing this triggers a filmstrip refresh after any ongoing update finishes.
+ 	* @param value - The new frequency in seconds.
+ 	*/
+	set frequency(value: number) {
+		if(value !== this.options.frequency) {
+			this.options.frequency = value
+			this.#update()
+		}
+	}
+
+	get frequency() {
+		return this.options.frequency
+	}
+
+	#computeActiveRange([start, end]: TimeRange): TimeRange {
+		const tileSize = end - start
+		return [start - tileSize, end + tileSize]
+	}
+
+	async #generateTiles() {
+		const [rangeStart, rangeEnd] = this.#activeRange
+		const neededTimestamps = new Set<number>()
+
+		// duration should be computed but with trim etc also
+		const duration = await this.videoTrack.computeDuration()
+		for (
+			let timestamp = rangeStart;
+			timestamp <= rangeEnd;
+			timestamp += this.options.frequency
+		) {
+			// Clamp to valid time range
+			if (timestamp >= 0 && timestamp <= duration)
+				neededTimestamps.add(+timestamp.toFixed(3))
+		}
+
+		const missingTimestamps = [...neededTimestamps]
+			.filter(t => !this.#cache.has(t))
+
+		let i = 0
+		for await (const canvas of this.#sink.canvasesAtTimestamps(missingTimestamps)) {
+			if(canvas) {
+				const requestedTime = missingTimestamps[i++]
+				this.#cache.set(requestedTime, canvas)
+			}
+		}
+
+		// Dispose canvases outside the new range
+		for (const key of this.#cache.keys()) {
+			if (!neededTimestamps.has(key)) {
+				this.#cache.delete(key)
+			}
+		}
+
+		const tiles = [...this.#cache.entries()]
+			.map(([time, canvas]) => ({time, canvas}))
+		this.options.onChange(tiles)
+	}
+
+	/**
+ 	* Updates the visible time range for the filmstrip.
+ 	*
+ 	* Triggers a thumbnails update, with extended margins to preload
+ 	* thumbnails slightly outside the visible range.
+ 	* @param visibleRange - The current timeline viewport as a [start, end] tuple in seconds.
+ 	*/
+	set range(visibleRange: TimeRange) {
+		const newRange = this.#computeActiveRange(visibleRange)
+		// Avoid redundant updates
+		if (
+			this.#activeRange[0] === newRange[0] &&
+			this.#activeRange[1] === newRange[1]
+		)
+			return
+
+		this.#activeRange = newRange
+		this.#update()
+	}
+
+	#updating: Promise<void> | null = null
+	#shouldRunAgain = false
+
+	async #update() {
+		// Perform update immediately. If multiple updates are requested while updating,
+		// only the latest one will run after the current finishes (skips intermediate ones).
+		if(this.#updating) {
+			this.#shouldRunAgain = true
+			return
+		}
+
+		this.#updating = this.#generateTiles()
+		await this.#updating
+		this.#updating = null
+
+		if(this.#shouldRunAgain) {
+			this.#shouldRunAgain = false
+			await this.#update()
+		}
+	}
+	/**
+ 	* Returns the cached thumbnail (if any) for a given timestamp.
+ 	* @param time - The timestamp to retrieve the canvas for.
+ 	*/
+	getThumbnail(time: number) {
+		return this.#cache.get(time)
+	}
+}
+
+type TimeRange = [number, number]
+
+interface FilmstripOptions {
+	frequency?: number
+	canvasSinkOptions?: CanvasSinkOptions
+	onChange: (tiles: {
+		time: number
+		canvas: WrappedCanvas
+	}[]) => void
+}

package/s/timeline/parts/item.ts

@@ -0,0 +1,58 @@
+
+import {Id, Hash} from "./basics.js"
+
+export enum Kind {
+	Sequence,
+	Stack,
+	Clip,
+	Text,
+	Transition,
+}
+
+export enum Effect {
+	Crossfade,
+}
+
+export namespace Item {
+	export type Sequence = {
+		id: Id
+		kind: Kind.Sequence
+		children: Id[]
+	}
+
+	export type Stack = {
+		id: Id
+		kind: Kind.Stack
+		children: Id[]
+	}
+
+	export type Clip = {
+		id: Id
+		kind: Kind.Clip
+		mediaHash: Hash
+		start: number
+		duration: number
+	}
+
+	export type Text = {
+		id: Id
+		kind: Kind.Text
+		content: string
+	}
+
+	export type Transition = {
+		id: Id
+		kind: Kind.Transition
+		effect: Effect.Crossfade
+		duration: number
+	}
+
+	export type Any = (
+		| Sequence
+		| Stack
+		| Clip
+		| Text
+		| Transition
+	)
+}
+

package/s/timeline/parts/media.ts

@@ -0,0 +1,14 @@
+
+import {Datafile} from "../utils/datafile.js"
+
+export class Media {
+	duration = 0
+	constructor(public datafile: Datafile) {}
+
+	static async analyze(datafile: Datafile) {
+		const media = new this(datafile)
+		media.duration = 10
+		return media
+	}
+}
+

package/s/timeline/parts/resource-pool.ts

@@ -0,0 +1,27 @@
+
+import {MapG} from "@e280/stz"
+import {Hash} from "./basics.js"
+import {Media} from "./media.js"
+import {Resource} from "./resource.js"
+import {Datafile} from "../utils/datafile.js"
+
+export class ResourcePool {
+	#map = new MapG<Hash, Resource.Any>
+
+	/** store a media file (avoids duplicates via hash) */
+	async store(datafile: Datafile) {
+		const media = await Media.analyze(datafile)
+		const {hash} = media.datafile.checksum
+		const {filename, bytes} = media.datafile
+
+		if (this.#map.has(hash)) {
+			const alreadyExists = this.#map.require(hash)
+			alreadyExists.filename = filename
+		}
+		else
+			this.#map.set(hash, {kind: "media", filename, bytes})
+
+		return media
+	}
+}
+

package/s/timeline/parts/resource.ts

@@ -0,0 +1,11 @@
+
+export namespace Resource {
+	export type Media = {
+		kind: "media"
+		filename: string
+		bytes: Uint8Array
+	}
+
+	export type Any = Media
+}
+

package/s/timeline/parts/waveform.ts

@@ -0,0 +1,62 @@
+import WaveSurfer from "wavesurfer.js"
+
+import {context} from "../../context.js"
+import {DecoderSource} from "../../driver/fns/schematic.js"
+
+export class Waveform {
+	wavesurfer: WaveSurfer
+
+	constructor(peaks: number[], container: HTMLElement, duration: number) {
+		this.wavesurfer = WaveSurfer.create({
+  		container,
+  		waveColor: 'rgb(200, 0, 200)',
+  		progressColor: 'rgb(100, 0, 100)',
+  		barWidth: 10,
+  		barRadius: 10,
+  		barGap: 2,
+  		peaks: [peaks],
+  		duration
+		})
+	}
+
+	static async init(source: DecoderSource, container: HTMLElement) {
+		const driver = await context.driver
+		const reader = driver.decode({ source }).audio.getReader()
+
+		const peaks: number[] = []
+		let buffer: number[] = []
+		const samplesPerPeak = 1024
+		const duration = await driver.getAudioDuration(source)
+
+		while (true) {
+			const {done, value: audioData} = await reader.read()
+			if (done) break
+
+			const frames = audioData.numberOfFrames
+			const plane = new Float32Array(frames)
+			audioData.copyTo(plane, {planeIndex: 0}) // Use left channel only
+
+			for (let i = 0; i < plane.length; i++) {
+				buffer.push(plane[i])
+				if (buffer.length >= samplesPerPeak) {
+					const chunk = buffer.splice(0, samplesPerPeak)
+					const min = Math.min(...chunk)
+					const max = Math.max(...chunk)
+					peaks.push(min, max)
+				}
+			}
+
+			audioData.close()
+		}
+
+		return new Waveform(peaks, container, duration ?? 0)
+	}
+
+	// set zoom(value: number) {
+	// 	this.wavesurfer.zoom(value)
+	// }
+
+	set width(value: number) {
+		this.wavesurfer.setOptions({width: value})
+	}
+}

package/s/timeline/sugar/o.ts

@@ -0,0 +1,60 @@
+
+import {MapG} from "@e280/stz"
+import {Id} from "../parts/basics.js"
+import {Media} from "../parts/media.js"
+import {Effect, Item, Kind} from "../parts/item.js"
+
+export class O {
+	#nextId = 0
+	#items = new MapG<Id, Item.Any>()
+
+	#getId() {
+		return this.#nextId++
+	}
+
+	register(item: Item.Any) {
+		if (!this.#items.has(item.id))
+			this.#items.set(item.id, item)
+		return item.id
+	}
+
+	get items() {
+		return [...this.#items.values()]
+	}
+
+	sequence = (...items: Item.Any[]): Item.Sequence => ({
+		id: this.#getId(),
+		kind: Kind.Sequence,
+		children: items.map(item => this.register(item)),
+	})
+
+	stack = (...items: Item.Any[]): Item.Stack => ({
+		id: this.#getId(),
+		kind: Kind.Stack,
+		children: items.map(item => this.register(item)),
+	})
+
+	clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
+		id: this.#getId(),
+		kind: Kind.Clip,
+		mediaHash: media.datafile.checksum.hash,
+		start: start ?? 0,
+		duration: duration ?? media.duration,
+	})
+
+	text = (content: string): Item.Text => ({
+		id: this.#getId(),
+		kind: Kind.Text,
+		content,
+	})
+
+	transition = {
+		crossfade: (duration: number): Item.Transition => ({
+			id: this.#getId(),
+			kind: Kind.Transition,
+			effect: Effect.Crossfade,
+			duration,
+		}),
+	}
+}
+

package/s/timeline/sugar/omni-test.ts

@@ -0,0 +1,38 @@
+
+import {Omni} from "./omni.js"
+import {dummyData} from "../utils/dummy-data.js"
+
+//
+// create an omni context
+//
+
+const omni = new Omni()
+
+//
+// load in some media resources
+//
+
+const {mediaA, mediaB} = await omni.load({
+	mediaA: dummyData(),
+	mediaB: dummyData(),
+})
+
+//
+// create a timeline
+//
+
+const timeline = omni.timeline(o => o.sequence(
+	o.clip(mediaA),
+	o.transition.crossfade(600),
+	o.stack(
+		o.clip(mediaB),
+		o.text("hello world"),
+	),
+))
+
+//
+// log the timeline
+//
+
+console.log(JSON.stringify(timeline, undefined, "  "))
+

package/s/timeline/sugar/omni.ts

@@ -0,0 +1,30 @@
+
+import {O} from "./o.js"
+import {Item} from "../parts/item.js"
+import {Media} from "../parts/media.js"
+import {TimelineFile} from "../parts/basics.js"
+import {Datafile} from "../utils/datafile.js"
+import {ResourcePool} from "../parts/resource-pool.js"
+
+export class Omni {
+	resources = new ResourcePool()
+
+	load = async<S extends Record<string, Promise<Datafile>>>(spec: S) => {
+		return Object.fromEntries(await Promise.all(Object.entries(spec).map(
+			async([key, value]) => [key, await this.resources.store(await value)]
+		))) as {[K in keyof S]: Media}
+	}
+
+	timeline = (fn: (o: O) => Item.Sequence): TimelineFile => {
+		const o = new O()
+		const sequence = fn(o)
+		return {
+			format: "timeline",
+			info: "https://omniclip.app/",
+			version: 0,
+			root: o.register(sequence),
+			items: o.items,
+		}
+	}
+}
+

package/s/timeline/utils/checksum.ts

@@ -0,0 +1,19 @@
+
+import {Hex, Thumbprint} from "@e280/stz"
+
+export class Checksum {
+	constructor(
+		public data: Uint8Array,
+		public bytes: Uint8Array,
+		public hash: string,
+		public nickname: string,
+	) {}
+
+	static async make(data: Uint8Array) {
+		const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data))
+		const hash = Hex.fromBytes(bytes)
+		const nickname = Thumbprint.sigil.fromBytes(bytes)
+		return new this(data, bytes, hash, nickname)
+	}
+}
+

package/s/timeline/utils/datafile.ts

@@ -0,0 +1,21 @@
+
+import {Checksum} from "./checksum.js"
+
+export class Datafile {
+	constructor(
+		public bytes: Uint8Array,
+		public filename: string,
+		public checksum: Checksum,
+	) {}
+
+	static async make(bytes: Uint8Array, name?: string) {
+		const checksum = await Checksum.make(bytes)
+		const filename = name ?? checksum.nickname
+		return new this(bytes, filename, checksum)
+	}
+
+	static async load(_path: string): Promise<Datafile> {
+		throw new Error("TODO implement")
+	}
+}
+

package/s/timeline/utils/dummy-data.ts

@@ -0,0 +1,7 @@
+
+import {Datafile} from "./datafile.js"
+
+export const dummyData = () => Datafile.make(
+	new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])
+)
+

package/x/context.d.ts

@@ -0,0 +1,4 @@
+import { Driver } from "./driver/driver.js";
+export declare const context: {
+    driver: Promise<Driver>;
+};

package/x/context.js

@@ -0,0 +1,6 @@
+import { Driver } from "./driver/driver.js";
+const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url);
+export const context = {
+    driver: Driver.setup({ workerUrl })
+};
+//# sourceMappingURL=context.js.map

package/x/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAA;AAEzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,mCAAmC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAC,CAAC;CACjC,CAAA"}

package/x/demo/demo.bundle.d.ts

@@ -0,0 +1 @@
+export {};

package/x/demo/demo.bundle.js

@@ -0,0 +1,51 @@
+import { context } from "../context.js";
+import { waveformTest } from "./routines/waveform-test.js";
+import { filmstripTest } from "./routines/filmstrip-test.js";
+import { setupTranscodeTest } from "./routines/transcode-test.js";
+const driver = await context.driver;
+const results = document.querySelector(".results");
+const fetchButton = document.querySelector(".fetch");
+const importButton = document.querySelector(".import");
+fetchButton?.addEventListener("click", startDemoFetch);
+importButton?.addEventListener("click", startDemoImport);
+waveformTest();
+// hello world test
+{
+    await driver.thread.work.hello();
+    if (driver.machina.count === 1)
+        console.log("✅ driver works");
+    else
+        console.error("❌ FAIL driver call didn't work");
+}
+// transcoding tests
+async function startDemoImport() {
+    const [fileHandle] = await window.showOpenFilePicker();
+    const transcode = setupTranscodeTest(driver, fileHandle);
+    await filmstripTest(fileHandle);
+    run(transcode, fileHandle.name);
+}
+async function startDemoFetch() {
+    // which videos to run tests on
+    const videos = [
+        "/assets/temp/gl.mp4",
+    ];
+    // running each test in sequence
+    for (const url of videos) {
+        const transcode = setupTranscodeTest(driver, "/assets/temp/gl.mp4");
+        run(transcode, url);
+    }
+}
+async function run(transcode, label) {
+    // create result div
+    const div = document.createElement("div");
+    results.append(div);
+    // add video label
+    const p = document.createElement("p");
+    p.textContent = label;
+    div.append(p);
+    // add the canvas to dom
+    div.append(transcode.canvas);
+    // run the test
+    await transcode.run();
+}
+//# sourceMappingURL=demo.bundle.js.map

package/x/demo/demo.bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAC1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAE/D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,YAAY,EAAE,CAAA;AAEd,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACxD,MAAM,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/B,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}