@jadujoel/web-audio-clip-node 0.1.6 → 0.1.8

package/examples/streaming/src/useStreamingClipNode.ts

@@ -0,0 +1,540 @@
+import {
+	useCallback,
+	useRef,
+	useState,
+	type RefObject,
+} from "./react-runtime";
+import {
+	ClipNode,
+	detectStreamFormat,
+	float32ArrayFromAudioBuffer,
+	getProcessorBlobUrl,
+	getStreamingWorkerUrl,
+	linFromDb,
+	usesBufferedContainerDecode,
+} from "./clip-node-lib";
+import type { ControlKey } from "./clip-node-lib";
+import type { ClipNodeState, FrameData } from "./clip-node-lib";
+import type { StreamFormat } from "./clip-node-lib";
+import {
+	clampSeekTargetSamples,
+	secondsFromSamples,
+} from "./streamTimeline";
+
+function getWorkerUrl(format: StreamFormat): string {
+	return getStreamingWorkerUrl(format);
+}
+
+function applyValueToClip(node: ClipNode, key: ControlKey, value: number) {
+	switch (key) {
+		case "playhead":
+			node.playhead = value;
+			break;
+		case "offset":
+			node.offset = value;
+			break;
+		case "duration":
+			node.duration = value;
+			break;
+		case "loopStart":
+			node.loopStart = value;
+			break;
+		case "loopEnd":
+			node.loopEnd = value;
+			break;
+		case "loopCrossfade":
+			node.loopCrossfade = value;
+			break;
+		case "fadeIn":
+			node.fadeIn = value;
+			break;
+		case "fadeOut":
+			node.fadeOut = value;
+			break;
+		case "playbackRate":
+			node.playbackRate.value = value;
+			break;
+		case "detune":
+			node.detune.value = value;
+			break;
+		case "gain":
+			node.gain.value = linFromDb(value);
+			break;
+		case "pan":
+			node.pan.value = value;
+			break;
+		case "lowpass":
+			node.lowpass.value = value;
+			break;
+		case "highpass":
+			node.highpass.value = value;
+			break;
+		case "startDelay":
+		case "stopDelay":
+			break;
+	}
+}
+
+function applyToggleToClip(node: ClipNode, key: ControlKey, on: boolean) {
+	switch (key) {
+		case "fadeIn":
+			node.toggleFadeIn(on);
+			break;
+		case "fadeOut":
+			node.toggleFadeOut(on);
+			break;
+		case "loopCrossfade":
+			node.toggleLoopCrossfade(on);
+			break;
+		case "loopStart":
+			node.toggleLoopStart(on);
+			break;
+		case "loopEnd":
+			node.toggleLoopEnd(on);
+			break;
+		case "playbackRate":
+			node.togglePlaybackRate(on);
+			break;
+		case "detune":
+			node.toggleDetune(on);
+			break;
+		case "gain":
+			node.toggleGain(on);
+			break;
+		case "pan":
+			node.togglePan(on);
+			break;
+		case "lowpass":
+			node.toggleLowpass(on);
+			break;
+		case "highpass":
+			node.toggleHighpass(on);
+			break;
+		case "offset":
+			node.offset = on ? node.offset : 0;
+			break;
+		case "duration":
+			node.duration = on ? node.duration : -1;
+			break;
+		case "startDelay":
+		case "stopDelay":
+			break;
+	}
+}
+
+interface UseStreamingClipNodeParams {
+	values: Record<ControlKey, number>;
+	enabled: Record<ControlKey, boolean>;
+	loop: boolean;
+	setValue: (key: ControlKey, val: number) => void;
+}
+
+export function useStreamingClipNode({
+	values,
+	enabled,
+	loop,
+	setValue,
+}: UseStreamingClipNodeParams) {
+	const [nodeState, setNodeState] = useState<ClipNodeState>("initial");
+	const [statusMessage, setStatusMessage] = useState<string | null>("Idle");
+	const [progress, setProgress] = useState(0);
+	const [audioDuration, setAudioDuration] = useState<number | null>(null);
+	const [seekableDuration, setSeekableDuration] = useState<number | null>(null);
+	const [seekableSamples, setSeekableSamples] = useState<number | null>(null);
+
+	const configureClip = useCallback(
+		(ctx: AudioContext, clip: ClipNode) => {
+			clip.loop = loop;
+			clip.connect(ctx.destination);
+
+			for (const key of Object.keys(values) as ControlKey[]) {
+				applyValueToClip(clip, key, values[key]);
+			}
+			for (const key of Object.keys(enabled) as ControlKey[]) {
+				if (!enabled[key]) {
+					applyToggleToClip(clip, key, false);
+				}
+			}
+
+			clip.onstatechange = (s) => setNodeState(s);
+			clip.onlooped = () => {
+				timesLoopedRef.current = clip.timesLooped.toString();
+			};
+			clip.onframe = (data) => {
+				frameRef.current = data;
+			};
+
+			setInfoLatency(
+				ctx.outputLatency != null
+					? `base: ${Math.round(ctx.baseLatency * ctx.sampleRate)} | output: ${Math.round(ctx.outputLatency * ctx.sampleRate)}`
+					: "unknown",
+			);
+
+			clipRef.current = clip;
+		},
+		[enabled, loop, values],
+	);
+	const [infoLatency, setInfoLatency] = useState("unknown");
+	const statusRef = useRef<string | null>("Idle");
+	const finalDurationRef = useRef<number | null>(null);
+	const timelineSampleRateRef = useRef<number | null>(null);
+
+	const ctxRef = useRef<AudioContext | null>(null);
+	const clipRef = useRef<ClipNode | null>(null);
+	const workerRef = useRef<Worker | null>(null);
+	const frameRef = useRef<FrameData | null>(null);
+	const timesLoopedRef = useRef("0");
+
+	const ensureContext = useCallback(async () => {
+		if (ctxRef.current) return ctxRef.current;
+		const ctx = new AudioContext({ sampleRate: 48000 });
+		await ctx.audioWorklet.addModule(getProcessorBlobUrl());
+		ctxRef.current = ctx;
+		return ctx;
+	}, []);
+
+	const setStatus = useCallback((next: string) => {
+		if (statusRef.current === next) return;
+		statusRef.current = next;
+		setStatusMessage(next);
+	}, []);
+
+	const stream = useCallback(
+		async (url: string, throttle: number, format?: StreamFormat) => {
+			if (!url.trim()) {
+				setStatus("Enter a URL first.");
+				return;
+			}
+
+			// Tear down previous run
+			if (workerRef.current) {
+				workerRef.current.postMessage({ type: "abort" });
+				workerRef.current.terminate();
+				workerRef.current = null;
+			}
+			if (clipRef.current) {
+				clipRef.current.stop();
+				clipRef.current.disconnect();
+				clipRef.current = null;
+			}
+
+			try {
+				const ctx = await ensureContext();
+				await ctx.resume();
+				const selectedFormat = format ?? detectStreamFormat(url);
+				finalDurationRef.current = null;
+				timelineSampleRateRef.current = ctx.sampleRate;
+				setAudioDuration(null);
+				setSeekableDuration(null);
+				setSeekableSamples(null);
+				setProgress(0);
+
+				if (usesBufferedContainerDecode(selectedFormat)) {
+					const absoluteUrl = new URL(url, location.href).href;
+					const response = await fetch(absoluteUrl);
+					if (!response.ok) {
+						throw new Error(
+							`Fetch failed: ${response.status} ${response.statusText}`,
+						);
+					}
+					if (!response.body) {
+						throw new Error("Response has no body");
+					}
+
+					const contentLength = response.headers.get("content-length");
+					const totalBytes = contentLength
+						? Number.parseInt(contentLength, 10)
+						: null;
+					const reader = response.body.getReader();
+					const chunks: Uint8Array[] = [];
+					let bytesReceived = 0;
+
+					while (true) {
+						const { done, value } = await reader.read();
+						if (done) break;
+						chunks.push(value);
+						bytesReceived += value.length;
+						if (totalBytes != null) {
+							setProgress(bytesReceived / totalBytes);
+							setStatus(
+								`Downloading container… ${((bytesReceived / 1024) | 0)} / ${((totalBytes / 1024) | 0)} KB`,
+							);
+						} else {
+							setStatus(`Downloading container… ${((bytesReceived / 1024) | 0)} KB`);
+						}
+					}
+
+					const combined = new Uint8Array(bytesReceived);
+					let offset = 0;
+					for (const chunk of chunks) {
+						combined.set(chunk, offset);
+						offset += chunk.length;
+					}
+
+					setStatus("Decoding container audio…");
+					const decoded = await ctx.decodeAudioData(combined.buffer);
+					const clip = new ClipNode(ctx, {
+						processorOptions: {
+							buffer: float32ArrayFromAudioBuffer(decoded),
+						},
+					});
+					configureClip(ctx, clip);
+					finalDurationRef.current = decoded.duration;
+					setAudioDuration(decoded.duration);
+					setSeekableDuration(decoded.duration);
+					setSeekableSamples(decoded.length);
+					setProgress(1);
+					clip.start();
+					setStatus(
+						`Buffered & playing… ${decoded.sampleRate} Hz, ${decoded.numberOfChannels} ch`,
+					);
+					return;
+				}
+
+				// Create ClipNode (no buffer — streaming mode)
+				const clip = new ClipNode(ctx);
+				configureClip(ctx, clip);
+
+				// Create MessageChannel: port1 → Worker, port2 → Processor
+				const channel = new MessageChannel();
+				clip.transferPort(channel.port2);
+
+				// Create and start decode worker
+				const worker = new Worker(getWorkerUrl(selectedFormat));
+				workerRef.current = worker;
+				worker.onerror = (event: ErrorEvent) => {
+					setStatus(
+						`Error: worker startup failed${event.message ? ` (${event.message})` : ""}`,
+					);
+				};
+				worker.onmessageerror = () => {
+					setStatus("Error: worker message channel failed.");
+				};
+
+				worker.onmessage = (ev: MessageEvent) => {
+				const { type } = ev.data;
+				switch (type) {
+					case "streamMeta": {
+						const sampleRate = ev.data.sampleRate as number;
+						const estimatedTotalSamples =
+							ev.data.estimatedTotalSamples as number | null;
+						timelineSampleRateRef.current =
+							sampleRate > 0 ? sampleRate : timelineSampleRateRef.current;
+						if (
+							estimatedTotalSamples != null &&
+							timelineSampleRateRef.current != null
+						) {
+							const estimatedDuration = secondsFromSamples(
+								estimatedTotalSamples,
+								timelineSampleRateRef.current,
+							);
+							if (
+								estimatedDuration != null &&
+								finalDurationRef.current == null
+							) {
+								setAudioDuration(estimatedDuration);
+							}
+						}
+						break;
+					}
+					case "progress": {
+						const { bytesReceived, totalBytes } = ev.data;
+						if (totalBytes) {
+							setProgress(bytesReceived / totalBytes);
+							setStatus(
+								`Downloading… ${((bytesReceived / 1024) | 0)} / ${((totalBytes / 1024) | 0)} KB`,
+							);
+						} else {
+							setStatus(
+								`Downloading… ${((bytesReceived / 1024) | 0)} KB`,
+							);
+						}
+						break;
+					}
+					case "decoded": {
+						const { samplesDecoded } = ev.data;
+						setSeekableSamples(samplesDecoded);
+						if (timelineSampleRateRef.current != null) {
+							const nextSeekable = secondsFromSamples(
+								samplesDecoded,
+								timelineSampleRateRef.current,
+							);
+							if (nextSeekable != null) {
+								setSeekableDuration(nextSeekable);
+							}
+						}
+						if (
+							samplesDecoded > 0 &&
+							clipRef.current?.state === "initial"
+						) {
+							clipRef.current.start();
+							setStatus("Streaming & playing…");
+						}
+						break;
+					}
+					case "info": {
+						setStatus(
+							`Decoding: ${ev.data.sampleRate} Hz, ${ev.data.channels} ch`,
+						);
+						break;
+					}
+					case "done": {
+						const samples = ev.data.samplesDecoded as number;
+						setStatus(
+							`Done — ${samples} samples decoded.`,
+						);
+						setProgress(1);
+						if (ctx.sampleRate > 0) {
+							const duration = samples / ctx.sampleRate;
+							finalDurationRef.current = duration;
+							setAudioDuration(duration);
+							setSeekableDuration(duration);
+							setSeekableSamples(samples);
+						}
+						break;
+					}
+					case "error": {
+						setStatus(`Error: ${ev.data.message}`);
+						break;
+					}
+					case "aborted": {
+						setStatus("Aborted.");
+						break;
+					}
+				}
+			};
+
+				const absoluteUrl = new URL(url, location.href).href;
+				worker.postMessage(
+					{
+						type: "init",
+						port: channel.port1,
+						url: absoluteUrl,
+						throttle,
+						targetSampleRate: ctx.sampleRate,
+					},
+					[channel.port1],
+				);
+
+				setStatus(
+					throttle > 0
+						? `Starting stream… (${(throttle / 1024).toFixed(0)} KB/s)`
+						: "Starting stream…",
+				);
+				setNodeState("initial");
+			} catch (error) {
+				if (workerRef.current) {
+					workerRef.current.terminate();
+					workerRef.current = null;
+				}
+				if (clipRef.current) {
+					clipRef.current.disconnect();
+					clipRef.current = null;
+				}
+				setStatus(
+					`Error: ${error instanceof Error ? error.message : String(error)}`,
+				);
+			}
+		},
+		[ensureContext, loop, values, enabled, setStatus],
+	);
+
+	const pause = useCallback(() => {
+		const clip = clipRef.current;
+		if (!clip) return;
+		if (clip.state === "resumed" || clip.state === "started") {
+			clip.pause();
+			setStatus("Paused.");
+		} else if (clip.state === "paused") {
+			clip.start();
+			setStatus("Resumed.");
+		}
+	}, [setStatus]);
+
+	const stop = useCallback(() => {
+		const clip = clipRef.current;
+		if (!clip) return;
+		clip.stop();
+		if (workerRef.current) {
+			workerRef.current.postMessage({ type: "abort" });
+			workerRef.current.terminate();
+			workerRef.current = null;
+		}
+		setProgress(0);
+		setStatus("Stopped.");
+		setSeekableDuration(null);
+		setSeekableSamples(null);
+		if (finalDurationRef.current != null) {
+			setAudioDuration(finalDurationRef.current);
+		}
+	}, [setStatus]);
+
+	const seekPlayhead = useCallback(
+		(targetSample: number) => {
+			const { value: clampedValue, clamped } = clampSeekTargetSamples(
+				targetSample,
+				seekableSamples,
+			);
+			setValue("playhead", clampedValue);
+			const node = clipRef.current;
+			if (node) {
+				applyValueToClip(node, "playhead", clampedValue);
+			}
+			if (clamped) {
+				setStatus("Seek limited to decoded region while streaming.");
+			}
+		},
+		[seekableSamples, setStatus, setValue],
+	);
+
+	const applyValue = useCallback(
+		(key: ControlKey, val: number) => {
+			setValue(key, val);
+			const node = clipRef.current;
+			if (node) applyValueToClip(node, key, val);
+		},
+		[setValue],
+	);
+
+	const applyToggle = useCallback((key: ControlKey, on: boolean) => {
+		const node = clipRef.current;
+		if (node) applyToggleToClip(node, key, on);
+	}, []);
+
+	const applyValues = useCallback(
+		(valuesToApply: Partial<Record<ControlKey, number>>) => {
+			const node = clipRef.current;
+			if (!node) return;
+			for (const [key, value] of Object.entries(valuesToApply) as Array<
+				[ControlKey, number]
+			>) {
+				applyValueToClip(node, key, value);
+			}
+		},
+		[],
+	);
+
+	const setLoopOnNode = useCallback((checked: boolean) => {
+		const node = clipRef.current;
+		if (node) node.loop = checked;
+	}, []);
+
+	return {
+		nodeState,
+		statusMessage,
+		progress,
+		audioDuration,
+		seekableDuration,
+		frameRef: frameRef as RefObject<FrameData | null>,
+		timesLoopedRef: timesLoopedRef as RefObject<string>,
+		infoLatency,
+		stream,
+		pause,
+		stop,
+		seekPlayhead,
+		applyValue,
+		applyValues,
+		applyToggle,
+		setLoopOnNode,
+	};
+}

package/examples/streaming/src/webmOpusParser.test.ts

@@ -0,0 +1,153 @@
+import { describe, expect, it } from "bun:test";
+import {
+	appendWebmOpusBytes,
+	createWebmOpusParserState,
+	processWebmOpusElements,
+	type WebmElementDetail,
+} from "../webm-opus-parser";
+
+function detail<T extends object>(base: T): T & WebmElementDetail {
+	return {
+		data: new Uint8Array(0),
+		...base,
+		name: (base as { name?: string }).name ?? "",
+	} as T & WebmElementDetail;
+}
+
+function buildOpusHead(channels = 2, preSkip = 312): Uint8Array {
+	const head = new Uint8Array(19);
+	head.set(new TextEncoder().encode("OpusHead"), 0);
+	head[8] = 1;
+	head[9] = channels;
+	head[10] = preSkip & 0xff;
+	head[11] = (preSkip >>> 8) & 0xff;
+	return head;
+}
+
+function buildSimpleBlock(trackNumber: number, timecode: number, packet: Uint8Array): Uint8Array {
+	const out = new Uint8Array(4 + packet.length);
+	out[0] = 0x80 | trackNumber;
+	out[1] = (timecode >>> 8) & 0xff;
+	out[2] = timecode & 0xff;
+	out[3] = 0x80;
+	out.set(packet, 4);
+	return out;
+}
+
+function encodeVint(value: number): number[] {
+	for (let length = 1; length <= 4; length++) {
+		const maxValue = 2 ** (7 * length) - 2;
+		if (value <= maxValue) {
+			const out = new Array<number>(length).fill(0);
+			let remaining = value;
+			for (let index = length - 1; index >= 0; index--) {
+				out[index] = remaining & 0xff;
+				remaining >>>= 8;
+			}
+			out[0] |= 1 << (8 - length);
+			return out;
+		}
+	}
+	throw new Error("value too large for test vint");
+}
+
+function encodeUnsigned(value: number): Uint8Array {
+	if (value === 0) return new Uint8Array([0]);
+	const bytes: number[] = [];
+	let remaining = value;
+	while (remaining > 0) {
+		bytes.unshift(remaining & 0xff);
+		remaining >>>= 8;
+	}
+	return new Uint8Array(bytes);
+}
+
+function ebml(id: number[], data: Uint8Array): Uint8Array {
+	return new Uint8Array([...id, ...encodeVint(data.length), ...data]);
+}
+
+function concatBytes(...parts: Uint8Array[]): Uint8Array {
+	const totalLength = parts.reduce((sum, part) => sum + part.length, 0);
+	const out = new Uint8Array(totalLength);
+	let offset = 0;
+	for (const part of parts) {
+		out.set(part, offset);
+		offset += part.length;
+	}
+	return out;
+}
+
+function buildMinimalWebmStream(packet: Uint8Array): Uint8Array {
+	const opusHead = buildOpusHead();
+	const trackEntry = ebml(
+		[0xae],
+		concatBytes(
+			ebml([0xd7], new Uint8Array([1])),
+			ebml([0x83], new Uint8Array([2])),
+			ebml([0x86], new TextEncoder().encode("A_OPUS")),
+			ebml([0x63, 0xa2], opusHead),
+			ebml([0x56, 0xaa], encodeUnsigned(6_500_000)),
+			ebml([0xe1], ebml([0x9f], new Uint8Array([2]))),
+		),
+	);
+	const tracks = ebml([0x16, 0x54, 0xae, 0x6b], trackEntry);
+	const info = ebml(
+		[0x15, 0x49, 0xa9, 0x66],
+		ebml([0x2a, 0xd7, 0xb1], encodeUnsigned(1_000_000)),
+	);
+	const cluster = ebml(
+		[0x1f, 0x43, 0xb6, 0x75],
+		concatBytes(
+			ebml([0xe7], encodeUnsigned(100)),
+			ebml([0xa3], buildSimpleBlock(1, 20, packet)),
+		),
+	);
+	const segment = ebml([0x18, 0x53, 0x80, 0x67], concatBytes(info, tracks, cluster));
+	return concatBytes(segment);
+}
+
+describe("webm opus parser", () => {
+	it("extracts track metadata and simple block packets", () => {
+		const opusHead = buildOpusHead();
+		const packet = new Uint8Array([0xaa, 0xbb, 0xcc]);
+		const elements = [
+			detail({ name: "TrackEntry", type: "m", isEnd: false }),
+			detail({ name: "TrackNumber", type: "u", value: 1, data: Buffer.alloc(0) }),
+			detail({ name: "TrackType", type: "u", value: 2, data: Buffer.alloc(0) }),
+			detail({ name: "CodecID", type: "s", value: "A_OPUS", data: Buffer.alloc(0) }),
+			detail({ name: "CodecPrivate", type: "b", value: Buffer.from(opusHead), data: Buffer.from(opusHead) }),
+			detail({ name: "CodecDelay", type: "u", value: 6_500_000, data: Buffer.alloc(0) }),
+			detail({ name: "Audio", type: "m", isEnd: false }),
+			detail({ name: "Channels", type: "u", value: 2, data: Buffer.alloc(0) }),
+			detail({ name: "Audio", type: "m", isEnd: true }),
+			detail({ name: "TrackEntry", type: "m", isEnd: true }),
+			detail({ name: "TimestampScale", type: "u", value: 1_000_000, data: Buffer.alloc(0) }),
+			detail({ name: "Timestamp", type: "u", value: 100, data: Buffer.alloc(0) }),
+			detail({ name: "SimpleBlock", type: "b", value: Buffer.from(packet), data: Buffer.from(buildSimpleBlock(1, 20, packet)) }),
+		] as WebmElementDetail[];
+		const state = createWebmOpusParserState();
+		const parsed = processWebmOpusElements(elements, state);
+		expect(parsed.head?.channels).toBe(2);
+		expect(parsed.head?.preSkip).toBe(312);
+		expect(parsed.packets).toHaveLength(1);
+		expect(parsed.packets[0]?.packet).toEqual(packet);
+		expect(parsed.packets[0]?.timestampUs).toBe(113_500);
+	});
+
+	it("parses incremental webm bytes without ts-ebml", () => {
+		const packet = new Uint8Array([0xaa, 0xbb, 0xcc]);
+		const bytes = buildMinimalWebmStream(packet);
+		const state = createWebmOpusParserState();
+
+		const first = appendWebmOpusBytes(bytes.subarray(0, 17), state);
+		expect(first.head).toBeNull();
+		expect(first.packets).toHaveLength(0);
+
+		const second = appendWebmOpusBytes(bytes.subarray(17), state);
+		expect(second.head?.channels).toBe(2);
+		expect(second.head?.preSkip).toBe(312);
+		expect(second.packets).toHaveLength(1);
+		expect(second.packets[0]?.packet).toEqual(packet);
+		expect(second.packets[0]?.timestampUs).toBe(113_500);
+	});
+});

package/examples/streaming/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "bundler",
+    "jsx": "react-jsx",
+    "strict": true,
+    "esModuleInterop": true,
+    "types": ["bun-types"],
+    "skipLibCheck": true,
+    "noEmit": true,
+    "paths": {
+      "@jadujoel/web-audio-clip-node": ["../../src/lib"],
+      "@jadujoel/web-audio-clip-node/react": ["../../src/lib-react"],
+      "@jadujoel/web-audio-clip-node/styles.css": ["../../src/styles.css"]
+    }
+  },
+  "include": ["src/**/*", "*.ts", "generated/**/*"]
+}