gemi 0.133.0 → 0.134.0

package/dist/ai/voice/index.js

@@ -428,6 +428,606 @@ function useVoiceSession(path, params = {}) {
 	};
 }
 //#endregion
-export { useVoiceSession };
+//#region ai/voice/energyVad.ts
+var DEFAULT_SILENCE_MS = 350;
+function createEnergyVad(options = {}) {
+	const silenceMs = options.silenceMs ?? 350;
+	const minSpeechMs = options.minSpeechMs ?? 120;
+	const minThreshold = options.threshold ?? .015;
+	const floorRatio = options.floorRatio ?? 3;
+	let floor = minThreshold / floorRatio;
+	let speaking = false;
+	/** When the current run of loud frames began, before it counts as speech. */
+	let loudSince = null;
+	/** When the current run of quiet frames began, while speaking. */
+	let quietSince = null;
+	return {
+		get speaking() {
+			return speaking;
+		},
+		reset() {
+			speaking = false;
+			loudSince = null;
+			quietSince = null;
+		},
+		push(rms, at) {
+			const threshold = Math.max(minThreshold, floor * floorRatio);
+			if (!speaking) {
+				if (rms > threshold) {
+					loudSince ??= at;
+					if (at - loudSince >= minSpeechMs) {
+						speaking = true;
+						quietSince = null;
+						return "start";
+					}
+				} else {
+					loudSince = null;
+					floor = rms < floor ? floor * .9 + rms * .1 : floor * .995 + rms * .005;
+				}
+				return null;
+			}
+			if (rms > threshold * .7) {
+				quietSince = null;
+				return null;
+			}
+			quietSince ??= at;
+			if (at - quietSince >= silenceMs) {
+				speaking = false;
+				loudSince = null;
+				quietSince = null;
+				return "end";
+			}
+			return null;
+		}
+	};
+}
+/** RMS of float time-domain samples. */
+function rmsOf(samples) {
+	let sum = 0;
+	for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i];
+	return samples.length ? Math.sqrt(sum / samples.length) : 0;
+}
+//#endregion
+//#region ai/voice/liveTranscription.ts
+/**
+* The browser's half of a live transcription session (#929): the microphone,
+* a raw `RTCPeerConnection` to the vendor, the `oai-events` data channel, a
+* level meter and end-of-speech detection that commits each turn.
+*
+* No vendor SDK, and nothing from `gemi/ai`'s server half: the secret (or the
+* SDP answer) comes from the app's own route, which holds the key.
+*
+* @experimental Follows OpenAI's realtime transcription and WebRTC guides; not
+* yet run against a live session.
+*/
+/** OpenAI's WebRTC endpoint, for a secret whose route doesn't say otherwise.
+*  `clientSecret()` on the server returns `callsUrl` (Azure's host included). */
+var OPENAI_REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
+var LiveTranscriptionFailure = class extends Error {
+	code;
+	status;
+	constructor(message, options = {}) {
+		super(message, { cause: options.cause });
+		this.name = "LiveTranscriptionError";
+		this.code = options.code;
+		this.status = options.status;
+	}
+};
+/** How often the level meter (and the built-in detector) reads the mic. */
+var FRAME_MS = 30;
+function microphoneError(error) {
+	const name = error?.name;
+	return name === "NotAllowedError" || name === "SecurityError" ? new LiveTranscriptionFailure("Microphone access was denied.", {
+		code: "microphone_denied",
+		cause: error
+	}) : new LiveTranscriptionFailure("No microphone is available.", {
+		code: "microphone_unavailable",
+		cause: error
+	});
+}
+async function routeError(response, what) {
+	const body = await response.json().catch(() => null);
+	return new LiveTranscriptionFailure(body?.error?.message ?? `${what} (${response.status}).`, {
+		status: response.status,
+		code: body?.error?.code
+	});
+}
+/**
+* Opens a live transcription session. Resolves once the SDP answer is applied;
+* `onOpen` follows when the data channel opens. `signal` aborts a start in
+* flight (unmount, `stop()` while connecting): everything opened so far is
+* closed and the promise rejects with an `AbortError`.
+*/
+async function startLiveTranscription(options, signal) {
+	if (typeof RTCPeerConnection === "undefined" || !navigator.mediaDevices?.getUserMedia) throw new LiveTranscriptionFailure("This browser can't stream audio.", { code: "unsupported" });
+	const connect = options.connect ?? "token";
+	const post = (url, payload) => fetch(url, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			...options.headers
+		},
+		body: JSON.stringify(payload),
+		signal
+	});
+	const tokenPromise = connect === "sdp" ? Promise.resolve(null) : typeof options.source === "function" ? options.source() : post(options.source, options.body ?? {}).then(async (response) => {
+		if (!response.ok) throw await routeError(response, "Live transcription is unavailable");
+		return response.json();
+	});
+	tokenPromise.catch(() => void 0);
+	let stream;
+	try {
+		stream = await navigator.mediaDevices.getUserMedia({ audio: {
+			echoCancellation: true,
+			noiseSuppression: true,
+			autoGainControl: true
+		} });
+	} catch (error) {
+		throw microphoneError(error);
+	}
+	const track = stream.getAudioTracks()[0];
+	let pc;
+	let channel;
+	let audioContext;
+	let meter;
+	let detector;
+	let closed = false;
+	/** `onClose` is for a session that started; a failed start throws instead. */
+	let started = false;
+	/** Speech heard since the last commit: a commit with none is pointless. */
+	let heardSpeech = false;
+	const close = (error) => {
+		if (closed) return;
+		closed = true;
+		if (meter) clearInterval(meter);
+		try {
+			detector?.stop();
+		} catch {}
+		try {
+			channel?.close();
+		} catch {}
+		pc?.close();
+		for (const t of stream.getTracks()) t.stop();
+		audioContext?.close().catch(() => void 0);
+		if (started) options.onClose(error);
+	};
+	const send = (event) => {
+		if (closed || channel?.readyState !== "open") return false;
+		channel.send(JSON.stringify(event));
+		return true;
+	};
+	const commit = () => {
+		const sent = send({ type: "input_audio_buffer.commit" });
+		if (sent) heardSpeech = false;
+		return sent;
+	};
+	const speech = {
+		speechStart() {
+			if (closed) return;
+			heardSpeech = true;
+			options.onSpeechStart();
+		},
+		speechEnd() {
+			if (closed) return;
+			options.onSpeechEnd(heardSpeech ? commit() : false);
+		}
+	};
+	try {
+		if (!track) throw new LiveTranscriptionFailure("No microphone is available.", { code: "microphone_unavailable" });
+		pc = new RTCPeerConnection();
+		pc.addTrack(track, stream);
+		channel = pc.createDataChannel("oai-events");
+		channel.addEventListener("open", () => {
+			if (!closed) options.onOpen();
+		});
+		channel.addEventListener("message", (message) => {
+			let event;
+			try {
+				event = JSON.parse(String(message.data));
+			} catch {
+				return;
+			}
+			if (event && typeof event === "object" && typeof event.type === "string") options.onEvent(event);
+		});
+		channel.addEventListener("close", () => close());
+		const peer = pc;
+		peer.addEventListener("connectionstatechange", () => {
+			if (peer.connectionState === "failed") close(new LiveTranscriptionFailure("The transcription connection failed.", { code: "connection_failed" }));
+			if (peer.connectionState === "closed") close();
+		});
+		const offer = await pc.createOffer();
+		await pc.setLocalDescription(offer);
+		let answer;
+		if (connect === "sdp") {
+			if (typeof options.source !== "string") throw new TypeError("connect: \"sdp\" needs the route's URL as the source.");
+			const response = await post(options.source, {
+				...options.body,
+				sdp: offer.sdp
+			});
+			if (!response.ok) throw await routeError(response, "Live transcription is unavailable");
+			answer = String((await response.json()).sdp ?? "");
+		} else {
+			const token = await tokenPromise;
+			if (!token || typeof token.value !== "string") throw new LiveTranscriptionFailure("The token route answered no secret.", { code: "no_token" });
+			const response = await fetch(token.callsUrl ?? "https://api.openai.com/v1/realtime/calls", {
+				method: "POST",
+				headers: {
+					Authorization: `Bearer ${token.value}`,
+					"Content-Type": "application/sdp"
+				},
+				body: offer.sdp,
+				signal
+			});
+			if (!response.ok) throw await routeError(response, "The transcription session could not start");
+			answer = await response.text();
+		}
+		if (!answer) throw new LiveTranscriptionFailure("No SDP answer.", { code: "provider_error" });
+		if (signal?.aborted) throw signal.reason ?? new DOMException("Aborted", "AbortError");
+		await pc.setRemoteDescription({
+			type: "answer",
+			sdp: answer
+		});
+		const energy = options.vad === void 0 || options.vad === "energy" ? createEnergyVad(options.energy) : null;
+		if (typeof AudioContext !== "undefined") {
+			audioContext = new AudioContext();
+			audioContext.resume?.().catch(() => void 0);
+			const analyser = audioContext.createAnalyser();
+			analyser.fftSize = 1024;
+			audioContext.createMediaStreamSource(stream).connect(analyser);
+			const samples = new Float32Array(analyser.fftSize);
+			let lastLevel = -1;
+			meter = setInterval(() => {
+				analyser.getFloatTimeDomainData(samples);
+				const rms = track.enabled ? rmsOf(samples) : 0;
+				const level = Math.min(1, rms / .2);
+				if (Math.abs(level - lastLevel) >= .02) {
+					lastLevel = level;
+					options.onLevel(level);
+				}
+				const change = energy?.push(rms, Date.now());
+				if (change === "start") speech.speechStart();
+				else if (change === "end") speech.speechEnd();
+			}, FRAME_MS);
+		} else if (energy) throw new LiveTranscriptionFailure("This browser can't analyse audio.", { code: "unsupported" });
+		if (typeof options.vad === "function") detector = await options.vad(stream, speech);
+	} catch (error) {
+		close();
+		if (signal?.aborted || error?.name === "AbortError") throw error;
+		throw error instanceof LiveTranscriptionFailure ? error : new LiveTranscriptionFailure("Live transcription could not start.", { cause: error });
+	}
+	if (closed) throw new LiveTranscriptionFailure("The transcription connection closed while starting.", { code: "connection_failed" });
+	started = true;
+	return {
+		commit,
+		flush: () => heardSpeech || options.vad === false ? commit() : false,
+		send,
+		mute(on) {
+			if (track) track.enabled = !on;
+		},
+		close: () => close()
+	};
+}
+//#endregion
+//#region ai/voice/transcriptionState.ts
+function initialTranscriptionState() {
+	return {
+		order: [],
+		partials: {},
+		turns: [],
+		pendingCommits: 0
+	};
+}
+/** The live text: every unfinished item's text, in order. */
+function partialText(state) {
+	return state.order.map((id) => state.partials[id]).filter((text) => !!text).join(" ").trim();
+}
+/** Vendor error codes that are part of normal operation, not failures: a
+*  commit with nothing in the buffer (a manual `commit()` in silence). */
+var BENIGN_ERRORS = /* @__PURE__ */ new Set(["input_audio_buffer_commit_empty"]);
+function applyTranscriptionEvent(state, event) {
+	const itemId = typeof event.item_id === "string" ? event.item_id : void 0;
+	const seen = (id) => state.order.includes(id) ? state.order : [...state.order, id];
+	switch (event.type) {
+		case "conversation.item.input_audio_transcription.delta":
+			if (!itemId || typeof event.delta !== "string") break;
+			if (state.turns.some((turn) => turn.id === itemId)) break;
+			return {
+				state: {
+					...state,
+					order: seen(itemId),
+					partials: {
+						...state.partials,
+						[itemId]: (state.partials[itemId] ?? "") + event.delta
+					}
+				},
+				effects: []
+			};
+		case "input_audio_buffer.committed":
+			if (!itemId) break;
+			return {
+				state: {
+					...state,
+					order: seen(itemId)
+				},
+				effects: []
+			};
+		case "conversation.item.input_audio_transcription.completed": {
+			if (!itemId || state.turns.some((turn) => turn.id === itemId)) break;
+			const usage = event.usage;
+			const seconds = usage?.type === "duration" ? Number(usage.seconds) : NaN;
+			const turn = {
+				id: itemId,
+				text: typeof event.transcript === "string" ? event.transcript.trim() : "",
+				...Number.isFinite(seconds) ? { seconds } : {}
+			};
+			const order = seen(itemId);
+			const { [itemId]: _done, ...partials } = state.partials;
+			const all = [...state.turns, turn].sort((a, b) => order.indexOf(a.id) - order.indexOf(b.id));
+			const turns = all.slice(-50);
+			const dropped = new Set(all.slice(0, -50).map((t) => t.id));
+			return {
+				state: {
+					...state,
+					order: dropped.size ? order.filter((id) => !dropped.has(id)) : order,
+					partials,
+					turns,
+					pendingCommits: Math.max(0, state.pendingCommits - 1)
+				},
+				effects: [{
+					type: "final",
+					turn
+				}]
+			};
+		}
+		case "conversation.item.input_audio_transcription.failed": {
+			const { [itemId ?? ""]: _dropped, ...partials } = state.partials;
+			const error = event.error ?? {};
+			return {
+				state: {
+					...state,
+					partials,
+					pendingCommits: Math.max(0, state.pendingCommits - 1)
+				},
+				effects: [{
+					type: "error",
+					code: error.code,
+					message: error.message ?? "Transcription failed."
+				}]
+			};
+		}
+		case "error": {
+			const error = event.error ?? {};
+			if (error.code && BENIGN_ERRORS.has(error.code)) return {
+				state: {
+					...state,
+					pendingCommits: Math.max(0, state.pendingCommits - 1)
+				},
+				effects: []
+			};
+			return {
+				state,
+				effects: [{
+					type: "error",
+					code: error.code,
+					message: error.message ?? "Vendor error."
+				}]
+			};
+		}
+	}
+	return {
+		state,
+		effects: []
+	};
+}
+//#endregion
+//#region ai/voice/useLiveTranscription.ts
+/**
+* Live speech-to-text (#929): the microphone streams to a transcription
+* session (`gpt-live-transcribe` by default) over WebRTC. `partial` grows as
+* the user speaks; when they stop for `silenceMs`, the turn is committed and
+* its final transcript arrives in `turns` and `onFinal`. Nothing is spoken
+* back.
+*
+* `source` is the app's route that answers a client secret (from
+* `realtime.clientSecret({ session: { type: "transcription", … } })`), or a
+* function returning one. With `connect: "sdp"`, it is the route that takes
+* the SDP offer and answers `{ sdp }`.
+*
+* ```tsx
+* const live = useLiveTranscription("/api/voice/token", {
+*   onFinal: (turn) => route(turn.text),
+* });
+* <button onClick={() => (live.status === "idle" ? live.start() : live.stop())}>Voice</button>
+* <p>{live.partial}</p>
+* ```
+*
+* From `gemi/ai/voice`, so only the views that use it load it.
+*
+* @experimental Tested against faked browser APIs only.
+*/
+function useLiveTranscription(source, options = {}) {
+	const [status, setStatus] = useState("idle");
+	const [transcript, setTranscript] = useState(initialTranscriptionState);
+	const [speaking, setSpeaking] = useState(false);
+	const [level, setLevel] = useState(0);
+	const [muted, setMuted] = useState(false);
+	const [error, setError] = useState(null);
+	const latest = useRef({
+		source,
+		options
+	});
+	latest.current = {
+		source,
+		options
+	};
+	const stateRef = useRef(transcript);
+	const connectionRef = useRef(null);
+	const abortRef = useRef(null);
+	/** Resolves `stop({ flush })`'s wait when the last pending final arrives. */
+	const drainedRef = useRef(null);
+	const mounted = useRef(true);
+	const update = useCallback((next) => {
+		const before = partialText(stateRef.current);
+		stateRef.current = next;
+		if (!mounted.current) return;
+		setTranscript(next);
+		const after = partialText(next);
+		if (after !== before) latest.current.options.onPartial?.(after);
+	}, []);
+	const fail = useCallback((failure) => {
+		if (!mounted.current) return;
+		setError(failure);
+		latest.current.options.onError?.(failure);
+	}, []);
+	const teardown = useCallback(() => {
+		abortRef.current?.abort();
+		abortRef.current = null;
+		const connection = connectionRef.current;
+		connectionRef.current = null;
+		connection?.close();
+	}, []);
+	useEffect(() => {
+		mounted.current = true;
+		return () => {
+			mounted.current = false;
+			teardown();
+		};
+	}, [teardown]);
+	const start = useCallback(async () => {
+		if (connectionRef.current || abortRef.current) return;
+		const abort = new AbortController();
+		abortRef.current = abort;
+		setError(null);
+		setStatus("connecting");
+		const { source: src, options: opts } = latest.current;
+		let opened = false;
+		try {
+			const connection = await startLiveTranscription({
+				connect: opts.connect,
+				source: src,
+				body: opts.body,
+				headers: opts.headers,
+				vad: opts.vad,
+				energy: {
+					...opts.energy,
+					silenceMs: opts.silenceMs
+				},
+				onOpen: () => {
+					opened = true;
+					if (mounted.current && connectionRef.current) setStatus("listening");
+				},
+				onEvent: (event) => {
+					const { state, effects } = applyTranscriptionEvent(stateRef.current, event);
+					if (state !== stateRef.current) update(state);
+					for (const effect of effects) if (effect.type === "final") {
+						if (mounted.current) latest.current.options.onFinal?.(effect.turn);
+					} else fail(Object.assign(new Error(effect.message), { code: effect.code }));
+					if (stateRef.current.pendingCommits === 0) drainedRef.current?.();
+				},
+				onSpeechStart: () => {
+					if (!mounted.current) return;
+					setSpeaking(true);
+					latest.current.options.onSpeechStart?.();
+				},
+				onSpeechEnd: (committed) => {
+					if (committed) update({
+						...stateRef.current,
+						pendingCommits: stateRef.current.pendingCommits + 1
+					});
+					if (!mounted.current) return;
+					setSpeaking(false);
+					latest.current.options.onSpeechEnd?.();
+				},
+				onLevel: (value) => {
+					if (mounted.current) setLevel(value);
+				},
+				onClose: (closeError) => {
+					connectionRef.current = null;
+					drainedRef.current?.();
+					if (!mounted.current) return;
+					setStatus("idle");
+					setSpeaking(false);
+					setLevel(0);
+					if (closeError) fail(closeError);
+				}
+			}, abort.signal);
+			if (abort.signal.aborted) {
+				connection.close();
+				return;
+			}
+			connectionRef.current = connection;
+			connection.mute(muted);
+			if (opened && mounted.current) setStatus("listening");
+		} catch (failure) {
+			if (abort.signal.aborted) return;
+			if (mounted.current) setStatus("idle");
+			fail(failure instanceof Error ? failure : new Error(String(failure)));
+		} finally {
+			if (abortRef.current === abort) abortRef.current = null;
+		}
+	}, [
+		fail,
+		update,
+		muted
+	]);
+	const commit = useCallback(() => {
+		if (connectionRef.current?.commit()) update({
+			...stateRef.current,
+			pendingCommits: stateRef.current.pendingCommits + 1
+		});
+	}, [update]);
+	const stop = useCallback(async ({ flush = false, timeoutMs = 1500 } = {}) => {
+		const connection = connectionRef.current;
+		if (flush && connection) {
+			if (connection.flush()) update({
+				...stateRef.current,
+				pendingCommits: stateRef.current.pendingCommits + 1
+			});
+			if (stateRef.current.pendingCommits > 0) {
+				await new Promise((resolve) => {
+					const timer = setTimeout(resolve, timeoutMs);
+					drainedRef.current = () => {
+						clearTimeout(timer);
+						resolve();
+					};
+				});
+				drainedRef.current = null;
+			}
+		}
+		teardown();
+		if (mounted.current) {
+			setStatus("idle");
+			setSpeaking(false);
+			setLevel(0);
+		}
+	}, [teardown, update]);
+	const mute = useCallback((on) => {
+		setMuted((was) => {
+			const next = on ?? !was;
+			connectionRef.current?.mute(next);
+			return next;
+		});
+	}, []);
+	const reset = useCallback(() => update({
+		...initialTranscriptionState(),
+		pendingCommits: stateRef.current.pendingCommits
+	}), [update]);
+	return {
+		status,
+		partial: partialText(transcript),
+		turns: transcript.turns,
+		speaking,
+		level,
+		muted,
+		error,
+		start,
+		stop,
+		commit,
+		mute,
+		reset
+	};
+}
+//#endregion
+export { DEFAULT_SILENCE_MS, OPENAI_REALTIME_CALLS_URL, createEnergyVad, useLiveTranscription, useVoiceSession };
 
 //# sourceMappingURL=index.js.map