@kernl-sdk/react 0.0.1

package/src/hooks/use-realtime.ts

@@ -0,0 +1,173 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import { RealtimeSession, RealtimeAgent, Context } from "kernl";
+import type {
+  RealtimeModel,
+  RealtimeChannel,
+  ClientCredential,
+  TransportStatus,
+} from "@kernl-sdk/protocol";
+
+/**
+ * Credential input that accepts expiresAt as either Date or string.
+ * Derived from ClientCredential to stay in sync.
+ */
+type FlexibleExpiry<T> = T extends { expiresAt: Date }
+  ? Omit<T, "expiresAt"> & { expiresAt: Date | string }
+  : never;
+
+export type CredentialInput = FlexibleExpiry<ClientCredential>;
+
+/**
+ * Options for the useRealtime hook.
+ */
+export interface UseRealtimeOptions<TContext> {
+  /**
+   * The realtime model to use.
+   */
+  model: RealtimeModel;
+
+  /**
+   * Audio I/O channel for mic capture and playback.
+   */
+  channel?: RealtimeChannel;
+
+  /**
+   * Context passed to tool executions.
+   */
+  ctx?: TContext;
+}
+
+/**
+ * Return value from the useRealtime hook.
+ */
+export interface UseRealtimeReturn {
+  /**
+   * Current connection status.
+   */
+  status: TransportStatus;
+
+  /**
+   * Connect to the realtime model with the given credential.
+   */
+  connect: (credential: CredentialInput) => Promise<void>;
+
+  /**
+   * Disconnect from the realtime model.
+   */
+  disconnect: () => void;
+
+  /**
+   * Whether audio input is muted.
+   */
+  muted: boolean;
+
+  /**
+   * Mute audio input.
+   */
+  mute: () => void;
+
+  /**
+   * Unmute audio input.
+   */
+  unmute: () => void;
+
+  /**
+   * Send a text message to the model.
+   */
+  sendMessage: (text: string) => void;
+}
+
+/**
+ * React hook for managing a realtime voice session.
+ *
+ * Handles connection lifecycle, status updates, and cleanup on unmount.
+ *
+ * @example
+ * ```tsx
+ * const { status, connect, disconnect } = useRealtime(agent, {
+ *   model: openai.realtime("gpt-4o-realtime"),
+ *   channel,
+ *   ctx: { setCart },
+ * });
+ *
+ * const start = async () => {
+ *   const { credential } = await fetch("/api/credential").then(r => r.json());
+ *   await channel.init();
+ *   connect(credential);
+ * };
+ * ```
+ */
+export function useRealtime<TContext>(
+  agent: RealtimeAgent<TContext>,
+  options: UseRealtimeOptions<TContext>,
+): UseRealtimeReturn {
+  const [status, setStatus] = useState<TransportStatus>("disconnected");
+  const [muted, setMuted] = useState(false);
+  const sessionRef = useRef<RealtimeSession<TContext> | null>(null);
+
+  const connect = useCallback(
+    async (input: CredentialInput) => {
+      if (sessionRef.current) return;
+
+      // Convert expiresAt to Date if needed
+      const expiresAt =
+        typeof input.expiresAt === "string"
+          ? new Date(input.expiresAt)
+          : input.expiresAt;
+
+      const credential: ClientCredential =
+        input.kind === "token"
+          ? { kind: "token", token: input.token, expiresAt }
+          : { kind: "url", url: input.url, expiresAt };
+
+      const session = new RealtimeSession(agent, {
+        model: options.model,
+        credential,
+        channel: options.channel,
+        context: options.ctx
+          ? (new Context("react", options.ctx) as Context<TContext>)
+          : undefined,
+      });
+
+      // Ignore events from sessions we've already disconnected from
+      session.on("status", (s) => {
+        if (sessionRef.current === session) {
+          setStatus(s);
+        }
+      });
+      sessionRef.current = session;
+      await session.connect();
+    },
+    [agent, options.model, options.channel, options.ctx],
+  );
+
+  const disconnect = useCallback(() => {
+    sessionRef.current?.close();
+    sessionRef.current = null;
+    setStatus("disconnected");
+    setMuted(false);
+  }, []);
+
+  const mute = useCallback(() => {
+    sessionRef.current?.mute();
+    setMuted(true);
+  }, []);
+
+  const unmute = useCallback(() => {
+    sessionRef.current?.unmute();
+    setMuted(false);
+  }, []);
+
+  const sendMessage = useCallback((text: string) => {
+    sessionRef.current?.sendMessage(text);
+  }, []);
+
+  // cleanup
+  useEffect(() => {
+    return () => {
+      sessionRef.current?.close();
+    };
+  }, []);
+
+  return { status, connect, disconnect, muted, mute, unmute, sendMessage };
+}

package/src/index.ts

@@ -0,0 +1,17 @@
+// hooks
+export { useRealtime } from "./hooks/use-realtime";
+export type {
+  UseRealtimeOptions,
+  UseRealtimeReturn,
+  CredentialInput,
+} from "./hooks/use-realtime";
+
+export { useBrowserAudio } from "./hooks/use-browser-audio";
+export type { UseBrowserAudioReturn } from "./hooks/use-browser-audio";
+
+// components
+export { LiveWaveform } from "./components/live-waveform";
+export type { LiveWaveformProps, AudioSource } from "./components/live-waveform";
+
+// lib
+export { BrowserChannel } from "./lib/browser-channel";

package/src/lib/audio-capture-worklet.ts

@@ -0,0 +1,82 @@
+/**
+ * AudioWorklet processor for capturing and resampling audio.
+ *
+ * This runs on the audio rendering thread for low-latency processing.
+ * Resamples from device sample rate to target rate (24kHz for realtime API).
+ */
+
+// This code runs inside an AudioWorkletGlobalScope
+const workletCode = `
+const TARGET_SAMPLE_RATE = 24000;
+
+class AudioCaptureProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.resampleBuffer = [];
+    this.resampleRatio = sampleRate / TARGET_SAMPLE_RATE;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || !input[0]) return true;
+
+    const inputData = input[0];
+
+    // Resample using linear interpolation
+    const resampled = this.resample(inputData);
+
+    // Convert to PCM16
+    const pcm16 = new Int16Array(resampled.length);
+    for (let i = 0; i < resampled.length; i++) {
+      const s = Math.max(-1, Math.min(1, resampled[i]));
+      pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+    }
+
+    // Send to main thread
+    this.port.postMessage({ pcm16: pcm16.buffer }, [pcm16.buffer]);
+
+    return true;
+  }
+
+  resample(input) {
+    // Add input to buffer
+    for (let i = 0; i < input.length; i++) {
+      this.resampleBuffer.push(input[i]);
+    }
+
+    // Calculate how many output samples we can produce
+    const outputLength = Math.floor(this.resampleBuffer.length / this.resampleRatio);
+    if (outputLength === 0) return new Float32Array(0);
+
+    const output = new Float32Array(outputLength);
+
+    for (let i = 0; i < outputLength; i++) {
+      const srcIndex = i * this.resampleRatio;
+      const srcIndexFloor = Math.floor(srcIndex);
+      const srcIndexCeil = Math.min(srcIndexFloor + 1, this.resampleBuffer.length - 1);
+      const t = srcIndex - srcIndexFloor;
+
+      // Linear interpolation
+      output[i] = this.resampleBuffer[srcIndexFloor] * (1 - t) +
+                  this.resampleBuffer[srcIndexCeil] * t;
+    }
+
+    // Remove consumed samples from buffer
+    const consumed = Math.floor(outputLength * this.resampleRatio);
+    this.resampleBuffer = this.resampleBuffer.slice(consumed);
+
+    return output;
+  }
+}
+
+registerProcessor('audio-capture-processor', AudioCaptureProcessor);
+`;
+
+/**
+ * Create a blob URL for the audio worklet processor.
+ * This allows loading the worklet without a separate file.
+ */
+export function createWorkletUrl(): string {
+  const blob = new Blob([workletCode], { type: "application/javascript" });
+  return URL.createObjectURL(blob);
+}

package/src/lib/browser-channel.ts

@@ -0,0 +1,178 @@
+import type {
+  RealtimeChannel,
+  RealtimeChannelEvents,
+} from "@kernl-sdk/protocol";
+import { Emitter, base64ToPcm16, pcm16ToFloat32 } from "@kernl-sdk/shared";
+
+import { createWorkletUrl } from "./audio-capture-worklet";
+
+/** Standard wire format sample rate for realtime audio (PCM16). */
+const WIRE_FORMAT_SAMPLE_RATE = 24000;
+
+/** Lookahead buffer to prevent gaps from network jitter (seconds). */
+const PLAYBACK_LOOKAHEAD = 0.05;
+
+/**
+ * Browser-based audio channel for realtime voice sessions.
+ *
+ * Uses the standard wire format (24kHz PCM16 base64) for audio I/O.
+ * Captures microphone audio and plays received audio through Web Audio API.
+ * Resamples from device sample rate to wire format using AudioWorklet.
+ */
+export class BrowserChannel
+  extends Emitter<RealtimeChannelEvents>
+  implements RealtimeChannel
+{
+  private audioContext: AudioContext | null = null;
+  private mediaStream: MediaStream | null = null;
+  private workletNode: AudioWorkletNode | null = null;
+  private workletUrl: string | null = null;
+  private nextPlayTime = 0;
+  private activeSources: AudioBufferSourceNode[] = [];
+  private _output: AnalyserNode | null = null;
+  private _input: AnalyserNode | null = null;
+
+  /**
+   * Initialize audio context and start capturing from the microphone.
+   */
+  async init(): Promise<void> {
+    this.audioContext = new AudioContext();
+
+    // resume AudioContext (required after user gesture in some browsers)
+    if (this.audioContext.state === "suspended") {
+      await this.audioContext.resume();
+    }
+
+    // get microphone stream
+    this.mediaStream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      },
+    });
+
+    // Load AudioWorklet processor (resamples from device rate to wire format)
+    this.workletUrl = createWorkletUrl();
+    await this.audioContext.audioWorklet.addModule(this.workletUrl);
+
+    // Create worklet node
+    this.workletNode = new AudioWorkletNode(
+      this.audioContext,
+      "audio-capture-processor",
+    );
+
+    // Handle resampled PCM16 audio from worklet
+    this.workletNode.port.onmessage = (event) => {
+      const pcm16 = new Int16Array(event.data.pcm16);
+      if (pcm16.length === 0) return;
+
+      const base64 = base64ToPcm16.decode(pcm16);
+      this.emit("audio", base64);
+    };
+
+    // Create input analyser for mic visualization
+    this._input = this.audioContext.createAnalyser();
+    this._input.fftSize = 256;
+    this._input.smoothingTimeConstant = 0.5;
+
+    // Connect: mic → input analyser (for viz) and mic → worklet (for sending)
+    const source = this.audioContext.createMediaStreamSource(this.mediaStream);
+    source.connect(this._input);
+    source.connect(this.workletNode);
+
+    // Create output analyser for visualization
+    this._output = this.audioContext.createAnalyser();
+    this._output.fftSize = 256;
+    this._output.smoothingTimeConstant = 0.8;
+    this._output.connect(this.audioContext.destination);
+  }
+
+  /**
+   * Analyser node for speaker output (model audio).
+   */
+  get output(): AnalyserNode | null {
+    return this._output;
+  }
+
+  /**
+   * Analyser node for mic input (user audio).
+   */
+  get input(): AnalyserNode | null {
+    return this._input;
+  }
+
+  /**
+   * Send audio to be played through speakers.
+   * Audio is in wire format (24kHz PCM16), Web Audio resamples to device rate.
+   */
+  sendAudio(audio: string): void {
+    if (!this.audioContext || !this._output) return;
+
+    const pcm16 = base64ToPcm16.encode(audio);
+    const float32 = pcm16ToFloat32.encode(pcm16);
+
+    // Create buffer at wire format rate - Web Audio resamples automatically
+    const buffer = this.audioContext.createBuffer(
+      1,
+      float32.length,
+      WIRE_FORMAT_SAMPLE_RATE,
+    );
+    buffer.getChannelData(0).set(float32);
+
+    const source = this.audioContext.createBufferSource();
+    source.buffer = buffer;
+    // Route through analyser for visualization (analyser → destination)
+    source.connect(this._output);
+
+    // Track source for interruption
+    this.activeSources.push(source);
+    source.onended = () => {
+      const idx = this.activeSources.indexOf(source);
+      if (idx !== -1) this.activeSources.splice(idx, 1);
+    };
+
+    // Schedule playback with lookahead to prevent gaps from network jitter
+    const now = this.audioContext.currentTime;
+    const minStartTime = now + PLAYBACK_LOOKAHEAD;
+    const startTime = Math.max(minStartTime, this.nextPlayTime);
+    source.start(startTime);
+    this.nextPlayTime = startTime + buffer.duration;
+  }
+
+  /**
+   * Interrupt audio playback.
+   */
+  interrupt(): void {
+    for (const source of this.activeSources) {
+      try {
+        source.stop();
+      } catch {
+        // Already stopped
+      }
+    }
+    this.activeSources = [];
+    this.nextPlayTime = 0;
+  }
+
+  /**
+   * Clean up resources.
+   */
+  close(): void {
+    this.interrupt();
+    this.workletNode?.disconnect();
+    this.workletNode = null;
+    this._output?.disconnect();
+    this._output = null;
+    this._input?.disconnect();
+    this._input = null;
+    if (this.workletUrl) {
+      URL.revokeObjectURL(this.workletUrl);
+      this.workletUrl = null;
+    }
+    this.mediaStream?.getTracks().forEach((track) => track.stop());
+    this.mediaStream = null;
+    this.audioContext?.close();
+    this.audioContext = null;
+  }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": ".",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "jsx": "react-jsx",
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}