@kernl-sdk/react 0.0.1

package/dist/hooks/use-realtime.js

@@ -0,0 +1,78 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import { RealtimeSession, Context } from "kernl";
+/**
+ * React hook for managing a realtime voice session.
+ *
+ * Handles connection lifecycle, status updates, and cleanup on unmount.
+ *
+ * @example
+ * ```tsx
+ * const { status, connect, disconnect } = useRealtime(agent, {
+ *   model: openai.realtime("gpt-4o-realtime"),
+ *   channel,
+ *   ctx: { setCart },
+ * });
+ *
+ * const start = async () => {
+ *   const { credential } = await fetch("/api/credential").then(r => r.json());
+ *   await channel.init();
+ *   connect(credential);
+ * };
+ * ```
+ */
+export function useRealtime(agent, options) {
+    const [status, setStatus] = useState("disconnected");
+    const [muted, setMuted] = useState(false);
+    const sessionRef = useRef(null);
+    const connect = useCallback(async (input) => {
+        if (sessionRef.current)
+            return;
+        // Convert expiresAt to Date if needed
+        const expiresAt = typeof input.expiresAt === "string"
+            ? new Date(input.expiresAt)
+            : input.expiresAt;
+        const credential = input.kind === "token"
+            ? { kind: "token", token: input.token, expiresAt }
+            : { kind: "url", url: input.url, expiresAt };
+        const session = new RealtimeSession(agent, {
+            model: options.model,
+            credential,
+            channel: options.channel,
+            context: options.ctx
+                ? new Context("react", options.ctx)
+                : undefined,
+        });
+        // Ignore events from sessions we've already disconnected from
+        session.on("status", (s) => {
+            if (sessionRef.current === session) {
+                setStatus(s);
+            }
+        });
+        sessionRef.current = session;
+        await session.connect();
+    }, [agent, options.model, options.channel, options.ctx]);
+    const disconnect = useCallback(() => {
+        sessionRef.current?.close();
+        sessionRef.current = null;
+        setStatus("disconnected");
+        setMuted(false);
+    }, []);
+    const mute = useCallback(() => {
+        sessionRef.current?.mute();
+        setMuted(true);
+    }, []);
+    const unmute = useCallback(() => {
+        sessionRef.current?.unmute();
+        setMuted(false);
+    }, []);
+    const sendMessage = useCallback((text) => {
+        sessionRef.current?.sendMessage(text);
+    }, []);
+    // cleanup
+    useEffect(() => {
+        return () => {
+            sessionRef.current?.close();
+        };
+    }, []);
+    return { status, connect, disconnect, muted, mute, unmute, sendMessage };
+}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { useRealtime } from "./hooks/use-realtime.js";
+export type { UseRealtimeOptions, UseRealtimeReturn, CredentialInput, } from "./hooks/use-realtime.js";
+export { useBrowserAudio } from "./hooks/use-browser-audio.js";
+export type { UseBrowserAudioReturn } from "./hooks/use-browser-audio.js";
+export { LiveWaveform } from "./components/live-waveform.js";
+export type { LiveWaveformProps, AudioSource } from "./components/live-waveform.js";
+export { BrowserChannel } from "./lib/browser-channel.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,YAAY,EACV,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,YAAY,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAGvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAGjF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/index.js

@@ -0,0 +1,7 @@
+// hooks
+export { useRealtime } from "./hooks/use-realtime.js";
+export { useBrowserAudio } from "./hooks/use-browser-audio.js";
+// components
+export { LiveWaveform } from "./components/live-waveform.js";
+// lib
+export { BrowserChannel } from "./lib/browser-channel.js";

package/dist/lib/audio-capture-worklet.d.ts

@@ -0,0 +1,12 @@
+/**
+ * AudioWorklet processor for capturing and resampling audio.
+ *
+ * This runs on the audio rendering thread for low-latency processing.
+ * Resamples from device sample rate to target rate (24kHz for realtime API).
+ */
+/**
+ * Create a blob URL for the audio worklet processor.
+ * This allows loading the worklet without a separate file.
+ */
+export declare function createWorkletUrl(): string;
+//# sourceMappingURL=audio-capture-worklet.d.ts.map

package/dist/lib/audio-capture-worklet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-capture-worklet.d.ts","sourceRoot":"","sources":["../../src/lib/audio-capture-worklet.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAqEH;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAGzC"}

package/dist/lib/audio-capture-worklet.js

@@ -0,0 +1,80 @@
+/**
+ * AudioWorklet processor for capturing and resampling audio.
+ *
+ * This runs on the audio rendering thread for low-latency processing.
+ * Resamples from device sample rate to target rate (24kHz for realtime API).
+ */
+// This code runs inside an AudioWorkletGlobalScope
+const workletCode = `
+const TARGET_SAMPLE_RATE = 24000;
+
+class AudioCaptureProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.resampleBuffer = [];
+    this.resampleRatio = sampleRate / TARGET_SAMPLE_RATE;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || !input[0]) return true;
+
+    const inputData = input[0];
+
+    // Resample using linear interpolation
+    const resampled = this.resample(inputData);
+
+    // Convert to PCM16
+    const pcm16 = new Int16Array(resampled.length);
+    for (let i = 0; i < resampled.length; i++) {
+      const s = Math.max(-1, Math.min(1, resampled[i]));
+      pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+    }
+
+    // Send to main thread
+    this.port.postMessage({ pcm16: pcm16.buffer }, [pcm16.buffer]);
+
+    return true;
+  }
+
+  resample(input) {
+    // Add input to buffer
+    for (let i = 0; i < input.length; i++) {
+      this.resampleBuffer.push(input[i]);
+    }
+
+    // Calculate how many output samples we can produce
+    const outputLength = Math.floor(this.resampleBuffer.length / this.resampleRatio);
+    if (outputLength === 0) return new Float32Array(0);
+
+    const output = new Float32Array(outputLength);
+
+    for (let i = 0; i < outputLength; i++) {
+      const srcIndex = i * this.resampleRatio;
+      const srcIndexFloor = Math.floor(srcIndex);
+      const srcIndexCeil = Math.min(srcIndexFloor + 1, this.resampleBuffer.length - 1);
+      const t = srcIndex - srcIndexFloor;
+
+      // Linear interpolation
+      output[i] = this.resampleBuffer[srcIndexFloor] * (1 - t) +
+                  this.resampleBuffer[srcIndexCeil] * t;
+    }
+
+    // Remove consumed samples from buffer
+    const consumed = Math.floor(outputLength * this.resampleRatio);
+    this.resampleBuffer = this.resampleBuffer.slice(consumed);
+
+    return output;
+  }
+}
+
+registerProcessor('audio-capture-processor', AudioCaptureProcessor);
+`;
+/**
+ * Create a blob URL for the audio worklet processor.
+ * This allows loading the worklet without a separate file.
+ */
+export function createWorkletUrl() {
+    const blob = new Blob([workletCode], { type: "application/javascript" });
+    return URL.createObjectURL(blob);
+}

package/dist/lib/browser-channel.d.ts

@@ -0,0 +1,45 @@
+import type { RealtimeChannel, RealtimeChannelEvents } from "@kernl-sdk/protocol";
+import { Emitter } from "@kernl-sdk/shared";
+/**
+ * Browser-based audio channel for realtime voice sessions.
+ *
+ * Uses the standard wire format (24kHz PCM16 base64) for audio I/O.
+ * Captures microphone audio and plays received audio through Web Audio API.
+ * Resamples from device sample rate to wire format using AudioWorklet.
+ */
+export declare class BrowserChannel extends Emitter<RealtimeChannelEvents> implements RealtimeChannel {
+    private audioContext;
+    private mediaStream;
+    private workletNode;
+    private workletUrl;
+    private nextPlayTime;
+    private activeSources;
+    private _output;
+    private _input;
+    /**
+     * Initialize audio context and start capturing from the microphone.
+     */
+    init(): Promise<void>;
+    /**
+     * Analyser node for speaker output (model audio).
+     */
+    get output(): AnalyserNode | null;
+    /**
+     * Analyser node for mic input (user audio).
+     */
+    get input(): AnalyserNode | null;
+    /**
+     * Send audio to be played through speakers.
+     * Audio is in wire format (24kHz PCM16), Web Audio resamples to device rate.
+     */
+    sendAudio(audio: string): void;
+    /**
+     * Interrupt audio playback.
+     */
+    interrupt(): void;
+    /**
+     * Clean up resources.
+     */
+    close(): void;
+}
+//# sourceMappingURL=browser-channel.d.ts.map

package/dist/lib/browser-channel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-channel.d.ts","sourceRoot":"","sources":["../../src/lib/browser-channel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAiC,MAAM,mBAAmB,CAAC;AAU3E;;;;;;GAMG;AACH,qBAAa,cACX,SAAQ,OAAO,CAAC,qBAAqB,CACrC,YAAW,eAAe;IAE1B,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAA6B;IAE3C;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqD3B;;OAEG;IACH,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,GAAG,IAAI,CAE/B;IAED;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAkC9B;;OAEG;IACH,SAAS,IAAI,IAAI;IAYjB;;OAEG;IACH,KAAK,IAAI,IAAI;CAiBd"}

package/dist/lib/browser-channel.js

@@ -0,0 +1,144 @@
+import { Emitter, base64ToPcm16, pcm16ToFloat32 } from "@kernl-sdk/shared";
+import { createWorkletUrl } from "./audio-capture-worklet.js";
+/** Standard wire format sample rate for realtime audio (PCM16). */
+const WIRE_FORMAT_SAMPLE_RATE = 24000;
+/** Lookahead buffer to prevent gaps from network jitter (seconds). */
+const PLAYBACK_LOOKAHEAD = 0.05;
+/**
+ * Browser-based audio channel for realtime voice sessions.
+ *
+ * Uses the standard wire format (24kHz PCM16 base64) for audio I/O.
+ * Captures microphone audio and plays received audio through Web Audio API.
+ * Resamples from device sample rate to wire format using AudioWorklet.
+ */
+export class BrowserChannel extends Emitter {
+    audioContext = null;
+    mediaStream = null;
+    workletNode = null;
+    workletUrl = null;
+    nextPlayTime = 0;
+    activeSources = [];
+    _output = null;
+    _input = null;
+    /**
+     * Initialize audio context and start capturing from the microphone.
+     */
+    async init() {
+        this.audioContext = new AudioContext();
+        // resume AudioContext (required after user gesture in some browsers)
+        if (this.audioContext.state === "suspended") {
+            await this.audioContext.resume();
+        }
+        // get microphone stream
+        this.mediaStream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                echoCancellation: true,
+                noiseSuppression: true,
+                autoGainControl: true,
+            },
+        });
+        // Load AudioWorklet processor (resamples from device rate to wire format)
+        this.workletUrl = createWorkletUrl();
+        await this.audioContext.audioWorklet.addModule(this.workletUrl);
+        // Create worklet node
+        this.workletNode = new AudioWorkletNode(this.audioContext, "audio-capture-processor");
+        // Handle resampled PCM16 audio from worklet
+        this.workletNode.port.onmessage = (event) => {
+            const pcm16 = new Int16Array(event.data.pcm16);
+            if (pcm16.length === 0)
+                return;
+            const base64 = base64ToPcm16.decode(pcm16);
+            this.emit("audio", base64);
+        };
+        // Create input analyser for mic visualization
+        this._input = this.audioContext.createAnalyser();
+        this._input.fftSize = 256;
+        this._input.smoothingTimeConstant = 0.5;
+        // Connect: mic → input analyser (for viz) and mic → worklet (for sending)
+        const source = this.audioContext.createMediaStreamSource(this.mediaStream);
+        source.connect(this._input);
+        source.connect(this.workletNode);
+        // Create output analyser for visualization
+        this._output = this.audioContext.createAnalyser();
+        this._output.fftSize = 256;
+        this._output.smoothingTimeConstant = 0.8;
+        this._output.connect(this.audioContext.destination);
+    }
+    /**
+     * Analyser node for speaker output (model audio).
+     */
+    get output() {
+        return this._output;
+    }
+    /**
+     * Analyser node for mic input (user audio).
+     */
+    get input() {
+        return this._input;
+    }
+    /**
+     * Send audio to be played through speakers.
+     * Audio is in wire format (24kHz PCM16), Web Audio resamples to device rate.
+     */
+    sendAudio(audio) {
+        if (!this.audioContext || !this._output)
+            return;
+        const pcm16 = base64ToPcm16.encode(audio);
+        const float32 = pcm16ToFloat32.encode(pcm16);
+        // Create buffer at wire format rate - Web Audio resamples automatically
+        const buffer = this.audioContext.createBuffer(1, float32.length, WIRE_FORMAT_SAMPLE_RATE);
+        buffer.getChannelData(0).set(float32);
+        const source = this.audioContext.createBufferSource();
+        source.buffer = buffer;
+        // Route through analyser for visualization (analyser → destination)
+        source.connect(this._output);
+        // Track source for interruption
+        this.activeSources.push(source);
+        source.onended = () => {
+            const idx = this.activeSources.indexOf(source);
+            if (idx !== -1)
+                this.activeSources.splice(idx, 1);
+        };
+        // Schedule playback with lookahead to prevent gaps from network jitter
+        const now = this.audioContext.currentTime;
+        const minStartTime = now + PLAYBACK_LOOKAHEAD;
+        const startTime = Math.max(minStartTime, this.nextPlayTime);
+        source.start(startTime);
+        this.nextPlayTime = startTime + buffer.duration;
+    }
+    /**
+     * Interrupt audio playback.
+     */
+    interrupt() {
+        for (const source of this.activeSources) {
+            try {
+                source.stop();
+            }
+            catch {
+                // Already stopped
+            }
+        }
+        this.activeSources = [];
+        this.nextPlayTime = 0;
+    }
+    /**
+     * Clean up resources.
+     */
+    close() {
+        this.interrupt();
+        this.workletNode?.disconnect();
+        this.workletNode = null;
+        this._output?.disconnect();
+        this._output = null;
+        this._input?.disconnect();
+        this._input = null;
+        if (this.workletUrl) {
+            URL.revokeObjectURL(this.workletUrl);
+            this.workletUrl = null;
+        }
+        this.mediaStream?.getTracks().forEach((track) => track.stop());
+        this.mediaStream = null;
+        this.audioContext?.close();
+        this.audioContext = null;
+    }
+}

package/dist/use-realtime.d.ts

@@ -0,0 +1,68 @@
+import { RealtimeAgent } from "kernl";
+import type { RealtimeModel, ClientCredential, TransportStatus } from "@kernl-sdk/protocol";
+/**
+ * Options for the useRealtime hook.
+ */
+export interface UseRealtimeOptions<TContext> {
+    /**
+     * The realtime model to use.
+     */
+    model: RealtimeModel;
+    /**
+     * Ephemeral credential from model.authenticate().
+     */
+    credential: ClientCredential;
+    /**
+     * Context passed to tool executions.
+     */
+    ctx?: TContext;
+}
+/**
+ * Return value from the useRealtime hook.
+ */
+export interface UseRealtimeReturn {
+    /**
+     * Current connection status.
+     */
+    status: TransportStatus;
+    /**
+     * Connect to the realtime model.
+     */
+    connect: () => Promise<void>;
+    /**
+     * Disconnect from the realtime model.
+     */
+    disconnect: () => void;
+    /**
+     * Whether audio input is muted.
+     */
+    muted: boolean;
+    /**
+     * Mute audio input.
+     */
+    mute: () => void;
+    /**
+     * Unmute audio input.
+     */
+    unmute: () => void;
+    /**
+     * Send a text message to the model.
+     */
+    sendMessage: (text: string) => void;
+}
+/**
+ * React hook for managing a realtime voice session.
+ *
+ * Handles connection lifecycle, status updates, and cleanup on unmount.
+ *
+ * @example
+ * ```tsx
+ * const { status, connect, mute, unmute } = useRealtime(agent, {
+ *   model: openai.realtime("gpt-4o-realtime"),
+ *   credential,
+ *   ctx: { setCart },
+ * });
+ * ```
+ */
+export declare function useRealtime<TContext>(agent: RealtimeAgent<TContext>, options: UseRealtimeOptions<TContext>): UseRealtimeReturn;
+//# sourceMappingURL=use-realtime.d.ts.map

package/dist/use-realtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-realtime.d.ts","sourceRoot":"","sources":["../src/use-realtime.ts"],"names":[],"mappings":"AACA,OAAO,EAAmB,aAAa,EAAW,MAAM,OAAO,CAAC;AAChE,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,QAAQ;IAC1C;;OAEG;IACH,KAAK,EAAE,aAAa,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,gBAAgB,CAAC;IAE7B;;OAEG;IACH,GAAG,CAAC,EAAE,QAAQ,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC;IAExB;;OAEG;IACH,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7B;;OAEG;IACH,UAAU,EAAE,MAAM,IAAI,CAAC;IAEvB;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,IAAI,EAAE,MAAM,IAAI,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,IAAI,CAAC;IAEnB;;OAEG;IACH,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACrC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAClC,KAAK,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC9B,OAAO,EAAE,kBAAkB,CAAC,QAAQ,CAAC,GACpC,iBAAiB,CAqDnB"}

package/dist/use-realtime.js

@@ -0,0 +1,60 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import { RealtimeSession, Context } from "kernl";
+/**
+ * React hook for managing a realtime voice session.
+ *
+ * Handles connection lifecycle, status updates, and cleanup on unmount.
+ *
+ * @example
+ * ```tsx
+ * const { status, connect, mute, unmute } = useRealtime(agent, {
+ *   model: openai.realtime("gpt-4o-realtime"),
+ *   credential,
+ *   ctx: { setCart },
+ * });
+ * ```
+ */
+export function useRealtime(agent, options) {
+    const [status, setStatus] = useState("disconnected");
+    const [muted, setMuted] = useState(false);
+    const sessionRef = useRef(null);
+    const connect = useCallback(async () => {
+        if (sessionRef.current) {
+            return;
+        }
+        const session = new RealtimeSession(agent, {
+            model: options.model,
+            credential: options.credential,
+            context: options.ctx
+                ? new Context("react", options.ctx)
+                : undefined,
+        });
+        session.on("status", setStatus);
+        sessionRef.current = session;
+        await session.connect();
+    }, [agent, options.model, options.credential, options.ctx]);
+    const disconnect = useCallback(() => {
+        sessionRef.current?.close();
+        sessionRef.current = null;
+        setStatus("disconnected");
+        setMuted(false);
+    }, []);
+    const mute = useCallback(() => {
+        sessionRef.current?.mute();
+        setMuted(true);
+    }, []);
+    const unmute = useCallback(() => {
+        sessionRef.current?.unmute();
+        setMuted(false);
+    }, []);
+    const sendMessage = useCallback((text) => {
+        sessionRef.current?.sendMessage(text);
+    }, []);
+    // cleanup
+    useEffect(() => {
+        return () => {
+            sessionRef.current?.close();
+        };
+    }, []);
+    return { status, connect, disconnect, muted, mute, unmute, sendMessage };
+}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@kernl-sdk/react",
+  "version": "0.0.1",
+  "description": "React bindings for kernl",
+  "keywords": [
+    "kernl",
+    "react",
+    "ai",
+    "realtime",
+    "voice"
+  ],
+  "author": "dremnik",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/kernl-sdk/kernl.git",
+    "directory": "packages/react"
+  },
+  "homepage": "https://github.com/kernl-sdk/kernl#readme",
+  "bugs": {
+    "url": "https://github.com/kernl-sdk/kernl/issues"
+  },
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc && tsc-alias --resolve-full-paths",
+    "dev": "tsc --watch",
+    "lint": "eslint src/",
+    "check-types": "tsc --noEmit"
+  },
+  "dependencies": {
+    "kernl": "workspace:*",
+    "@kernl-sdk/protocol": "workspace:*",
+    "@kernl-sdk/shared": "workspace:*"
+  },
+  "peerDependencies": {
+    "react": "^18 || ^19"
+  },
+  "devDependencies": {
+    "@types/node": "^24.10.0",
+    "@types/react": "^19.1.8",
+    "react": "^19.1.0",
+    "tsc-alias": "^1.8.10",
+    "typescript": "5.9.2"
+  }
+}