@graineai/inapp-react-native 0.2.0 → 0.3.1

package/README.md

@@ -130,6 +130,17 @@ particular native audio module into your build. `liveAudioStreamAdapter` wraps
 
 Capture must be **16 kHz mono PCM16**. Interruption is handled for you.
 
+**Buffer before you play.** A player that plays each frame as it arrives will
+break up: a WebSocket gives no timing guarantee, so the gap between frames
+varies with the network and a player with no cushion runs dry in that gap. Hold
+**at least 150ms** (`MIN_PLAYOUT_BUFFER_SECONDS`) before starting, and when the
+queue does run dry, refill to that again before resuming — otherwise one hiccup
+becomes continuous stuttering for the rest of the turn.
+
+Report what you are holding through `bufferedSeconds()` and the SDK will
+acknowledge the agent's audio at playout rather than on arrival, which is what
+the runtime uses to know when it has finished speaking.
+
 ## Privacy
 
 Identifiers are masked on the device before any screen data is sent — PAN,

package/dist/audio.d.ts

@@ -0,0 +1,17 @@
+export interface AudioFormat {
+    kind: "mulaw" | "pcm";
+    sampleRate: number;
+}
+export declare function decodeMuLaw(bytes: Uint8Array): Int16Array;
+export declare function decodePcm16(bytes: Uint8Array): Int16Array;
+export interface DecodedAudio {
+    pcm16: Int16Array;
+    sampleRate: number;
+    kind: "mulaw" | "pcm";
+    latch: AudioFormat | null;
+}
+export declare function decodeAgentAudio(bytes: Uint8Array, declared: {
+    format?: string | null;
+    sampleRate?: number | null;
+} | null, latched: AudioFormat | null, fallbackPcmRate?: number): DecodedAudio | null;
+export declare function base64ToBytes(b64: string): Uint8Array;

package/dist/audio.js

@@ -0,0 +1,82 @@
+export function decodeMuLaw(bytes) {
+    const out = new Int16Array(bytes.length);
+    const BIAS = 0x84;
+    for (let i = 0; i < bytes.length; i++) {
+        const mu = ~bytes[i] & 0xff;
+        const sign = mu & 0x80;
+        const exponent = (mu >> 4) & 0x07;
+        const mantissa = mu & 0x0f;
+        const t = ((mantissa << 3) + BIAS) << exponent;
+        out[i] = sign ? BIAS - t : t - BIAS;
+    }
+    return out;
+}
+export function decodePcm16(bytes) {
+    const out = new Int16Array(bytes.length >> 1);
+    for (let i = 0; i < out.length; i++) {
+        out[i] = ((bytes[i * 2] | (bytes[i * 2 + 1] << 8)) << 16) >> 16;
+    }
+    return out;
+}
+function zeroCrossingRate(pcm) {
+    if (pcm.length < 2)
+        return 1;
+    let crossings = 0;
+    for (let i = 1; i < pcm.length; i++) {
+        if ((pcm[i] < 0) !== (pcm[i - 1] < 0))
+            crossings++;
+    }
+    return crossings / (pcm.length - 1);
+}
+function rms(pcm) {
+    if (pcm.length === 0)
+        return 0;
+    let sum = 0;
+    for (let i = 0; i < pcm.length; i++)
+        sum += pcm[i] * pcm[i];
+    return Math.sqrt(sum / pcm.length);
+}
+const CONFIDENT_RMS = 500;
+export function decodeAgentAudio(bytes, declared, latched, fallbackPcmRate = 16000) {
+    if (bytes.length === 0)
+        return null;
+    const fmt = declared?.format?.toLowerCase();
+    if (fmt === "mulaw" || fmt === "ulaw" || fmt === "g711_ulaw" || fmt === "pcmu") {
+        const sampleRate = declared?.sampleRate || 8000;
+        return { pcm16: decodeMuLaw(bytes), sampleRate, kind: "mulaw", latch: { kind: "mulaw", sampleRate } };
+    }
+    if (fmt === "pcm" || fmt === "linear16" || fmt === "raw") {
+        const sampleRate = declared?.sampleRate || fallbackPcmRate;
+        return { pcm16: decodePcm16(bytes), sampleRate, kind: "pcm", latch: { kind: "pcm", sampleRate } };
+    }
+    if (latched) {
+        return latched.kind === "mulaw"
+            ? { pcm16: decodeMuLaw(bytes), sampleRate: latched.sampleRate, kind: "mulaw", latch: latched }
+            : { pcm16: decodePcm16(bytes), sampleRate: latched.sampleRate, kind: "pcm", latch: latched };
+    }
+    const asMuLaw = decodeMuLaw(bytes);
+    const asPcm = decodePcm16(bytes);
+    const muLawWins = zeroCrossingRate(asMuLaw) <= zeroCrossingRate(asPcm);
+    const pcm16 = muLawWins ? asMuLaw : asPcm;
+    const kind = muLawWins ? "mulaw" : "pcm";
+    const sampleRate = muLawWins ? 8000 : declared?.sampleRate || fallbackPcmRate;
+    const confident = rms(pcm16) >= CONFIDENT_RMS;
+    return { pcm16, sampleRate, kind, latch: confident ? { kind, sampleRate } : null };
+}
+const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+export function base64ToBytes(b64) {
+    const clean = b64.replace(/[^A-Za-z0-9+/]/g, "");
+    const bytes = new Uint8Array((clean.length * 3) >> 2);
+    let byte = 0;
+    let acc = 0;
+    let bits = 0;
+    for (let i = 0; i < clean.length; i++) {
+        acc = (acc << 6) | B64.indexOf(clean[i]);
+        bits += 6;
+        if (bits >= 8) {
+            bits -= 8;
+            bytes[byte++] = (acc >> bits) & 0xff;
+        }
+    }
+    return bytes.subarray(0, byte);
+}

package/dist/client.d.ts

@@ -35,6 +35,8 @@ export declare class GraineInAppClient {
     private pingTimer;
     private agentSpeaking;
     private bargedIn;
+    private muted;
+    private playoutClock;
     constructor(opts: GraineInAppOptions);
     on(event: string, fn: Listener): () => void;
     private emit;
@@ -56,6 +58,9 @@ export declare class GraineInAppClient {
     say(text: string): boolean;
     sendAudio(base64: string, sampleRate?: number): boolean;
     endUtterance(): boolean;
+    setMuted(muted: boolean): void;
+    get isMuted(): boolean;
+    setPlayoutClock(fn: (() => number) | null): void;
     get isAgentSpeaking(): boolean;
     submitWidget(widgetId: string, data: Record<string, unknown>, summary: string): void;
     private stopTimers;

package/dist/client.js

@@ -19,6 +19,8 @@ export class GraineInAppClient {
         this.pingTimer = null;
         this.agentSpeaking = false;
         this.bargedIn = false;
+        this.muted = false;
+        this.playoutClock = null;
         if (!opts?.publishableKey)
             throw new Error("[Graine] publishableKey is required.");
         if (!opts?.baseUrl)
@@ -69,7 +71,10 @@ export class GraineInAppClient {
             const res = await this.fetch(`${this.opts.baseUrl}/api/embed/ticket`, {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ publishableKey: this.opts.publishableKey }),
+                body: JSON.stringify({
+                    publishableKey: this.opts.publishableKey,
+                    mode: this.opts.voice ? "voice" : "text",
+                }),
             });
             if (!res.ok)
                 return null;
@@ -118,7 +123,11 @@ export class GraineInAppClient {
         };
         this.send({
             type: "init",
-            meta_data: { context_data: { ...(this.config.variables ?? {}), ...(this.opts.variables ?? {}) } },
+            event: "start",
+            meta_data: {
+                client: "graine-inapp-sdk",
+                context_data: { ...(this.config.variables ?? {}), ...(this.opts.variables ?? {}) },
+            },
         });
         this.pingTimer = setInterval(() => this.send({ type: "ping" }), PING_MS);
         if (this.screen)
@@ -158,7 +167,8 @@ export class GraineInAppClient {
             case "audio":
                 this.emit("audio", {
                     data: msg.data,
-                    sampleRate: msg.meta_info?.sample_rate ?? 16000,
+                    format: msg.meta_info?.format ?? null,
+                    sampleRate: msg.meta_info?.sample_rate ?? null,
                 });
                 break;
             case "beginning_of_stream":
@@ -180,6 +190,18 @@ export class GraineInAppClient {
             case "transcript":
                 this.emit("transcript", { text: msg.data ?? "", final: true });
                 break;
+            case "mark": {
+                const name = msg.name;
+                if (!name)
+                    break;
+                const waitMs = Math.min(Math.max(0, (this.playoutClock?.() ?? 0) * 1000), 10000);
+                const ack = () => this.send({ type: "mark", name });
+                if (waitMs <= 0)
+                    ack();
+                else
+                    setTimeout(ack, waitMs);
+                break;
+            }
             case "pong":
             case "ack":
                 break;
@@ -296,6 +318,8 @@ export class GraineInAppClient {
         return this.send({ type: "text", data: text });
     }
     sendAudio(base64, sampleRate = 16000) {
+        if (this.muted)
+            return false;
         if (this.agentSpeaking && !this.bargedIn) {
             this.bargedIn = true;
             this.send({ type: "barge_in" });
@@ -312,6 +336,18 @@ export class GraineInAppClient {
     endUtterance() {
         return this.send({ type: "utterance_end" });
     }
+    setMuted(muted) {
+        if (this.muted === muted)
+            return;
+        this.muted = muted;
+        this.emit("muted", muted);
+    }
+    get isMuted() {
+        return this.muted;
+    }
+    setPlayoutClock(fn) {
+        this.playoutClock = fn;
+    }
     get isAgentSpeaking() {
         return this.agentSpeaking;
     }
@@ -328,11 +364,22 @@ export class GraineInAppClient {
     }
     close() {
         this.stopTimers();
+        const sock = this.ws;
+        this.ws = null;
+        if (!sock)
+            return;
         try {
-            this.ws?.close();
+            if (sock.readyState === 1) {
+                sock.send(JSON.stringify({ type: "stop", event: "stop", data: "client_hangup" }));
+                setTimeout(() => { try {
+                    sock.close();
+                }
+                catch { } }, 200);
+                return;
+            }
+            sock.close();
         }
         catch {
         }
-        this.ws = null;
     }
 }

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export type { ActionHandler, AppActionRequest, AppActionResult, AppEvent, ScreenContext, ScreenField, } from "./protocol";
 export { GraineInAppClient, type GraineInAppOptions, type SessionConfig } from "./client";
 export { VoiceSession, liveAudioStreamAdapter, base64ToPcm16, CAPTURE_SAMPLE_RATE, type AudioAdapter, type VoiceSessionOptions, } from "./voice";
+export { decodeAgentAudio, decodeMuLaw, decodePcm16, base64ToBytes, type AudioFormat, type DecodedAudio, } from "./audio";
 export { addMaskRule, maskDeep, maskString } from "./mask";
-export { GraineProvider, useGraineAgent, useGraineScreen, useGraineAction, useGraineVoice, type GraineProviderProps, type Turn, } from "./react-native/index";
+export { GraineProvider, useGraineAgent, useGraineScreen, useGraineAction, useGraineVoice, type GraineProviderProps, type Turn, type Caption, } from "./react-native/index";
 export { GraineAgentBar, GraineLauncher, type GraineAgentBarProps, type GraineLauncherProps } from "./react-native/ui";

package/dist/index.js

@@ -1,5 +1,6 @@
 export { GraineInAppClient } from "./client";
 export { VoiceSession, liveAudioStreamAdapter, base64ToPcm16, CAPTURE_SAMPLE_RATE, } from "./voice";
+export { decodeAgentAudio, decodeMuLaw, decodePcm16, base64ToBytes, } from "./audio";
 export { addMaskRule, maskDeep, maskString } from "./mask";
 export { GraineProvider, useGraineAgent, useGraineScreen, useGraineAction, useGraineVoice, } from "./react-native/index";
 export { GraineAgentBar, GraineLauncher } from "./react-native/ui";

package/dist/react-native/index.d.ts

@@ -12,6 +12,15 @@ interface GraineContextValue {
     messages: Turn[];
     widgets: any[];
     send: (text: string) => void;
+    muted: boolean;
+    setMuted: (muted: boolean) => void;
+    agentSpeaking: boolean;
+    caption: Caption | null;
+}
+export interface Caption {
+    role: "agent" | "customer";
+    text: string;
+    live: boolean;
 }
 export interface Turn {
     role: "agent" | "customer";

package/dist/react-native/index.js

@@ -14,6 +14,9 @@ export function GraineProvider({ children, autoConnect = true, onProactive, ...o
     const [open, setOpen] = useState(false);
     const [messages, setMessages] = useState([]);
     const [widgets, setWidgets] = useState([]);
+    const [muted, setMutedState] = useState(false);
+    const [agentSpeaking, setAgentSpeaking] = useState(false);
+    const [caption, setCaption] = useState(null);
     const proactiveRef = useRef(onProactive);
     proactiveRef.current = onProactive;
     const openRef = useRef(open);
@@ -39,6 +42,19 @@ export function GraineProvider({ children, autoConnect = true, onProactive, ...o
                 return [...prev, { role: "agent", text, live: true }];
             })),
             client.on("widget", (widget) => setWidgets((prev) => [...prev, widget])),
+            client.on("muted", (m) => setMutedState(m)),
+            client.on("agent_speaking", (speaking) => {
+                setAgentSpeaking(speaking);
+                if (speaking)
+                    setCaption(null);
+            }),
+            client.on("transcript", ({ text, final }) => {
+                if (text)
+                    setCaption({ role: "customer", text, live: !final });
+            }),
+            client.on("chunk", (text) => setCaption((prev) => prev && prev.role === "agent" && prev.live
+                ? { ...prev, text: prev.text + text }
+                : { role: "agent", text, live: true })),
         ];
         if (autoConnect) {
             setConnecting(true);
@@ -62,6 +78,10 @@ export function GraineProvider({ children, autoConnect = true, onProactive, ...o
         setOpen,
         messages,
         widgets,
+        muted,
+        agentSpeaking,
+        caption,
+        setMuted: (m) => client.setMuted(m),
         send: (text) => {
             if (!text.trim())
                 return;
@@ -72,7 +92,7 @@ export function GraineProvider({ children, autoConnect = true, onProactive, ...o
             client.noteInteraction();
             client.say(text);
         },
-    }), [client, connected, connecting, error, open, messages, widgets]);
+    }), [client, connected, connecting, error, open, messages, widgets, muted, agentSpeaking, caption]);
     return _jsx(Ctx.Provider, { value: value, children: children });
 }
 function useGraine() {

package/dist/react-native/ui.d.ts

@@ -2,9 +2,11 @@ import React from "react";
 export interface GraineAgentBarProps {
     accent?: string;
     name?: string;
+    avatarUrl?: string;
     bottomInset?: number;
     hidden?: boolean;
+    captionsOn?: boolean;
 }
-export declare function GraineAgentBar({ accent, name, bottomInset, hidden, }: GraineAgentBarProps): React.JSX.Element | null;
+export declare function GraineAgentBar({ accent, name, avatarUrl, bottomInset, hidden, captionsOn, }: GraineAgentBarProps): React.JSX.Element | null;
 export declare const GraineLauncher: typeof GraineAgentBar;
 export type GraineLauncherProps = GraineAgentBarProps;

package/dist/react-native/ui.js

@@ -1,19 +1,21 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useCallback, useEffect, useRef, useState } from "react";
-import { ActivityIndicator, KeyboardAvoidingView, Platform, Pressable, ScrollView, StyleSheet, Text, TextInput, View, } from "react-native";
+import { ActivityIndicator, Image, KeyboardAvoidingView, Platform, Pressable, ScrollView, StyleSheet, Text, TextInput, View, } from "react-native";
 import { useGraineAgent } from "./index";
-export function GraineAgentBar({ accent = "#318CE7", name = "Assistant", bottomInset = 96, hidden = false, }) {
-    const { connected, connecting, error, messages, send } = useGraineAgent();
+export function GraineAgentBar({ accent = "#318CE7", name = "Assistant", avatarUrl, bottomInset = 96, hidden = false, captionsOn = false, }) {
+    const { connected, connecting, error, messages, send, muted, setMuted, caption, agentSpeaking } = useGraineAgent();
     const [expanded, setExpanded] = useState(false);
+    const [captions, setCaptions] = useState(captionsOn);
     const [draft, setDraft] = useState("");
     const scrollRef = useRef(null);
     const last = messages[messages.length - 1];
     const status = error
         ? "unavailable"
         : connecting ? "connecting…"
-            : last?.role === "agent" && last.live ? "speaking"
-                : connected ? "listening"
-                    : "starting…";
+            : muted ? "muted"
+                : agentSpeaking || (last?.role === "agent" && last.live) ? "speaking"
+                    : connected ? "listening"
+                        : "starting…";
     useEffect(() => {
         if (expanded)
             requestAnimationFrame(() => scrollRef.current?.scrollToEnd({ animated: true }));
@@ -34,7 +36,7 @@ export function GraineAgentBar({ accent = "#318CE7", name = "Assistant", bottomI
     return (_jsxs(View, { pointerEvents: "box-none", style: [styles.root, { bottom: bottomInset }], children: [expanded && (_jsxs(View, { style: styles.panel, children: [_jsxs(ScrollView, { ref: scrollRef, style: styles.transcript, contentContainerStyle: styles.transcriptInner, showsVerticalScrollIndicator: false, children: [messages.length === 0 && (_jsx(Text, { style: styles.empty, children: "I can see what you're working on \u2014 ask me anything, or I'll step in if you get stuck." })), messages.map((m, i) => (_jsx(View, { style: [
                                     styles.bubble,
                                     m.role === "agent" ? styles.agentBubble : [styles.customerBubble, { backgroundColor: accent }],
-                                ], children: _jsx(Text, { style: m.role === "agent" ? styles.agentText : styles.customerText, children: m.text }) }, i)))] }), _jsx(KeyboardAvoidingView, { behavior: Platform.OS === "ios" ? "padding" : undefined, children: _jsxs(View, { style: styles.composer, children: [_jsx(TextInput, { style: styles.input, value: draft, onChangeText: setDraft, placeholder: "Type a message", placeholderTextColor: "#8a8f98", onSubmitEditing: submit, returnKeyType: "send", blurOnSubmit: false }), _jsx(Pressable, { onPress: submit, disabled: !draft.trim(), accessibilityLabel: "Send", style: [styles.send, { backgroundColor: accent, opacity: draft.trim() ? 1 : 0.4 }], children: _jsx(Text, { style: styles.sendLabel, children: "\u2191" }) })] }) })] })), _jsxs(View, { style: styles.bar, children: [_jsx(View, { style: [styles.avatar, { backgroundColor: accent }], children: _jsx(Text, { style: styles.avatarGlyph, children: "\u2726" }) }), _jsxs(Pressable, { onPress: () => setExpanded((v) => !v), style: { flex: 1 }, accessibilityLabel: expanded ? "Collapse" : "Expand", children: [_jsx(Text, { style: styles.name, children: name }), _jsx(Text, { style: styles.status, children: status })] }), connecting ? _jsx(ActivityIndicator, { size: "small", color: accent }) : null, _jsx(Pressable, { onPress: () => setExpanded((v) => !v), hitSlop: 8, accessibilityLabel: expanded ? "Collapse" : "Expand", style: styles.round, children: _jsx(Text, { style: styles.chevron, children: expanded ? "⌄" : "⌃" }) })] })] }));
+                                ], children: _jsx(Text, { style: m.role === "agent" ? styles.agentText : styles.customerText, children: m.text }) }, i)))] }), _jsx(KeyboardAvoidingView, { behavior: Platform.OS === "ios" ? "padding" : undefined, children: _jsxs(View, { style: styles.composer, children: [_jsx(TextInput, { style: styles.input, value: draft, onChangeText: setDraft, placeholder: "Type a message", placeholderTextColor: "#8a8f98", onSubmitEditing: submit, returnKeyType: "send", blurOnSubmit: false }), _jsx(Pressable, { onPress: submit, disabled: !draft.trim(), accessibilityLabel: "Send", style: [styles.send, { backgroundColor: accent, opacity: draft.trim() ? 1 : 0.4 }], children: _jsx(Text, { style: styles.sendLabel, children: "\u2191" }) })] }) })] })), captions && caption?.text ? (_jsx(View, { style: styles.captionWrap, children: _jsx(Text, { numberOfLines: 3, style: [styles.caption, caption.live && styles.captionLive], children: caption.text }) })) : null, _jsxs(View, { style: styles.bar, children: [avatarUrl ? (_jsx(Image, { source: { uri: avatarUrl }, style: styles.avatarImage, accessibilityIgnoresInvertColors: true })) : (_jsx(View, { style: [styles.avatar, { backgroundColor: accent }], children: _jsx(Text, { style: styles.avatarGlyph, children: "\u2726" }) })), _jsxs(Pressable, { onPress: () => setExpanded((v) => !v), style: { flex: 1 }, accessibilityLabel: expanded ? "Collapse" : "Expand", children: [_jsx(Text, { style: styles.name, children: name }), _jsx(Text, { style: styles.status, children: status })] }), connecting ? _jsx(ActivityIndicator, { size: "small", color: accent }) : null, _jsx(Pressable, { onPress: () => setCaptions((v) => !v), hitSlop: 8, accessibilityLabel: captions ? "Hide captions" : "Show captions", style: [styles.round, captions && { backgroundColor: accent }], children: _jsx(Text, { style: [styles.glyph, captions && { color: "#fff" }], children: "CC" }) }), _jsx(Pressable, { onPress: () => setMuted(!muted), disabled: !connected, hitSlop: 8, accessibilityLabel: muted ? "Unmute" : "Mute", style: [styles.round, muted && styles.muted, !connected && { opacity: 0.4 }], children: _jsx(Text, { style: [styles.glyph, muted && { color: "#fff" }], children: muted ? "🔇" : "🎙" }) }), _jsx(Pressable, { onPress: () => setExpanded((v) => !v), hitSlop: 8, accessibilityLabel: expanded ? "Collapse" : "Expand", style: styles.round, children: _jsx(Text, { style: styles.chevron, children: expanded ? "⌄" : "⌃" }) })] })] }));
 }
 export const GraineLauncher = GraineAgentBar;
 const styles = StyleSheet.create({
@@ -65,6 +67,14 @@ const styles = StyleSheet.create({
         shadowColor: "#000", shadowOpacity: 0.3, shadowRadius: 16, shadowOffset: { width: 0, height: 6 }, elevation: 12,
     },
     avatar: { width: 38, height: 38, borderRadius: 19, alignItems: "center", justifyContent: "center" },
+    avatarImage: { width: 38, height: 38, borderRadius: 19, backgroundColor: "rgba(255,255,255,0.08)" },
+    captionWrap: {
+        backgroundColor: "rgba(0,0,0,0.72)", borderRadius: 16, paddingHorizontal: 14, paddingVertical: 10, marginBottom: 8,
+    },
+    caption: { color: "#fff", fontSize: 15, lineHeight: 21, fontWeight: "600" },
+    captionLive: { opacity: 0.8 },
+    glyph: { color: "#F2F3F5", fontSize: 13, fontWeight: "800" },
+    muted: { backgroundColor: "#E5484D" },
     avatarGlyph: { color: "#fff", fontSize: 17, lineHeight: 20 },
     name: { color: "#F2F3F5", fontSize: 15, fontWeight: "800" },
     status: { color: "#9aa0a6", fontSize: 12.5, marginTop: 1 },

package/dist/voice.d.ts

@@ -1,10 +1,12 @@
 import type { GraineInAppClient } from "./client";
 export declare const CAPTURE_SAMPLE_RATE = 16000;
+export declare const MIN_PLAYOUT_BUFFER_SECONDS = 0.15;
 export interface AudioAdapter {
     startCapture(onChunk: (base64: string) => void): Promise<void> | void;
     stopCapture(): Promise<void> | void;
     play(pcm16: Int16Array, sampleRate: number): void;
     clear(): void;
+    bufferedSeconds?(): number;
 }
 export declare function base64ToPcm16(b64: string): Int16Array;
 export interface VoiceSessionOptions {
@@ -17,6 +19,7 @@ export declare class VoiceSession {
     private options;
     private offs;
     private capturing;
+    private format;
     constructor(client: GraineInAppClient, adapter: AudioAdapter, options?: VoiceSessionOptions);
     get active(): boolean;
     start(): Promise<void>;

package/dist/voice.js

@@ -1,24 +1,8 @@
+import { base64ToBytes, decodeAgentAudio, decodePcm16 } from "./audio";
 export const CAPTURE_SAMPLE_RATE = 16000;
-const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+export const MIN_PLAYOUT_BUFFER_SECONDS = 0.15;
 export function base64ToPcm16(b64) {
-    const clean = b64.replace(/[^A-Za-z0-9+/]/g, "");
-    const bytes = new Uint8Array((clean.length * 3) >> 2);
-    let byte = 0;
-    let acc = 0;
-    let bits = 0;
-    for (let i = 0; i < clean.length; i++) {
-        acc = (acc << 6) | B64.indexOf(clean[i]);
-        bits += 6;
-        if (bits >= 8) {
-            bits -= 8;
-            bytes[byte++] = (acc >> bits) & 0xff;
-        }
-    }
-    const samples = new Int16Array(byte >> 1);
-    for (let i = 0; i < samples.length; i++) {
-        samples[i] = (bytes[i * 2] | (bytes[i * 2 + 1] << 8)) << 16 >> 16;
-    }
-    return samples;
+    return decodePcm16(base64ToBytes(b64));
 }
 export class VoiceSession {
     constructor(client, adapter, options = {}) {
@@ -27,6 +11,7 @@ export class VoiceSession {
         this.options = options;
         this.offs = [];
         this.capturing = false;
+        this.format = null;
     }
     get active() {
         return this.capturing;
@@ -36,8 +21,13 @@ export class VoiceSession {
             return;
         this.capturing = true;
         this.offs = [
-            this.client.on("audio", ({ data, sampleRate }) => {
-                this.adapter.play(base64ToPcm16(data), sampleRate);
+            this.client.on("audio", ({ data, format, sampleRate, }) => {
+                const decoded = decodeAgentAudio(base64ToBytes(data), { format, sampleRate }, this.format);
+                if (!decoded)
+                    return;
+                if (decoded.latch)
+                    this.format = decoded.latch;
+                this.adapter.play(decoded.pcm16, decoded.sampleRate);
             }),
             this.client.on("clear", () => {
                 this.adapter.clear();
@@ -50,12 +40,14 @@ export class VoiceSession {
                 this.options.onTranscript?.(text, final);
             }),
         ];
+        this.client.setPlayoutClock(() => this.adapter.bufferedSeconds?.() ?? 0);
         await this.adapter.startCapture((base64) => this.client.sendAudio(base64, CAPTURE_SAMPLE_RATE));
     }
     async stop() {
         if (!this.capturing)
             return;
         this.capturing = false;
+        this.format = null;
         this.offs.forEach((off) => off());
         this.offs = [];
         this.adapter.clear();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@graineai/inapp-react-native",
-  "version": "0.2.0",
-  "description": "Graine in-app agent for React Native \u2014 an agent that sees the screen your customer is on and can act on it.",
+  "version": "0.3.1",
+  "description": "Graine in-app agent for React Native — an agent that sees the screen your customer is on and can act on it.",
   "license": "MIT",
   "private": false,
   "main": "dist/index.js",